<commit_before>#!/usr/bin/python -Ott

import sys
from optparse import OptionParser, OptionError
import subprocess
import os
import re
import ConfigParser
import getpass
import OpenSSL
import datetime
import glob
import logging

# Declare variables
mirror_dir = '/mirror/upstream'
confd_dir = '/etc/upstream_sync'

# directory that contains authentication credentials for sles
sles_auth_cred_dir = '/etc/nccs/sles_mirror'

# directory to store generated repo confs
user = getpass.getuser()
tmp_dir = '/var/tmp/upstream_sync-{0}'.format(user)

def build_yum_config(name, url, sslcacert, sslcert, sslkey):
    # Check tmp path exist
    if not os.path.isdir(tmp_dir):
        os.mkdir(tmp_dir)
    repo_conf = os.path.join(tmp_dir, '{0}.repo'.format(name))

    f = open(repo_conf, 'w')
    f.write('[main]\n')
    f.write('reposdir=/dev/null\n')
    f.write('[{0}]\n'.format(name))
    f.write('name = {0}\n'.format(name))
    f.write('baseurl = {0}\n'.format(url))
    f.write('enabled = 1\n')
    f.write('gpgcheck = 0\n')

    if sslcacert and sslcert and sslkey:
        check_sslcert_expiration(sslcert)
        f.write('sslverify = 1\n')
        f.write('sslcacert = {0}\n'.format(sslcacert))
        f.write('sslclientcert = {0}\n'.format(sslcert))
        f.write('sslclientkey = {0}\n'.format(sslkey))

    f.write('metadata_expire = 60\n')

    f.close()

    return repo_conf

def check_sslcert_expiration(sslcert):
    """checks to see if the ssl cert is going to expire soon"""
    cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, file(sslcert).read())
    cert_expires = datetime.datetime.strptime(cert.get_notAfter(), "%Y%m%d%H%M%SZ")

    if datetime.datetime.now() > cert_expires:
        logging.warn('SSL Certificate (%s) expired on %s' % (sslcert, cert_expires))
    elif datetime.datetime.now()+datetime.timedelta(days=14) > cert_expires:
        logging.warn('SSL Certificate (%s) is going to expire on %s' % (sslcert, cert_expires))

    return

def get_repos(rfilter=None):
    """
    parse configuration files and return repos.

    if rfilter is set, only repos that match rfilter will be returned
    """
    defaults = {
        'createrepo': 'False',
        'copylinks': 'False',
        'exclude': '',
        'sslcacert': '',
        'sslcert': '',
        'sslkey': ''
    }
    config = ConfigParser.ConfigParser(defaults)
    config.read(glob.glob(os.path.join(confd_dir, '*.repo')))

    # sort all the repos by alpha order, ConfigParser return sections
    # in the order that they appear in the config file
    repos = []
    for title in config.sections():
        if rfilter and (title not in rfilter):
            continue
        repo = dict(config.items(title))
        repo['name'] = title
        repo['path'] = os.path.join(mirror_dir, repo['path'])  # absolute path of repository
        repos.append(repo)

    repos = sorted(repos, key=lambda k: k['name'])
    return repos

def get_auths():
    config = ConfigParser.ConfigParser()
    config.read(os.path.join(confd_dir, 'auth.conf'))

    auths = dict()

    for title in config.sections():
        items = dict(config.items(title))
        auths[title] = items

    return auths

def list_repos(repos):
    for repo in repos:
        print repo['name']

def sync_cmd_reposync(repo):
    sslcacert = None
    sslcert = None
    sslkey = None

    reposync_opts = []

    name = repo['name']
    url = repo['url']
    path = repo['path']

    if repo.has_key('auth'):
        auth = get_auths()[repo['auth']]
        sslcacert = auth['sslcacert']
        sslcert = auth['sslcert']
        sslkey = auth['sslkey']

    yum_conf = build_yum_config(name, url, sslcacert, sslcert, sslkey)

    reposync_opts.extend(('-c', yum_conf))
    reposync_opts.extend(('-r', name))
    reposync_opts.extend(('-p', path))

    # detect arch
    match_arch = re.match(r'.*(?:/|-)(ppc64|ppc64le|x86_64|i386|i686|armhfp|amd64|x86)(?:/|$)', url)
    if match_arch:
        arch = match_arch.groups()[0]
        if arch in ['i386', 'x86']:
            arch = 'i686'
        elif arch in ['amd64', 'x86_64']:
            arch = 'x86_64'
        reposync_opts.extend(('--arch', arch))
    else:
        logging.warn('unable to detect architecture for %s' % name)

    # build options
    if repo.has_key('sync_opts'):
        opt_list = repo['sync_opts'].split()
        for opt in opt_list:
            reposync_opts.append(opt)
    else:
        reposync_opts.append('--tempcache')
        reposync_opts.append('--norepopath')
        reposync_opts.append('--downloadcomps')
        reposync_opts.append('--newest-only')
        reposync_opts.append('--delete')

    # be quiet if verbose is not set
    if not verbose:
        reposync_opts.append('-q')

    sync_cmd = ['reposync'] + reposync_opts
    return sync_cmd

def sync_cmd_rhnget(repo):
    systemid = os.path.join(os.path.split(repo['path'])[0], 'systemid')
    if not os.path.isfile(systemid):
        logging.warn("rhn: can not find systemid (%s)" % systemid)
        return

    sync_cmd = ['rhnget', '-q', '-s', systemid, repo['url'], repo['path']]
    return sync_cmd

def sync_cmd_rsync(repo):
    rsync_opts = []
    # build options
    if repo.has_key('sync_opts'):
        opt_list = repo['sync_opts'].split()
        for opt in opt_list:
            rsync_opts.append(opt)
    else:
        rsync_opts.append('--no-motd')
        rsync_opts.append('--recursive')
        rsync_opts.append('--delete')
        rsync_opts.append('--times')
        rsync_opts.append('--contimeout=30')

    if repo['copylinks'].lower() == 'true':
        rsync_opts.append('--copy-links')

    exclude_list = repo['exclude'].split(',')
    for item in exclude_list:
        # split() will return an empty list element
        if item:
            rsync_opts.append('--exclude')
            rsync_opts.append(item)

    if verbose:
        rsync_opts.append('--itemize-changes')

    sync_cmd = ['rsync'] + rsync_opts + [repo['url'], repo['path']]
    return sync_cmd

def sync_cmd_you(repo):
    # checking for sles credentials
    deviceid = os.path.join(sles_auth_cred_dir, 'deviceid')
    secret = os.path.join(sles_auth_cred_dir, 'secret')

    if not os.path.isfile(deviceid):
        logging.warn("you: can not find deviceid file (%s)" % deviceid)
        return
    elif not os.path.isfile(secret):
        logging.warn('you: can not find secret file (%s)' % secret)
        return

    url = re.sub('^you://', 'https://', repo['url'])
    sync_cmd = ['/opt/bin/youget', '-q', '--source', '-d', sles_auth_cred_dir, '--delete', url, repo['path']]

    return sync_cmd

def main():
    """main subroutine"""

    parser = OptionParser()
    parser.add_option("-v", "--verbose", help="Be verbose", action="count")
    parser.add_option("-l", "--list", help="list repos", action="store_true")
    parser.add_option("-c", "--command", help="print the sync command", action="store_true")
    parser.add_option("-r", "--repos", help="syncs specific repo(s) (comma seperated list)", dest="rfilter")
    parser.add_option("--root", help="run script as root", action="store_true")
    try:
        (options, args) = parser.parse_args()
    except OptionError:
        parser.print_help()
        return 1

    global verbose
    if options.verbose:
        logging.basicConfig(level=logging.DEBUG)
        verbose = True
    else:
        logging.basicConfig(level=logging.WARNING)
        verbose = False

    show_command = options.command
    run_as_root = options.root

    # get repos from config
    if options.rfilter:
        repos = get_repos(rfilter=options.rfilter.split(','))
    else:
        repos = get_repos()

    # list repos
    if options.list:
        list_repos(repos)
        sys.exit(0)

    if not run_as_root:
        if os.geteuid() == 0:
            print("It is strongly advised not to run this script as root!")
            print("Running as root will most likely mess up filesystem permissions.")
            print("If you are sure you want to run as root, pass --root")
            sys.exit(2)

    for repo in repos:
        # set variables based on values in config
        url = repo['url']
        name = repo['name']
        path = os.path.join(mirror_dir, repo['path'])  # absolute path of repository

        # create repo directory
        if not show_command:
            if not os.path.isdir(path):
                os.makedirs(path, 0775)

        createrepo = False
        if repo['createrepo'].lower() == "true":
            createrepo = True

        # Generate the sync and createrepo commands to be used based on repository type
        createrepo_exec = ['createrepo']
        createrepo_opts = ['--pretty', '--database', '--update', '--cachedir', os.path.join(path, '.cache'), path]
        if not options.verbose:
            createrepo_opts.append('-q')

        if re.match('^(http|https)://', url):
            sync_cmd = sync_cmd_reposync(repo)
        elif re.match('^rhns:///', url):
            sync_cmd = sync_cmd_rhnget(repo)
        elif re.match('^you://', url):
            sync_cmd = sync_cmd_you(repo)
        elif re.match('^rsync://', url):
            sync_cmd = sync_cmd_rsync(repo)
        else:
            logging.warn('url type unknown - %s' % url)
            continue

        if not sync_cmd:
            continue

        # if option -c is passed print commands and continue to loop item
        if show_command:
            print('%s:' % name)
            print('  '+' '.join(sync_cmd))
            if createrepo:
                print('  '+' '.join(createrepo_exec + createrepo_opts))
            continue

        # preform sync - rhnget/rsync
        logging.info('syncing %s' % name)
        if options.verbose:
            stdout_pipe = sys.stdout
            stderr_pipe = sys.stderr
        else:
            stdout_pipe = subprocess.PIPE
            stderr_pipe = subprocess.STDOUT

        p1 = subprocess.Popen(sync_cmd, stdout=stdout_pipe, stderr=stderr_pipe, stdin=subprocess.PIPE)
        p1_rc = p1.wait()
        stdout, _ = p1.communicate()

        # display output if the sync fails
        if p1_rc > 0:
            if not options.verbose:
                logging.warn(stdout)
            logging.warn('sync failed: %s' % name)
            continue  # no need to run createrepo if sync failed

        # run createrepo to generate package metadata
        if createrepo:
            logging.info('generating package metadata: {0}'.format(name))

            # if comps.xml exists, use it to generate group data
            comps_file = os.path.join(path, 'comps.xml')
            if os.path.isfile(comps_file):
                createrepo_opts = ['-g', comps_file] + createrepo_opts

            createrepo_cmd = createrepo_exec + createrepo_opts

            p2 = subprocess.Popen(createrepo_cmd, stdout=stdout_pipe, stderr=stderr_pipe, stdin=subprocess.PIPE)
            p2_rc = p2.wait()
            stdout, _ = p2.communicate()

            if p2_rc > 0:
                if not options.verbose:
                    logging.warn(stdout)
                logging.warn('createrepo failed: %s' % name)

if __name__ == "__main__":
    sys.exit(main())

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<commit_msg>Feature add exclude support for reposync (#3)<commit_after>#!/usr/bin/python -Ott

import sys
from optparse import OptionParser, OptionError
import subprocess
import os
import re
import ConfigParser
import getpass
import OpenSSL
import datetime
import glob
import logging

# Declare variables
mirror_dir = '/mirror/upstream'
confd_dir = '/etc/upstream_sync'

# directory that contains authentication credentials for sles
sles_auth_cred_dir = '/etc/nccs/sles_mirror'

# directory to store generated repo confs
user = getpass.getuser()
tmp_dir = '/var/tmp/upstream_sync-{0}'.format(user)

def make_dir(dir_path, mode=None):
    """ checks if a directory exists and create it if necessary
    :param mode: mode for makedirs
    :param dir_path: dir path for check

    :return: None
    """
    if not os.path.isdir(dir_path):
        # set current umask if not defined as parameter
        try:
            if mode:
                os.makedirs(dir_path, mode)
            else:
                os.makedirs(dir_path)
        except OSError as e:
            print e
            sys.exit(1)

def build_yum_config(name, url, sslcacert, sslcert, sslkey, exclude):
    # Check tmp path exist
    make_dir(tmp_dir)
    repo_conf = os.path.join(tmp_dir, '{0}.repo'.format(name))

    f = open(repo_conf, 'w')
    f.write('[main]\n')
    f.write('reposdir=/dev/null\n')
    f.write('[{0}]\n'.format(name))
    f.write('name = {0}\n'.format(name))
    f.write('baseurl = {0}\n'.format(url))
    f.write('enabled = 1\n')
    f.write('gpgcheck = 0\n')

    if sslcacert and sslcert and sslkey:
        check_sslcert_expiration(sslcert)
        f.write('sslverify = 1\n')
        f.write('sslcacert = {0}\n'.format(sslcacert))
        f.write('sslclientcert = {0}\n'.format(sslcert))
        f.write('sslclientkey = {0}\n'.format(sslkey))

    if exclude:
        f.write('exclude = {0}\n'.format(exclude))

    f.write('metadata_expire = 60\n')

    f.close()

    return repo_conf

def check_sslcert_expiration(sslcert):
    """checks to see if the ssl cert is going to expire soon"""
    cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, file(sslcert).read())
    cert_expires = datetime.datetime.strptime(cert.get_notAfter(), "%Y%m%d%H%M%SZ")

    if datetime.datetime.now() > cert_expires:
        logging.warn('SSL Certificate (%s) expired on %s' % (sslcert, cert_expires))
    elif datetime.datetime.now()+datetime.timedelta(days=14) > cert_expires:
        logging.warn('SSL Certificate (%s) is going to expire on %s' % (sslcert, cert_expires))

    return

def get_repos(rfilter=None):
    """
    parse configuration files and return repos.

    if rfilter is set, only repos that match rfilter will be returned
    """
    defaults = {
        'createrepo': 'False',
        'copylinks': 'False',
        'exclude': '',
        'sslcacert': '',
        'sslcert': '',
        'sslkey': ''
    }
    config = ConfigParser.ConfigParser(defaults)
    config.read(glob.glob(os.path.join(confd_dir, '*.repo')))

    # sort all the repos by alpha order, ConfigParser return sections
    # in the order that they appear in the config file
    repos = []
    for title in config.sections():
        if rfilter and (title not in rfilter):
            continue
        repo = dict(config.items(title))
        repo['name'] = title
        repo['path'] = os.path.join(mirror_dir, repo['path'])  # absolute path of repository
        repos.append(repo)

    repos = sorted(repos, key=lambda k: k['name'])
    return repos

def get_auths():
    config = ConfigParser.ConfigParser()
    config.read(os.path.join(confd_dir, 'auth.conf'))

    auths = dict()

    for title in config.sections():
        items = dict(config.items(title))
        auths[title] = items

    return auths

def list_repos(repos):
    for repo in repos:
        print repo['name']

def sync_cmd_reposync(repo):
    sslcacert = None
    sslcert = None
    sslkey = None
    exclude = None

    reposync_opts = []

    name = repo['name']
    url = repo['url']
    path = repo['path']

    if repo.has_key('auth'):
        auth = get_auths()[repo['auth']]
        sslcacert = auth['sslcacert']
        sslcert = auth['sslcert']
        sslkey = auth['sslkey']

    if repo.has_key('exclude'):
        exclude_list = repo['exclude'].split(',')
        # split() will return an empty list element
        if exclude_list:
            exclude = ' '.strip().join(exclude_list)

    yum_conf = build_yum_config(name, url, sslcacert, sslcert, sslkey, exclude)

    reposync_opts.extend(('-c', yum_conf))
    reposync_opts.extend(('-r', name))
    reposync_opts.extend(('-p', path))

    # detect arch
    match_arch = re.match(r'.*(?:/|-)(ppc64|ppc64le|x86_64|i386|i686|armhfp|amd64|x86)(?:/|$)', url)
    if match_arch:
        arch = match_arch.groups()[0]
        if arch in ['i386', 'x86']:
            arch = 'i686'
        elif arch in ['amd64', 'x86_64']:
            arch = 'x86_64'
        reposync_opts.extend(('--arch', arch))
    else:
        logging.warn('unable to detect architecture for %s' % name)

    # build options
    if repo.has_key('sync_opts'):
        opt_list = repo['sync_opts'].split()
        for opt in opt_list:
            reposync_opts.append(opt)
    else:
        reposync_opts.append('--tempcache')
        reposync_opts.append('--norepopath')
        reposync_opts.append('--downloadcomps')
        reposync_opts.append('--newest-only')
        reposync_opts.append('--delete')

    # be quiet if verbose is not set
    if not verbose:
        reposync_opts.append('-q')

    sync_cmd = ['reposync'] + reposync_opts
    return sync_cmd

def sync_cmd_rhnget(repo):
    systemid = os.path.join(os.path.split(repo['path'])[0], 'systemid')
    if not os.path.isfile(systemid):
        logging.warn("rhn: can not find systemid (%s)" % systemid)
        return

    sync_cmd = ['rhnget', '-q', '-s', systemid, repo['url'], repo['path']]
    return sync_cmd

def sync_cmd_rsync(repo):
    rsync_opts = []
    # build options
    if repo.has_key('sync_opts'):
        opt_list = repo['sync_opts'].split()
        for opt in opt_list:
            rsync_opts.append(opt)
    else:
        rsync_opts.append('--no-motd')
        rsync_opts.append('--recursive')
        rsync_opts.append('--delete')
        rsync_opts.append('--times')
        rsync_opts.append('--contimeout=30')

    if repo['copylinks'].lower() == 'true':
        rsync_opts.append('--copy-links')

    exclude_list = repo['exclude'].split(',')
    for item in exclude_list:
        # split() will return an empty list element
        if item:
            rsync_opts.append('--exclude')
            rsync_opts.append(item)

    if verbose:
        rsync_opts.append('--itemize-changes')

    sync_cmd = ['rsync'] + rsync_opts + [repo['url'], repo['path']]
    return sync_cmd

def sync_cmd_you(repo):
    # checking for sles credentials
    deviceid = os.path.join(sles_auth_cred_dir, 'deviceid')
    secret = os.path.join(sles_auth_cred_dir, 'secret')

    if not os.path.isfile(deviceid):
        logging.warn("you: can not find deviceid file (%s)" % deviceid)
        return
    elif not os.path.isfile(secret):
        logging.warn('you: can not find secret file (%s)' % secret)
        return

    url = re.sub('^you://', 'https://', repo['url'])
    sync_cmd = ['/opt/bin/youget', '-q', '--source', '-d', sles_auth_cred_dir, '--delete', url, repo['path']]

    return sync_cmd

def main():
    """main subroutine"""

    parser = OptionParser()
    parser.add_option("-v", "--verbose", help="Be verbose", action="count")
    parser.add_option("-l", "--list", help="list repos", action="store_true")
    parser.add_option("-c", "--command", help="print the sync command", action="store_true")
    parser.add_option("-r", "--repos", help="syncs specific repo(s) (comma seperated list)", dest="rfilter")
    parser.add_option("--root", help="run script as root", action="store_true")
    try:
        (options, args) = parser.parse_args()
    except OptionError:
        parser.print_help()
        return 1

    global verbose
    if options.verbose:
        logging.basicConfig(level=logging.DEBUG)
        verbose = True
    else:
        logging.basicConfig(level=logging.WARNING)
        verbose = False

    show_command = options.command
    run_as_root = options.root

    # get repos from config
    if options.rfilter:
        repos = get_repos(rfilter=options.rfilter.split(','))
    else:
        repos = get_repos()

    # list repos
    if options.list:
        list_repos(repos)
        sys.exit(0)

    if not run_as_root:
        if os.geteuid() == 0:
            print("It is strongly advised not to run this script as root!")
            print("Running as root will most likely mess up filesystem permissions.")
            print("If you are sure you want to run as root, pass --root")
            sys.exit(2)

    if not show_command:
        make_dir(confd_dir)

    for repo in repos:
        # set variables based on values in config
        url = repo['url']
        name = repo['name']
        path = os.path.join(mirror_dir, repo['path'])  # absolute path of repository

        # create repo directory
        if not show_command:
            make_dir(path, 0775)

        createrepo = False
        if repo['createrepo'].lower() == "true":
            createrepo = True

        # Generate the sync and createrepo commands to be used based on repository type
        createrepo_exec = ['createrepo']
        createrepo_opts = ['--pretty', '--database', '--update', '--cachedir', os.path.join(path, '.cache'), path]
        if not options.verbose:
            createrepo_opts.append('-q')

        if re.match('^(http|https)://', url):
            sync_cmd = sync_cmd_reposync(repo)
        elif re.match('^rhns:///', url):
            sync_cmd = sync_cmd_rhnget(repo)
        elif re.match('^you://', url):
            sync_cmd = sync_cmd_you(repo)
        elif re.match('^rsync://', url):
            sync_cmd = sync_cmd_rsync(repo)
        else:
            logging.warn('url type unknown - %s' % url)
            continue

        if not sync_cmd:
            continue

        # if option -c is passed print commands and continue to loop item
        if show_command:
            print('%s:' % name)
            print('  '+' '.join(sync_cmd))
            if createrepo:
                print('  '+' '.join(createrepo_exec + createrepo_opts))
            continue

        # preform sync - rhnget/rsync
        logging.info('syncing %s' % name)
        if options.verbose:
            stdout_pipe = sys.stdout
            stderr_pipe = sys.stderr
        else:
            stdout_pipe = subprocess.PIPE
            stderr_pipe = subprocess.STDOUT

        p1 = subprocess.Popen(sync_cmd, stdout=stdout_pipe, stderr=stderr_pipe, stdin=subprocess.PIPE)
        p1_rc = p1.wait()
        stdout, _ = p1.communicate()

        # display output if the sync fails
        if p1_rc > 0:
            if not options.verbose:
                logging.warn(stdout)
            logging.warn('sync failed: %s' % name)
            continue  # no need to run createrepo if sync failed

        # run createrepo to generate package metadata
        if createrepo:
            logging.info('generating package metadata: {0}'.format(name))

            # if comps.xml exists, use it to generate group data
            comps_file = os.path.join(path, 'comps.xml')
            if os.path.isfile(comps_file):
                createrepo_opts = ['-g', comps_file] + createrepo_opts

            createrepo_cmd = createrepo_exec + createrepo_opts

            p2 = subprocess.Popen(createrepo_cmd, stdout=stdout_pipe, stderr=stderr_pipe, stdin=subprocess.PIPE)
            p2_rc = p2.wait()
            stdout, _ = p2.communicate()

            if p2_rc > 0:
                if not options.verbose:
                    logging.warn(stdout)
                logging.warn('createrepo failed: %s' % name)

if __name__ == "__main__":
    sys.exit(main())

<|endoftext|>


<commit_before>import copy
import datetime
import json
import logging
import os
import socket
import uuid

from uptime.exceptions import UptimeError

logging.getLogger('uptime')

class UptimeEncoder(json.JSONEncoder):

    def default(self, o):
        if hasattr(o, 'options'):
            _dict = {key: getattr(o, key) for key in o.options if key not in o.private}
            _dict['_uptime_object'] = o.__class__.__name__
            return _dict
        return json.JSONEncoder.default(self, o)

class UptimeObject:

    options = {}
    private = []

    @classmethod
    def from_json(cls, raw_json):
        """
        Translate raw JSON into an Uptime object
        """

        def _hook(parsed):
            """
            Look through each parsed field.
            """

            if '_uptime_object' in parsed:
                if parsed['_uptime_object'] != cls.__name__:
                    raise UptimeError('Incompatible object: {}'.format(parsed))
                del parsed['_uptime_object']

            return parsed

        try:
            return cls(**json.loads(raw_json, object_hook=_hook))
        except (TypeError, ValueError) as e:
            logging.warning('JSON Validation Failed for: {}'.format(e))
            raise

    def _log_config(self):
            logging.debug('Current configuration: \n {}'.format('\n '.join(
                {x + '=' + str(getattr(self, x)): getattr(self, x) for x in self.options}
            )))

class Config(UptimeObject):
    """
    The Config object holds configuration values across the Uptime application.
    Defaults are specified in the "options" dictionary, which may be overridden
    by environment variables with the same name capitalized prefixed with "UPTIME"
    i.e. UPTIME_APP_DIR=/some/path/here will override the Config.app_dir attribute.
    """
    options = {
        'app_dir': os.path.dirname(os.path.realpath(__file__)),
        'auth_key': str(uuid.uuid4()),
        'concurrency': 5,
        'debug': True,
        'encoding': 'UTF-8',
        'format': '%(asctime)s - %(levelname)s - %(name)s.%(module)s - %(message)s',
        'mode': 'api',
        'redis_host': 'localhost',
        'redis_port': 6379,
        'source': socket.getfqdn(),
        'slack_url': None
        'slack_channels': None
    }

    modes = ['api', 'server']

    def __init__(self, **kwargs):
        self.config = self.options
        self.config.update(kwargs)

        for k,v in self.config.items():
            self.__setattr__(k, v)

        self._get_env()

        if self.config.slack_channels:
            self.config.slack_channels.split(',')

    def _get_env(self):
        self._log_config()
        environment_vars = [x for x in self.options if 'UPTIME_' + x.upper() in os.environ]
        for env_variable in environment_vars:  # Check to see if an environment variable overrides a config value.
            name = 'UPTIME_' + env_variable.upper()
            current_name = env_variable  # Name of the config
            current_value = getattr(self, current_name)  # Existing value
            new_value = os.getenv(name)  # Environment variable to override with
            logging.warning(
                'Overriding config [{} = {}] --> [{} = {}] from environment variables'.format(
                    current_name, current_value, current_name, new_value
                )
            )
            setattr(self, current_name, new_value)  # Override the config

class Check(UptimeObject):
    """
    URL Check configuration object
    """

    options = {
        'check_id': None,
        'content': None,
        'failures': 0,
        'interval': 15,
        'name': None,
        'notified': False,
        'url': None,
    }

    private = ['last']

    def __init__(self, **kwargs):
        self._config = copy.copy(self.options)
        self._config.update(kwargs)

        self.check_id = self._config['check_id']
        self.content = self._config['content']
        self.failures = self._config['failures']
        self.interval = int(self._config['interval'])
        self.name = self.check_id
        self.notified = self._config['notified']
        self.url = self._config['url']
        self.last = datetime.datetime.utcnow()
        logging.info('loaded check %s for url %s' % (self.check_id, self.url))
        self._log_config()

    def dump_json(self):
        return json.dumps(self, cls=UptimeEncoder)

    def ok(self):
        """
        Method to reset failures and notify switches following successful check
        """
        self.failures = 0
        self.notified = False
<commit_msg>fix syntax error<commit_after>import copy
import datetime
import json
import logging
import os
import socket
import uuid

from uptime.exceptions import UptimeError

logging.getLogger('uptime')

class UptimeEncoder(json.JSONEncoder):

    def default(self, o):
        if hasattr(o, 'options'):
            _dict = {key: getattr(o, key) for key in o.options if key not in o.private}
            _dict['_uptime_object'] = o.__class__.__name__
            return _dict
        return json.JSONEncoder.default(self, o)

class UptimeObject:

    options = {}
    private = []

    @classmethod
    def from_json(cls, raw_json):
        """
        Translate raw JSON into an Uptime object
        """

        def _hook(parsed):
            """
            Look through each parsed field.
            """

            if '_uptime_object' in parsed:
                if parsed['_uptime_object'] != cls.__name__:
                    raise UptimeError('Incompatible object: {}'.format(parsed))
                del parsed['_uptime_object']

            return parsed

        try:
            return cls(**json.loads(raw_json, object_hook=_hook))
        except (TypeError, ValueError) as e:
            logging.warning('JSON Validation Failed for: {}'.format(e))
            raise

    def _log_config(self):
            logging.debug('Current configuration: \n {}'.format('\n '.join(
                {x + '=' + str(getattr(self, x)): getattr(self, x) for x in self.options}
            )))

class Config(UptimeObject):
    """
    The Config object holds configuration values across the Uptime application.
    Defaults are specified in the "options" dictionary, which may be overridden
    by environment variables with the same name capitalized prefixed with "UPTIME"
    i.e. UPTIME_APP_DIR=/some/path/here will override the Config.app_dir attribute.
    """
    options = {
        'app_dir': os.path.dirname(os.path.realpath(__file__)),
        'auth_key': str(uuid.uuid4()),
        'concurrency': 5,
        'debug': True,
        'encoding': 'UTF-8',
        'format': '%(asctime)s - %(levelname)s - %(name)s.%(module)s - %(message)s',
        'mode': 'api',
        'redis_host': 'localhost',
        'redis_port': 6379,
        'source': socket.getfqdn(),
        'slack_url': None,
        'slack_channels': None
    }

    modes = ['api', 'server']

    def __init__(self, **kwargs):
        self.config = self.options
        self.config.update(kwargs)

        for k,v in self.config.items():
            self.__setattr__(k, v)

        self._get_env()

        if self.config.slack_channels:
            self.config.slack_channels.split(',')

    def _get_env(self):
        self._log_config()
        environment_vars = [x for x in self.options if 'UPTIME_' + x.upper() in os.environ]
        for env_variable in environment_vars:  # Check to see if an environment variable overrides a config value.
            name = 'UPTIME_' + env_variable.upper()
            current_name = env_variable  # Name of the config
            current_value = getattr(self, current_name)  # Existing value
            new_value = os.getenv(name)  # Environment variable to override with
            logging.warning(
                'Overriding config [{} = {}] --> [{} = {}] from environment variables'.format(
                    current_name, current_value, current_name, new_value
                )
            )
            setattr(self, current_name, new_value)  # Override the config

class Check(UptimeObject):
    """
    URL Check configuration object
    """

    options = {
        'check_id': None,
        'content': None,
        'failures': 0,
        'interval': 15,
        'name': None,
        'notified': False,
        'url': None,
    }

    private = ['last']

    def __init__(self, **kwargs):
        self._config = copy.copy(self.options)
        self._config.update(kwargs)

        self.check_id = self._config['check_id']
        self.content = self._config['content']
        self.failures = self._config['failures']
        self.interval = int(self._config['interval'])
        self.name = self.check_id
        self.notified = self._config['notified']
        self.url = self._config['url']
        self.last = datetime.datetime.utcnow()
        logging.info('loaded check %s for url %s' % (self.check_id, self.url))
        self._log_config()

    def dump_json(self):
        return json.dumps(self, cls=UptimeEncoder)

    def ok(self):
        """
        Method to reset failures and notify switches following successful check
        """
        self.failures = 0
        self.notified = False
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
An example locustfile demonstrating how to load test
an AI Platform Prediction predict endpoint.
"""

import json
import logging
import math
import os

import locust

TEST_REQUEST_BODY = 'request-body.json'
TEST_CONFIG = 'test-config.json'

class StepLoad(locust.LoadTestShape):
    """
    The load shape class that progressively adds users up to the maximum.
    After the maximum load has been reached the class starts to progressively decrease
    the load till all users are stopped.
    """

    max_steps = 15
    step_time = 60
    step_load = 2
    spawn_rate = 1

    def tick(self):
        run_time = self.get_run_time()

        current_step = math.floor(run_time / self.step_time) + 1

        if current_step <= self.max_steps:
            return (current_step * self.step_load, self.spawn_rate)
        else:
            user_count = 2 * self.max_steps - current_step
            if user_count > 1:
                return (user_count/2, self.spawn_rate)
            else:
                return None

class TFServingClient(locust.HttpUser):
    """
    Simulated TF Serving client.
    """

    wait_time = locust.between(0.8, 0.9)

    @locust.task
    def predict(self):

        response =self.client.post(self.url,
                                   data=json.dumps(self.request_body))

        if 'error' in response.json():
            self.environment.runner.stats.log_error(
                response.request.method,
                response.request.path_url)

    def on_start(self):
        with open(TEST_CONFIG) as f:
            test_config = json.load(f)
        with open(TEST_REQUEST_BODY) as f:
            self.request_body = json.load(f)

        self.url = '{}/v1/models/{}/versions/{}:predict'.format(
            self.environment.host,
            test_config['model'],
            test_config['version'])

        <commit_msg>Updates to tasks.py<commit_after>#!/usr/bin/env python

# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
An example locustfile demonstrating how to load test
an AI Platform Prediction predict endpoint.
"""

import json
import logging
import math
import os

import locust

TEST_REQUEST_BODY = 'request-body.json'
TEST_CONFIG = 'test-config.json'

class StepLoad(locust.LoadTestShape):
    """
    The load shape class that progressively adds users up to the maximum.
    After the maximum load has been reached the class starts to progressively decrease
    the load till all users are stopped.
    """

    max_steps = 15
    step_time = 60
    step_load = 2
    spawn_rate = 1

    def tick(self):
        run_time = self.get_run_time()

        current_step = math.floor(run_time / self.step_time) + 1

        if current_step <= self.max_steps:
            return (current_step * self.step_load, self.spawn_rate)
        else:
            #user_count = 2 * self.max_steps - current_step
            #if user_count > 1:
            #    return (user_count/2, self.spawn_rate)
            #else:
            return None

class TFServingClient(locust.HttpUser):
    """
    Simulated TF Serving client.
    """

    wait_time = locust.between(0.8, 0.9)

    @locust.task
    def predict(self):

        response =self.client.post(self.url,
                                   data=json.dumps(self.request_body))

        if 'error' in response.json():
            self.environment.runner.stats.log_error(
                response.request.method,
                response.request.path_url)

    def on_start(self):
        with open(TEST_CONFIG) as f:
            test_config = json.load(f)
        with open(TEST_REQUEST_BODY) as f:
            self.request_body = json.load(f)

        self.url = '{}/v1/models/{}/versions/{}:predict'.format(
            self.environment.host,
            test_config['model'],
            test_config['version'])

        <|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import re
import urlparse

from django.core import exceptions
from django.utils import importlib
from django.utils import translation
from django.conf import settings

from urli18n import app_settings

def show_language(language):
    """Simple helper method to determine if the language
    passed should be shown in the url or not according to
   `` URLI18N_ALWAYS_SHOW_LANGUAGE`` flag in
    ``app_settings``.

    Params:
        - ``language`` - the language shortcut which should be checked

    Returns:
        A Boolean: ``True`` if language should be should, ``False`` otherwise
    """
    return not (app_settings.URLI18N_ALWAYS_SHOW_LANGUAGE is False and language == settings.LANGUAGE_CODE)

def is_included_path(path, strict_mode=True):
    """Helper to determine if a given path should be
    transformed to the show the language in the url. This
    is checked against ``URLI18N_INCLUDE_PATHS`` in
    ``app_settings``. The ``MEDIA_URL`` and ``STATIC_URL``
    from ``django.conf.settings`` are excluded automatically.

    Params:
        - ``path``: the url path which should be checked
        - ``strict_mode``: a boolean which indicates if the path should be checked strict or non-strict. This affects check within the function ``_edit_path_exp`` only.

    Returns:
        A Boolean: ``True`` if the given url path should be included,
        ``False`` otherwise.
    """
    path = urlparse.urlparse(path).path
    media_url = getattr(settings, 'MEDIA_URL')
    static_url = getattr(settings, 'STATIC_URL')
    if not (media_url and path.startswith(media_url))\
    and not (static_url and path.startswith(static_url)):
        for p in app_settings.URLI18N_INCLUDE_PATHS:
            p = _edit_path_exp(p, mode)
            regex_path = re.compile(p, re.UNICODE)
            if regex_path.match(path):
                return True
    return False

def _edit_path_exp(path_expression, strict_mode=True):
    """Private helper function transform a regular expression
    given in ``app_settings.URLI18N_INCLUDE_PATHS`` for
    checking. This can be performed in strict or non-strict mode.
    If strict mode it will make sure the given expression ends with
    ``/$``. In non strict mode it will simply not care how the
    expression ends. In both cases it will transform an expression
    to start with ``^/`` though.

    Params:
        - ``path_expression``: a regular expression to check against a url path
        - ``strict_mode``: a boolean indicating string or non-strict mode. See explainations above.

    Returns:
        - ``path_expression`` (modified if needed)
    """
    if not path_expression.startswith('^/'):
        if path_expression.startswith('^'):
            path_expression = path_expression.replace('^', '^/')
        elif path_expression.startswith('/'):
            path_expression = '^%s' % path_expression
        else:
            path_expression = '^/%s' % path_expression
    if strict_mode is True:
        if not path_expression.endswith('/$'):
            if path_expression.endswith('$'):
                path_expression = path_expression.replace('$', '/$')
            elif path_expression.endswith('/'):
                path_expression = '%s$' % path_expression
            else:
                path_expression = '%s/$' % path_expression
    return path_expression

def process_missing_requests(middleware_object, request):
    """Processes every ``process_request`` method from
    all middleware's which follow this middleware within
    ``settings.MIDDLEWARE_CLASSES``

    Args:
        - ``middleware_object``: the current middleware object
        - ``request``: the current django request object, this will be passed to the ``process_request`` calls

    Returns:
        - if a ``process_request`` call returns an HttpResponse it will return this response, else None
    """
    middleware_string = _create_middleware_string(middleware_object)
    middleware_tuple = []
    middleware_appeared = False
    for middleware_path in settings.MIDDLEWARE_CLASSES:
        if middleware_appeared is True:
            middleware_tuple.append(middleware_path)
        if middleware_appeared is False and middleware_path==middleware_string:
            middleware_appeared = True
    middlware_instances = _init_middleware_classes(middleware_tuple)
    for mw_instance in middlware_instances:
        if hasattr(mw_instance, 'process_request'):
            process_result = mw_instance.process_request(request)
            if process_result is not None:
                return process_result
    return None

def process_missing_views(middleware_object, request, view_func, view_args, view_kwargs):
    """Processes every ``process_view`` method from
    all middleware's in ``settings.MIDDLEWARE_CLASSES``.
    Since ``process_view`` calls are never made when calling
    a view (and returning its response) from a middleware class
    ``process_request`` method this helper will do the job instead.

    Args:
        - ``middleware_object``: the current middleware object
        - ``request``: the current django request object, this will be passed to the ``process_view`` calls
        - ``view_func``: the view function which will be passed to the ``process_view`` calls
        - ``view_args``: the view arguments which will be passed to the ``process_view`` calls
        - ``view_kwargs``: the view keyword arguments which will be passed to the ``process_view`` calls

   Returns:
        - if a ``process_view`` call returns an HttpResponse it will return this response, else None
    """
    middlware_instances = _init_middleware_classes(settings.MIDDLEWARE_CLASSES)
    for mw_instance in middlware_instances:
        if hasattr(mw_instance, 'process_view'):
            process_result = mw_instance.process_view(request, view_func, view_args, view_kwargs)
            if process_result is not None:
                return process_result
    return None

def _create_middleware_string(middleware_object):
    """A helper function to provide the string for a
    given middleware object which occur within
    ``settings.MIDDLEWARE_CLASSES``

    Args:
        - ``middleware_object``: the current middleware object

    Returns:
        - a string representing the string of the ``middleware_objects`` in ``settings.MIDDLEWARE_CLASSES``
    """
    return  '%s.%s' % (middleware_object.__module__, middleware_object.__class__.__name__)

def _init_middleware_classes(middleware_tuple):
    """A helppr function which initialize all middleware
    classes given by their string representation in a tuple
    similiar to settings.MIDDLEWARE_CLASSES. Uses the
    same approach as django.core.handlers.base.BaseHandler
    to initialize middleware classes.

    Args:
        - ``middleware_tuple``: a tuple of string representations of middleware classes which should be initialized here

    Returns:
        - a list of initialized middleware classes
    """
    mw_instances = []
    for middleware_path in middleware_tuple:
        try:
            mw_module, mw_classname = middleware_path.rsplit('.', 1)
        except ValueError:
            raise exceptions.ImproperlyConfigured('%s isn\'t a middleware module' % middleware_path)
        try:
            mod = importlib.import_module(mw_module)
        except ImportError, e:
            raise exceptions.ImproperlyConfigured('Error importing middleware %s: "%s"' % (mw_module, e))
        try:
            mw_class = getattr(mod, mw_classname)
        except AttributeError:
            raise exceptions.ImproperlyConfigured('Middleware module "%s" does not define a "%s" class' % (mw_module, mw_classname))
        try:
            mw_instance = mw_class()
            mw_instances.append(mw_instance)
        except exceptions.MiddlewareNotUsed:
            continue
    return mw_instances

def break_full_path(full_path):
    """Utility function which breaks the full path given from
    django ``request.get_full_path()`` into reasonable bits which can
    be processed by the middleware.

    Args:
        - ``full_path``: the full url path from ``request.get_full_path()``

    Returns:
        - a tuple containing the following elements:
            - ``path_parts``: the one or two parts of the full_path splitted on the "?"
            - ``querystring_parts``: if querystring was found the querystring parts splitted at the "&", else its an empty list
            - ``language_querystring``: the language querystring determining the current language found in the querystring, if not founded its an empty string
            - ``language_querystring_position``: the index of the language_querystring within ``querystring_parts`` if ``language_querystring`` was found, else None
    """
    querystring_name = app_settings.URLI18N_QUERYSTRING_NAME
    regex_query = re.compile('%s=[-\w]{2,5}' % querystring_name)
    path_parts = full_path.split('?')
    querystring = ''
    querystring_parts = []
    language_querystring_position = None
    language_querystring = ''
    if len(path_parts) > 1:
        querystring = path_parts[1]
        querystring_parts = querystring.split('&')
        for i, part in enumerate(querystring_parts):
            if regex_query.match(part) is not None:
                language_querystring_position = i
                language_querystring = querystring_parts[i]
                break
    return path_parts, querystring_parts, language_querystring, language_querystring_position

def reconstruct_full_path(path, querystring_parts):
    """Utility function which reconstruct the full_path
    according to new paremeters.

    Args:
        - ``path``: the absolute path without the querystring
        - ``querystring_parts``: the parts of the querystring as a list, similiar to ``querystring_parts`` from ``break_full_path`` function

    Returns:
        - the reconstructed full path with query string
    """
    full_path = path
    if querystring_parts:
        full_path = '?'.join([path, '&'.join(querystring_parts)])
    return full_path

def transform_path(path):
    """Utility function used by the template tags
    to transform given url paths according to active
    language. This helps to change already given urls
    to add the language parameter so the extra
    redirect within the middleware is not necessary.

    Args:
        - ``path``: the url path which should be transformed

    Returns:
        - the transformed url path according to which middleware is used and which settings are set
    """
    language = translation.get_language()
    if 'urli18n.middleware.UrlPathTransformMiddleware' in settings.MIDDLEWARE_CLASSES:
        if show_language(language) and not path.startswith('/%s/' % language)\
        and not path=='/%s' % language and is_included_path(path, strict_mode=False):
            path = '/%s%s' % (language, path)
    elif 'urli18n.middleware.UrlQuerystringTransformMiddleware' in settings.MIDDLEWARE_CLASSES:
        if show_language(language) and is_included_path(path, strict_mode=False):
            language_querystring = '%s=%s' % (app_settings.URLI18N_QUERYSTRING_NAME, language)
            path_parts = path.split('?')
            if len(path_parts)==1 or language_querystring not in path_parts[1]:
                if len(path_parts)==1:
                    querystring_parts = []
                else:
                    querystring_parts = path_parts[1].split('&')
                querystring_parts.append(language_querystring)
                path = reconstruct_full_path(path_parts[0], querystring_parts)
    return path

<commit_msg>fixing a NameError in `urli18n/utils.py` `_edit_path_exp`. Thx @pigletto<commit_after># -*- coding: utf-8 -*-

import re
import urlparse

from django.core import exceptions
from django.utils import importlib
from django.utils import translation
from django.conf import settings

from urli18n import app_settings

def show_language(language):
    """Simple helper method to determine if the language
    passed should be shown in the url or not according to
   `` URLI18N_ALWAYS_SHOW_LANGUAGE`` flag in
    ``app_settings``.

    Params:
        - ``language`` - the language shortcut which should be checked

    Returns:
        A Boolean: ``True`` if language should be should, ``False`` otherwise
    """
    return not (app_settings.URLI18N_ALWAYS_SHOW_LANGUAGE is False and language == settings.LANGUAGE_CODE)

def is_included_path(path, strict_mode=True):
    """Helper to determine if a given path should be
    transformed to the show the language in the url. This
    is checked against ``URLI18N_INCLUDE_PATHS`` in
    ``app_settings``. The ``MEDIA_URL`` and ``STATIC_URL``
    from ``django.conf.settings`` are excluded automatically.

    Params:
        - ``path``: the url path which should be checked
        - ``strict_mode``: a boolean which indicates if the path should be checked strict or non-strict. This affects check within the function ``_edit_path_exp`` only.

    Returns:
        A Boolean: ``True`` if the given url path should be included,
        ``False`` otherwise.
    """
    path = urlparse.urlparse(path).path
    media_url = getattr(settings, 'MEDIA_URL')
    static_url = getattr(settings, 'STATIC_URL')
    if not (media_url and path.startswith(media_url))\
    and not (static_url and path.startswith(static_url)):
        for p in app_settings.URLI18N_INCLUDE_PATHS:
            p = _edit_path_exp(p, strict_mode)
            regex_path = re.compile(p, re.UNICODE)
            if regex_path.match(path):
                return True
    return False

def _edit_path_exp(path_expression, strict_mode=True):
    """Private helper function transform a regular expression
    given in ``app_settings.URLI18N_INCLUDE_PATHS`` for
    checking. This can be performed in strict or non-strict mode.
    If strict mode it will make sure the given expression ends with
    ``/$``. In non strict mode it will simply not care how the
    expression ends. In both cases it will transform an expression
    to start with ``^/`` though.

    Params:
        - ``path_expression``: a regular expression to check against a url path
        - ``strict_mode``: a boolean indicating string or non-strict mode. See explainations above.

    Returns:
        - ``path_expression`` (modified if needed)
    """
    if not path_expression.startswith('^/'):
        if path_expression.startswith('^'):
            path_expression = path_expression.replace('^', '^/')
        elif path_expression.startswith('/'):
            path_expression = '^%s' % path_expression
        else:
            path_expression = '^/%s' % path_expression
    if strict_mode is True:
        if not path_expression.endswith('/$'):
            if path_expression.endswith('$'):
                path_expression = path_expression.replace('$', '/$')
            elif path_expression.endswith('/'):
                path_expression = '%s$' % path_expression
            else:
                path_expression = '%s/$' % path_expression
    return path_expression

def process_missing_requests(middleware_object, request):
    """Processes every ``process_request`` method from
    all middleware's which follow this middleware within
    ``settings.MIDDLEWARE_CLASSES``

    Args:
        - ``middleware_object``: the current middleware object
        - ``request``: the current django request object, this will be passed to the ``process_request`` calls

    Returns:
        - if a ``process_request`` call returns an HttpResponse it will return this response, else None
    """
    middleware_string = _create_middleware_string(middleware_object)
    middleware_tuple = []
    middleware_appeared = False
    for middleware_path in settings.MIDDLEWARE_CLASSES:
        if middleware_appeared is True:
            middleware_tuple.append(middleware_path)
        if middleware_appeared is False and middleware_path==middleware_string:
            middleware_appeared = True
    middlware_instances = _init_middleware_classes(middleware_tuple)
    for mw_instance in middlware_instances:
        if hasattr(mw_instance, 'process_request'):
            process_result = mw_instance.process_request(request)
            if process_result is not None:
                return process_result
    return None

def process_missing_views(middleware_object, request, view_func, view_args, view_kwargs):
    """Processes every ``process_view`` method from
    all middleware's in ``settings.MIDDLEWARE_CLASSES``.
    Since ``process_view`` calls are never made when calling
    a view (and returning its response) from a middleware class
    ``process_request`` method this helper will do the job instead.

    Args:
        - ``middleware_object``: the current middleware object
        - ``request``: the current django request object, this will be passed to the ``process_view`` calls
        - ``view_func``: the view function which will be passed to the ``process_view`` calls
        - ``view_args``: the view arguments which will be passed to the ``process_view`` calls
        - ``view_kwargs``: the view keyword arguments which will be passed to the ``process_view`` calls

   Returns:
        - if a ``process_view`` call returns an HttpResponse it will return this response, else None
    """
    middlware_instances = _init_middleware_classes(settings.MIDDLEWARE_CLASSES)
    for mw_instance in middlware_instances:
        if hasattr(mw_instance, 'process_view'):
            process_result = mw_instance.process_view(request, view_func, view_args, view_kwargs)
            if process_result is not None:
                return process_result
    return None

def _create_middleware_string(middleware_object):
    """A helper function to provide the string for a
    given middleware object which occur within
    ``settings.MIDDLEWARE_CLASSES``

    Args:
        - ``middleware_object``: the current middleware object

    Returns:
        - a string representing the string of the ``middleware_objects`` in ``settings.MIDDLEWARE_CLASSES``
    """
    return  '%s.%s' % (middleware_object.__module__, middleware_object.__class__.__name__)

def _init_middleware_classes(middleware_tuple):
    """A helppr function which initialize all middleware
    classes given by their string representation in a tuple
    similiar to settings.MIDDLEWARE_CLASSES. Uses the
    same approach as django.core.handlers.base.BaseHandler
    to initialize middleware classes.

    Args:
        - ``middleware_tuple``: a tuple of string representations of middleware classes which should be initialized here

    Returns:
        - a list of initialized middleware classes
    """
    mw_instances = []
    for middleware_path in middleware_tuple:
        try:
            mw_module, mw_classname = middleware_path.rsplit('.', 1)
        except ValueError:
            raise exceptions.ImproperlyConfigured('%s isn\'t a middleware module' % middleware_path)
        try:
            mod = importlib.import_module(mw_module)
        except ImportError, e:
            raise exceptions.ImproperlyConfigured('Error importing middleware %s: "%s"' % (mw_module, e))
        try:
            mw_class = getattr(mod, mw_classname)
        except AttributeError:
            raise exceptions.ImproperlyConfigured('Middleware module "%s" does not define a "%s" class' % (mw_module, mw_classname))
        try:
            mw_instance = mw_class()
            mw_instances.append(mw_instance)
        except exceptions.MiddlewareNotUsed:
            continue
    return mw_instances

def break_full_path(full_path):
    """Utility function which breaks the full path given from
    django ``request.get_full_path()`` into reasonable bits which can
    be processed by the middleware.

    Args:
        - ``full_path``: the full url path from ``request.get_full_path()``

    Returns:
        - a tuple containing the following elements:
            - ``path_parts``: the one or two parts of the full_path splitted on the "?"
            - ``querystring_parts``: if querystring was found the querystring parts splitted at the "&", else its an empty list
            - ``language_querystring``: the language querystring determining the current language found in the querystring, if not founded its an empty string
            - ``language_querystring_position``: the index of the language_querystring within ``querystring_parts`` if ``language_querystring`` was found, else None
    """
    querystring_name = app_settings.URLI18N_QUERYSTRING_NAME
    regex_query = re.compile('%s=[-\w]{2,5}' % querystring_name)
    path_parts = full_path.split('?')
    querystring = ''
    querystring_parts = []
    language_querystring_position = None
    language_querystring = ''
    if len(path_parts) > 1:
        querystring = path_parts[1]
        querystring_parts = querystring.split('&')
        for i, part in enumerate(querystring_parts):
            if regex_query.match(part) is not None:
                language_querystring_position = i
                language_querystring = querystring_parts[i]
                break
    return path_parts, querystring_parts, language_querystring, language_querystring_position

def reconstruct_full_path(path, querystring_parts):
    """Utility function which reconstruct the full_path
    according to new paremeters.

    Args:
        - ``path``: the absolute path without the querystring
        - ``querystring_parts``: the parts of the querystring as a list, similiar to ``querystring_parts`` from ``break_full_path`` function

    Returns:
        - the reconstructed full path with query string
    """
    full_path = path
    if querystring_parts:
        full_path = '?'.join([path, '&'.join(querystring_parts)])
    return full_path

def transform_path(path):
    """Utility function used by the template tags
    to transform given url paths according to active
    language. This helps to change already given urls
    to add the language parameter so the extra
    redirect within the middleware is not necessary.

    Args:
        - ``path``: the url path which should be transformed

    Returns:
        - the transformed url path according to which middleware is used and which settings are set
    """
    language = translation.get_language()
    if 'urli18n.middleware.UrlPathTransformMiddleware' in settings.MIDDLEWARE_CLASSES:
        if show_language(language) and not path.startswith('/%s/' % language)\
        and not path=='/%s' % language and is_included_path(path, strict_mode=False):
            path = '/%s%s' % (language, path)
    elif 'urli18n.middleware.UrlQuerystringTransformMiddleware' in settings.MIDDLEWARE_CLASSES:
        if show_language(language) and is_included_path(path, strict_mode=False):
            language_querystring = '%s=%s' % (app_settings.URLI18N_QUERYSTRING_NAME, language)
            path_parts = path.split('?')
            if len(path_parts)==1 or language_querystring not in path_parts[1]:
                if len(path_parts)==1:
                    querystring_parts = []
                else:
                    querystring_parts = path_parts[1].split('&')
                querystring_parts.append(language_querystring)
                path = reconstruct_full_path(path_parts[0], querystring_parts)
    return path

<|endoftext|>


<commit_before>"""
filename: controllers.py
description: Cronrollers for charges.
created by: Omar De La Hoz (oed7416@rit.edu), Chris Lemelin (cxl8826@rit.edu)
created on: 12/05/17
"""

from flask_socketio import emit
from app.decorators import ensure_dict, get_user
from app import socketio, db
from app.charges.models import *
from app.committees.models import Committees
from app.members.models import Roles
from app.charges.charges_response import Response
from app.users.models import Users

##
## @brief      Gets all public charges.
##
## @param      broadcast  The broadcast
##
## @return     All charges.
##
@socketio.on('get_all_charges')
def get_all_charges(broadcast = False):

    charges = Charges.query.filter_by().all()
    charge_ser = []

    for charge in charges:
        if charge.private:
            continue;

        charge_ser.append({
            "id": charge.id,
            "title": charge.title,
            "description": charge.description,
            "committee": charge.committee,
            "priority": charge.priority,
            "status": charge.status,
            "paw_links": charge.paw_links,
            "created_at": charge.created_at.isoformat()
        });
    emit("get_all_charges", charge_ser, broadcast = broadcast)

##
## @brief      Gets the charges for a specific committee.
##
## @param      committee_id  The committee identifier
## @param      broadcast     Flag to broadcast list of charges
##                           to all users.
##
## @emit       Emits a list of charges for committee.
##
@socketio.on('get_charges')
@ensure_dict
@get_user
def get_charges(user, user_data, broadcast = False):
    charges = Charges.query.filter_by(committee= user_data.get("committee_id", "")).all()
    charge_ser = []

    if len(charges) == 0:
        emit("get_charges", charge_ser, broadcast = broadcast)
        return;

    committee = Committees.query.filter_by(id = user_data.get("committee_id", "")).first()
    membership = committee.members.filter_by(member= user).first()

    for charge in charges:
        if charge.private and membership is None:
            continue;

        charge_ser.append({
            "id": charge.id,
            "title": charge.title,
            "description": charge.description,
            "committee": charge.committee,
            "priority": charge.priority,
            "status": charge.status,
            "paw_links": charge.paw_links,
            "private": charge.private,
            "created_at": charge.created_at.isoformat()
        });
    emit("get_charges", charge_ser, broadcast = broadcast)

##
## @brief      Gets a charge by its ID.
##
## @param      charge     The charge identifier
## @param      broadcast  Flag to broadcast list of charges
##                        to all users.
##
## @emit       An objet containing the detailed view of a
##             charge.
##
@socketio.on('get_charge')
@ensure_dict
@get_user
def get_charge(user, user_data, broadcast = False):

    charge = Charges.query.filter_by(id= user_data["charge"]).first()

    if charge is None:
        emit('get_charge', Response.UsrChargeDontExist)
        return;

    committee = Committees.query.filter_by(id = charge.committee).first()
    membership = committee.members.filter_by(member= user).first()

    if charge.private and (membership is None and not user.is_admin):
        emit('get_charge', Response.PermError)
        return;

    charge_info = {
        "id": charge.id,
        "title": charge.title,
        "description": charge.description,
        "committee": charge.committee,
        "priority": charge.priority,
        "status": charge.status,
        "paw_links": charge.paw_links,
        "private": charge.private,
        "created_at": charge.created_at.isoformat()
    }
    emit('get_charge', charge_info, broadcast= broadcast)

##
## @brief      Creates a charge for a Committee.
##
## A charge for a committee is created, the default
## status for the charge will be Unapproved, which
## means that the charge is a Charge Initiative.
##
## @param      user_data  The user data to create a
##             charge for a committee. This can include:
##
##             - token (required): Token of creator.
##             - title (required): Charge title.
##             - committee (required): The charge's committee.
##             - private (optional): Set charge to private or not,
##                                   only committee head and admins
##                                   can use them. If not defined,
##                                   private will be set to True.
##             - priority (required): The charge's priority.
##             - description: The purpose of the charge.
##             - objectives: The objectives of a charge (Array).
##
##
## @return     { description_of_the_return_value }
##
@socketio.on('create_charge')
@ensure_dict
@get_user
def create_charge(user, user_data):
    committee = Committees.query.filter_by(id = user_data.get("committee","")).first()

    if committee is None or user is None:
        emit("create_charge", Response.UsrChargeDontExist)
        return;

    # Get the members role.
    membership = committee.members.filter_by(member= user).first()

    if (user.id != committee.head and user.is_admin == False):
        emit("create_charge", Response.PermError)
        return;

    if "title" not in user_data:
        emit ("create_charge", Response.InvalidTitle)
        return;

    if ("priority" not in user_data or
        type(user_data["priority"]) != int or
        user_data["priority"] < 0 or
        user_data["priority"] > 2):
        emit ("create_charge", Response.InvalidPriority)
        return;

    # Only admins and committee heads can make charges public.
    if ('private' in user_data and not user_data['private'] and
        not user.is_admin and user.id != committee.head):
        emit("create_charge", Response.PermError)
        return;

    charge = Charges(title = user_data["title"])
    charge.author = user.id
    charge.description = user_data.get("description", "")
    charge.committee = committee.id
    charge.status = 0
    charge.priority = 0
    charge.objectives = user_data.get("objectives", [])
    charge.schedule = user_data.get("schedules", [])
    charge.resources = user_data.get("resources", [])
    charge.stakeholders = user_data.get("stakeholders", [])
    charge.paw_links = user_data.get("paw_links", "")
    charge.private = user_data.get("private", True)

    db.session.add(charge)

    try:
        db.session.commit()
        emit('create_charge', Response.AddSuccess)
        get_charges(user_data, broadcast= True)
    except Exception as e:
        db.session.rollback()
        db.session.flush()
        emit("create_charge", Response.AddError)

##
## @brief       edits a charge
##
## A charge for a committee is created, the default
## status for the charge will be Unapproved, which
## means that the charge is a Charge Initiative.
##
## @param      user_data  The user data to create a
##             charge for a committee. This can include:
##
##             - token (required): Token of creator.
##             - charge (requred): charge to edit
##             - title : new title
##             - description : new description
##             - priority: new priority
##             - status: new status
##             - private (optional): Set charge to private or not,
##                                   only committee head and admins
##                                   can set privacy. If not defined,
##                                   private will be set to True.
##
## @return     { description_of_the_return_value }
##
@socketio.on('edit_charge')
@ensure_dict
@get_user
def edit_charge(user, user_data):
    charge = Charges.query.filter_by(id = user_data.get("charge",-1)).first()

    if charge is None or user is None:
        emit('edit_charge', Response.UsrChargeDontExist)
        return

    committee = Committees.query.filter_by(id = charge.committee).first()
    membership = committee.members.filter_by(member= user).first()

    if (user.id != committee.head and user.is_admin == False and
        (membership is None or membership.role != Roles.ActiveMember)):
        emit("edit_charge", Response.PermError)
        return

    # Only admins and committee heads can make charges public.
    if ('private' in user_data and not user_data['private'] and
        not user.is_admin and user.id != committee.head):
        emit("edit_charge", Response.PermError)
        return

    # Only admins can move charges to a different committee.
    if ('committee' in user_data and user.is_admin == False):
        emit("edit_charge", Response.PermError)
        return

    for key in user_data:
        if (key == "description" or key == "title" or key == "priority" or
            key == "status" or key == "paw_links" or key == "private" or key == "committee"):
            setattr(charge, key, user_data[key])

    try:
        db.session.commit()
        # Send successful edit notification to user
        # and broadcast charge changes.
        emit("edit_charge", Response.EditSuccess)
        get_charge(user_data, broadcast= True)
        get_charges(user_data, broadcast= True)
    except Exception as e:
        db.session.rollback()
        emit("edit_charge", Response.EditError)
<commit_msg>Move filtering logic into database call for get_all_charges<commit_after>"""
filename: controllers.py
description: Cronrollers for charges.
created by: Omar De La Hoz (oed7416@rit.edu), Chris Lemelin (cxl8826@rit.edu)
created on: 12/05/17
"""

from flask_socketio import emit
from app.decorators import ensure_dict, get_user
from app import socketio, db
from app.charges.models import *
from app.committees.models import Committees
from app.members.models import Roles
from app.charges.charges_response import Response
from app.users.models import Users

##
## @brief      Gets all public charges.
##
## @param      broadcast  The broadcast
##
## @return     All public charges.
##
@socketio.on('get_all_charges')
def get_all_charges(broadcast = False):

    charges = Charges.query.filter_by(private = False).all()
    charge_ser = []

    for charge in charges:
        charge_ser.append({
            "id": charge.id,
            "title": charge.title,
            "description": charge.description,
            "committee": charge.committee,
            "priority": charge.priority,
            "status": charge.status,
            "paw_links": charge.paw_links,
            "created_at": charge.created_at.isoformat()
        });
    emit("get_all_charges", charge_ser, broadcast = broadcast)

##
## @brief      Gets the charges for a specific committee.
##
## @param      committee_id  The committee identifier
## @param      broadcast     Flag to broadcast list of charges
##                           to all users.
##
## @emit       Emits a list of charges for committee.
##
@socketio.on('get_charges')
@ensure_dict
@get_user
def get_charges(user, user_data, broadcast = False):
    charges = Charges.query.filter_by(committee= user_data.get("committee_id", "")).all()
    charge_ser = []

    if len(charges) == 0:
        emit("get_charges", charge_ser, broadcast = broadcast)
        return;

    committee = Committees.query.filter_by(id = user_data.get("committee_id", "")).first()
    membership = committee.members.filter_by(member= user).first()

    for charge in charges:
        if charge.private and membership is None:
            continue;

        charge_ser.append({
            "id": charge.id,
            "title": charge.title,
            "description": charge.description,
            "committee": charge.committee,
            "priority": charge.priority,
            "status": charge.status,
            "paw_links": charge.paw_links,
            "private": charge.private,
            "created_at": charge.created_at.isoformat()
        });
    emit("get_charges", charge_ser, broadcast = broadcast)

##
## @brief      Gets a charge by its ID.
##
## @param      charge     The charge identifier
## @param      broadcast  Flag to broadcast list of charges
##                        to all users.
##
## @emit       An objet containing the detailed view of a
##             charge.
##
@socketio.on('get_charge')
@ensure_dict
@get_user
def get_charge(user, user_data, broadcast = False):

    charge = Charges.query.filter_by(id= user_data["charge"]).first()

    if charge is None:
        emit('get_charge', Response.UsrChargeDontExist)
        return;

    committee = Committees.query.filter_by(id = charge.committee).first()
    membership = committee.members.filter_by(member= user).first()

    if charge.private and (membership is None and not user.is_admin):
        emit('get_charge', Response.PermError)
        return;

    charge_info = {
        "id": charge.id,
        "title": charge.title,
        "description": charge.description,
        "committee": charge.committee,
        "priority": charge.priority,
        "status": charge.status,
        "paw_links": charge.paw_links,
        "private": charge.private,
        "created_at": charge.created_at.isoformat()
    }
    emit('get_charge', charge_info, broadcast= broadcast)

##
## @brief      Creates a charge for a Committee.
##
## A charge for a committee is created, the default
## status for the charge will be Unapproved, which
## means that the charge is a Charge Initiative.
##
## @param      user_data  The user data to create a
##             charge for a committee. This can include:
##
##             - token (required): Token of creator.
##             - title (required): Charge title.
##             - committee (required): The charge's committee.
##             - private (optional): Set charge to private or not,
##                                   only committee head and admins
##                                   can use them. If not defined,
##                                   private will be set to True.
##             - priority (required): The charge's priority.
##             - description: The purpose of the charge.
##             - objectives: The objectives of a charge (Array).
##
##
## @return     { description_of_the_return_value }
##
@socketio.on('create_charge')
@ensure_dict
@get_user
def create_charge(user, user_data):
    committee = Committees.query.filter_by(id = user_data.get("committee","")).first()

    if committee is None or user is None:
        emit("create_charge", Response.UsrChargeDontExist)
        return;

    # Get the members role.
    membership = committee.members.filter_by(member= user).first()

    if (user.id != committee.head and user.is_admin == False):
        emit("create_charge", Response.PermError)
        return;

    if "title" not in user_data:
        emit ("create_charge", Response.InvalidTitle)
        return;

    if ("priority" not in user_data or
        type(user_data["priority"]) != int or
        user_data["priority"] < 0 or
        user_data["priority"] > 2):
        emit ("create_charge", Response.InvalidPriority)
        return;

    # Only admins and committee heads can make charges public.
    if ('private' in user_data and not user_data['private'] and
        not user.is_admin and user.id != committee.head):
        emit("create_charge", Response.PermError)
        return;

    charge = Charges(title = user_data["title"])
    charge.author = user.id
    charge.description = user_data.get("description", "")
    charge.committee = committee.id
    charge.status = 0
    charge.priority = 0
    charge.objectives = user_data.get("objectives", [])
    charge.schedule = user_data.get("schedules", [])
    charge.resources = user_data.get("resources", [])
    charge.stakeholders = user_data.get("stakeholders", [])
    charge.paw_links = user_data.get("paw_links", "")
    charge.private = user_data.get("private", True)

    db.session.add(charge)

    try:
        db.session.commit()
        emit('create_charge', Response.AddSuccess)
        get_charges(user_data, broadcast= True)
    except Exception as e:
        db.session.rollback()
        db.session.flush()
        emit("create_charge", Response.AddError)

##
## @brief       edits a charge
##
## A charge for a committee is created, the default
## status for the charge will be Unapproved, which
## means that the charge is a Charge Initiative.
##
## @param      user_data  The user data to create a
##             charge for a committee. This can include:
##
##             - token (required): Token of creator.
##             - charge (requred): charge to edit
##             - title : new title
##             - description : new description
##             - priority: new priority
##             - status: new status
##             - private (optional): Set charge to private or not,
##                                   only committee head and admins
##                                   can set privacy. If not defined,
##                                   private will be set to True.
##
## @return     { description_of_the_return_value }
##
@socketio.on('edit_charge')
@ensure_dict
@get_user
def edit_charge(user, user_data):
    charge = Charges.query.filter_by(id = user_data.get("charge",-1)).first()

    if charge is None or user is None:
        emit('edit_charge', Response.UsrChargeDontExist)
        return

    committee = Committees.query.filter_by(id = charge.committee).first()
    membership = committee.members.filter_by(member= user).first()

    if (user.id != committee.head and user.is_admin == False and
        (membership is None or membership.role != Roles.ActiveMember)):
        emit("edit_charge", Response.PermError)
        return

    # Only admins and committee heads can make charges public.
    if ('private' in user_data and not user_data['private'] and
        not user.is_admin and user.id != committee.head):
        emit("edit_charge", Response.PermError)
        return

    # Only admins can move charges to a different committee.
    if ('committee' in user_data and user.is_admin == False):
        emit("edit_charge", Response.PermError)
        return

    for key in user_data:
        if (key == "description" or key == "title" or key == "priority" or
            key == "status" or key == "paw_links" or key == "private" or key == "committee"):
            setattr(charge, key, user_data[key])

    try:
        db.session.commit()
        # Send successful edit notification to user
        # and broadcast charge changes.
        emit("edit_charge", Response.EditSuccess)
        get_charge(user_data, broadcast= True)
        get_charges(user_data, broadcast= True)
    except Exception as e:
        db.session.rollback()
        emit("edit_charge", Response.EditError)
<|endoftext|>


<commit_before>__author__ = 'richard'

import numpy as np
import pandas as pd

import scripts.plot_windtunnel as pwt

class Windtunnel():
    def __init__(self, experimental_condition):
        """
        boundary: (array)
            specify where walls are  (minx, maxx, miny, maxy)
        collision type:
            'elastic', 'crash'

        TODO: class  Wall. left wall right wall

        """
        self.walls = Walls()
        self.boundary = self.walls.boundary
        self.experimental_condition = experimental_condition

        self.heater_l = Heater("Left", self.experimental_condition)
        self.heater_r = Heater("Right", self.experimental_condition)

    def show(self):
        ax = pwt.plot_windtunnel(self)
        return ax

class Walls():
    def __init__(self):
        # these are real dims of our wind tunnel
        self.left = -0.127
        self.right = 0.127
        self.upwind = 1.0
        self.downwind = 0.0
        self.ceiling = 0.254
        self.floor = 0.
        self.boundary = [self.downwind, self.upwind, self.left, self.right, self.floor, self.ceiling]

    def in_bounds(self, position):
        xpos, ypos, zpos = position
        inside = True
        past_wall = []

        if xpos > self.upwind:  # beyond upwind(upwind) wall (end)
            inside = False
            past_wall.append('upwind')
        if xpos < self.downwind:  # too far behind
            inside = False
            past_wall.append('downwind')
        if ypos < self.left:  # too left
            inside = False
            past_wall.append('left')
        if ypos > self.right:  # too far right
            inside = False
            past_wall.append('right')
        if zpos > self.ceiling:  # too far above
            inside = False
            past_wall.append('ceiling')
        if zpos < self.floor:  # too far below
            inside = False
            past_wall.append('floor')

        return inside, past_wall

class Heater():
    def __init__(self, side, experimental_condition):
        ''' given {left, right, none, custom coords} place heater in the windtunnel

        Args:
        location

        returns [x,y, zmin, zmax, diam]
        '''
        self.side = side
        self.experimental_condition = experimental_condition

        if side == experimental_condition:
            self.is_on = True
        else:
            self.is_on = False

        colors = {False: 'black', True: 'red'}
        self.color = colors[self.is_on]

        self.zmin = 0.03800
        self.zmax = 0.11340
        self.diam = 0.00635
        (self.x_position, self.y_position) = self._set_xy_coords()

    def _set_xy_coords(self):
        x_coord = 0.864

        if self.side in "leftLeftLEFT":
            y_coord = -0.0507
        elif self.side in "rightRightRIGHT":
            y_coord = 0.0507
        elif self.side in 'controlControlCONTROL':
            x_coord, y_coord = None, None
        else:
            raise Exception('invalid location type specified')

        return (x_coord, y_coord)

class Plume():
    ''' Instantiates a plume object.

    Args:
    condition: {left|right|None}
    '''

    def __init__(self, experiment):
        # useful aliases
        self.experiment = experiment
        self.condition = experiment.condition
        self.walls = experiment.windtunnel.walls

        self.data = self.load_plume_data()
        try:
            self.resolution = abs(self.data.x_position.diff()[1])
        except AttributeError:  # if no plume, can't take diff() of no data
            self.resolution = None

    def load_plume_data(self):
        col_names = ['x_position', 'z_position', 'small_radius']

        if self.condition in 'controlControlCONTROL':
            return None  # TODO: make plume that's 0 everywhere
        elif self.condition in 'lLleftLeft':
            df = pd.read_csv('data/experiments/plume_data/left_plume_bounds.csv', names=col_names)
            df['y_position'] = self.experiment.windtunnel.heater_l.y_position
        elif self.condition in 'rightRight':
            df = pd.read_csv('data/experiments/plume_data/right_plume_bounds.csv', names=col_names)
            df['y_position'] = self.experiment.windtunnel.heater_r.y_position
        else:
            raise Exception('problem with loading plume data {}'.format(self.condition))

        return df

    def in_plume(self, position):
        in_bounds, _ = self.walls.in_bounds(position)
        x, y, z = position

        if self.condition in 'controlControlCONTROL':
            inPlume = False
        elif np.abs(self.data['x_position'] - x).min() > self.resolution:
            # too far from the plume in the upwind/downwind direction
            inPlume = False
        elif in_bounds is False:
            print("WARNING: sniffing outside of windtunnel bounds")
            inPlume = False
        else:
            plume_plane = self._get_nearest_plume_plane(x)
            minor_axis = plume_plane.small_radius
            minor_ax_major_ax_ratio = 3
            major_axis = minor_axis * minor_ax_major_ax_ratio

            # implementation of http://math.stackexchange.com/a/76463/291217

            value = (((y - plume_plane.y_position) ** 2) / minor_axis ** 2) + \
                    (((z - plume_plane.z_position) ** 2) / major_axis ** 2)

            if value <= 1:
                inPlume = True
            else:
                inPlume = False

        return inPlume

    def _get_nearest_plume_plane(self, x_position):
        """given x position, find nearest plan"""
        closest_plume_index = (np.abs(self.data['x_position'] - x_position)).argmin()
        plume_plane = self.data.loc[closest_plume_index]

        return plume_plane

    def show(self):
        fig, ax = pwt.plot_windtunnel(self.experiment.windtunnel)
        ax.axis('off')
        pwt.draw_plume(self, ax=ax)

if __name__ == '__main__':
    windtunnel = Windtunnel('left')<commit_msg>new Plume subtypes<commit_after>__author__ = 'richard'

import numpy as np
import pandas as pd

import scripts.plot_windtunnel as pwt

class Windtunnel():
    def __init__(self, experimental_condition):
        """
        boundary: (array)
            specify where walls are  (minx, maxx, miny, maxy)
        collision type:
            'elastic', 'crash'

        TODO: class  Wall. left wall right wall

        """
        self.walls = Walls()
        self.boundary = self.walls.boundary
        self.experimental_condition = experimental_condition

        self.heater_l = Heater("Left", self.experimental_condition)
        self.heater_r = Heater("Right", self.experimental_condition)

    def show(self):
        ax = pwt.plot_windtunnel(self)
        return ax

class Walls():
    def __init__(self):
        # these are real dims of our wind tunnel
        self.left = -0.127
        self.right = 0.127
        self.upwind = 1.0
        self.downwind = 0.0
        self.ceiling = 0.254
        self.floor = 0.
        self.boundary = [self.downwind, self.upwind, self.left, self.right, self.floor, self.ceiling]

    def in_bounds(self, position):
        xpos, ypos, zpos = position
        inside = True
        past_wall = []

        if xpos > self.upwind:  # beyond upwind(upwind) wall (end)
            inside = False
            past_wall.append('upwind')
        if xpos < self.downwind:  # too far behind
            inside = False
            past_wall.append('downwind')
        if ypos < self.left:  # too left
            inside = False
            past_wall.append('left')
        if ypos > self.right:  # too far right
            inside = False
            past_wall.append('right')
        if zpos > self.ceiling:  # too far above
            inside = False
            past_wall.append('ceiling')
        if zpos < self.floor:  # too far below
            inside = False
            past_wall.append('floor')

        return inside, past_wall

class Heater():
    def __init__(self, side, experimental_condition):
        ''' given {left, right, none, custom coords} place heater in the windtunnel

        Args:
        location

        returns [x,y, zmin, zmax, diam]
        '''
        self.side = side
        self.experimental_condition = experimental_condition

        if side == experimental_condition:
            self.is_on = True
        else:
            self.is_on = False

        colors = {False: 'black', True: 'red'}
        self.color = colors[self.is_on]

        self.zmin = 0.03800
        self.zmax = 0.11340
        self.diam = 0.00635
        (self.x_position, self.y_position) = self._set_xy_coords()

    def _set_xy_coords(self):
        x_coord = 0.864

        if self.side in "leftLeftLEFT":
            y_coord = -0.0507
        elif self.side in "rightRightRIGHT":
            y_coord = 0.0507
        elif self.side in 'controlControlCONTROL':
            x_coord, y_coord = None, None
        else:
            raise Exception('invalid location type specified')

        return (x_coord, y_coord)

class Plume(object):
    ''' Instantiates a plume object.

    Args:
    condition: {left|right|None}
    '''

    def __init__(self, experiment):
        # useful aliases
        self.experiment = experiment
        self.condition = experiment.condition
        self.walls = experiment.windtunnel.walls

class Boolean_Plume(Plume):
    def __init__(self, experiment):
        super(self.__class__, self).__init__(experiment)

        self.data = self._load_plume_data()

        try:
            self.resolution = abs(self.data.x_position.diff()[1])
        except AttributeError:  # if no plume, can't take diff() of no data
            self.resolution = None

    def in_plume(self, position):
        in_bounds, _ = self.walls.in_bounds(position)
        x, y, z = position

        if self.condition in 'controlControlCONTROL':
            inPlume = False
        elif np.abs(self.data['x_position'] - x).min() > self.resolution:
            # too far from the plume in the upwind/downwind direction
            inPlume = False
        elif in_bounds is False:
            print("WARNING: sniffing outside of windtunnel bounds")
            inPlume = False
        else:
            plume_plane = self._get_nearest_plume_plane(x)
            minor_axis = plume_plane.small_radius
            minor_ax_major_ax_ratio = 3
            major_axis = minor_axis * minor_ax_major_ax_ratio

            # implementation of http://math.stackexchange.com/a/76463/291217

            value = (((y - plume_plane.y_position) ** 2) / minor_axis ** 2) + \
                    (((z - plume_plane.z_position) ** 2) / major_axis ** 2)

            if value <= 1:
                inPlume = True
            else:
                inPlume = False

        return inPlume

    def show(self):
        fig, ax = pwt.plot_windtunnel(self.experiment.windtunnel)
        ax.axis('off')
        pwt.draw_plume(self, ax=ax)

    def _get_nearest_plume_plane(self, x_position):
        """given x position, find nearest plan"""
        closest_plume_index = (np.abs(self.data['x_position'] - x_position)).argmin()
        plume_plane = self.data.loc[closest_plume_index]

        return plume_plane

    def load_plume_data(self):
        col_names = ['x_position', 'z_position', 'small_radius']

        if self.condition in 'controlControlCONTROL':
            return None
        elif self.condition in 'lLleftLeft':
            df = pd.read_csv('data/experiments/plume_data/left_plume_bounds.csv', names=col_names)
            df['y_position'] = self.experiment.windtunnel.heater_l.y_position
        elif self.condition in 'rightRight':
            df = pd.read_csv('data/experiments/plume_data/right_plume_bounds.csv', names=col_names)
            df['y_position'] = self.experiment.windtunnel.heater_r.y_position
        else:
            raise Exception('problem with loading plume data {}'.format(self.condition))

        return df

class Timeavg_Plume(Plume):
    def __init__(self, experiment):
        super(Plume, self).__init__(experiment)

        self.data = self._load_plume_data()

    def in_plume(self, position):
        pass

    def show(self):
        pass

    def _load_plume_data(self):
        raise NotImplementedError

if __name__ == '__main__':
    windtunnel = Windtunnel('left')<|endoftext|>


<commit_before>from datetime import timedelta
import threading
import logging
from Queue import Queue

import dns
from dns.resolver import NoAnswer, NXDOMAIN, Timeout, NoNameservers
from dns.rdtypes.ANY.NS import NS as NS_class
from dns.rdtypes.IN.A import A as A_class

from core.analytics import ScheduledAnalytics
from core.datatypes import Hostname, Link, Element
from core.helpers import is_subdomain

class ResolveHostnames(ScheduledAnalytics):

    settings = {
        "frequency": timedelta(minutes=10),
        "name": "ProcessHostname",
        "description": "Resolves hostnames and extracts subdomains",
    }

    ACTS_ON = 'Hostname'
    EXPIRATION = timedelta(days=1)  # Analysis will expire after 1 day

    @classmethod
    def bulk(cls, hostnames):
        p = ParallelDnsResolver()
        results = p.mass_resolve(hostnames)
        for hostname, result in results.items():
            cls.each(hostname, result)

    @classmethod
    def each(cls, hostname, result):
        h = Hostname.get_or_create(hostname)
        domain = is_subdomain(hostname)
        if domain:
            d = Hostname.get_or_create(domain)
            l = Link.connect(h, d)
            l.add_history(tag='domain')
        for rtype, results in result.items():
            for rdata in results:
                try:
                    e = Element.add_text(rdata)
                    l = Link.connect(h, e)
                    l.add_history(tag=rtype, description='{} record'.format(rtype))
                except ValueError as e:
                    logging.warning("{} is not a valid datatype".format(rdata))

        h.analysis_done(cls.__name__)

class ParallelDnsResolver(object):
    """Will issue a producer-consumer object to bulk-resolve domains"""
    def __init__(self):
        self.queue = Queue()
        self.lock = threading.Lock()
        self.results = {}

    def mass_resolve(self, domains, num_threads=50):
        for d in domains:
            self.queue.put((d.value, 'A'))
            self.queue.put((d.value, 'NS'))

        threads = []
        for _ in xrange(num_threads):
            t = threading.Thread(target=self.consumer)
            t.start()
            threads.append(t)

        for t in threads:
            t.join()

        return self.results

    def consumer(self):
        while True:
            try:
                hostname, rtype = self.queue.get(False)
            except Exception:
                return
            try:
                results = dns.resolver.query(hostname, rtype)
                if results:
                    if hostname not in self.results:
                        self.results[hostname] = {}
                    text_results = []
                    for r in results:
                        if isinstance(r, NS_class):
                            text_results.append(r.target.to_text())
                        elif isinstance(r, A_class):
                            text_results.append(r.to_text())
                        else:
                            logging.error("Unknown record type: {}".format(type(r)))
                    self.results[hostname][rtype] = text_results
            except NoAnswer:
                continue
            except NXDOMAIN:
                continue
            except Timeout:
                continue
            except NoNameservers:
                continue
<commit_msg>Change log level<commit_after>from datetime import timedelta
import threading
import logging
from Queue import Queue

import dns
from dns.resolver import NoAnswer, NXDOMAIN, Timeout, NoNameservers
from dns.rdtypes.ANY.NS import NS as NS_class
from dns.rdtypes.IN.A import A as A_class

from core.analytics import ScheduledAnalytics
from core.datatypes import Hostname, Link, Element
from core.helpers import is_subdomain

class ResolveHostnames(ScheduledAnalytics):

    settings = {
        "frequency": timedelta(minutes=10),
        "name": "ProcessHostname",
        "description": "Resolves hostnames and extracts subdomains",
    }

    ACTS_ON = 'Hostname'
    EXPIRATION = timedelta(days=1)  # Analysis will expire after 1 day

    @classmethod
    def bulk(cls, hostnames):
        p = ParallelDnsResolver()
        results = p.mass_resolve(hostnames)
        for hostname, result in results.items():
            cls.each(hostname, result)

    @classmethod
    def each(cls, hostname, result):
        h = Hostname.get_or_create(hostname)
        domain = is_subdomain(hostname)
        if domain:
            d = Hostname.get_or_create(domain)
            l = Link.connect(h, d)
            l.add_history(tag='domain')
        for rtype, results in result.items():
            for rdata in results:
                logging.info("{} resoved to {} ({} record)".format(h.value, rdata, rtype))
                try:
                    e = Element.add_text(rdata)
                    l = Link.connect(h, e)
                    l.add_history(tag=rtype, description='{} record'.format(rtype))
                except ValueError as e:
                    logging.warning("{} is not a valid datatype".format(rdata))

        h.analysis_done(cls.__name__)

class ParallelDnsResolver(object):
    """Will issue a producer-consumer object to bulk-resolve domains"""
    def __init__(self):
        self.queue = Queue()
        self.lock = threading.Lock()
        self.results = {}

    def mass_resolve(self, domains, num_threads=50):
        for d in domains:
            self.queue.put((d.value, 'A'))
            self.queue.put((d.value, 'NS'))

        threads = []
        for _ in xrange(num_threads):
            t = threading.Thread(target=self.consumer)
            t.start()
            threads.append(t)

        for t in threads:
            t.join()

        return self.results

    def consumer(self):
        while True:
            try:
                hostname, rtype = self.queue.get(False)
            except Exception:
                return
            try:
                results = dns.resolver.query(hostname, rtype)
                if results:
                    if hostname not in self.results:
                        self.results[hostname] = {}
                    text_results = []
                    for r in results:
                        if isinstance(r, NS_class):
                            text_results.append(r.target.to_text())
                        elif isinstance(r, A_class):
                            text_results.append(r.to_text())
                        else:
                            logging.error("Unknown record type: {}".format(type(r)))
                    self.results[hostname][rtype] = text_results
            except NoAnswer:
                continue
            except NXDOMAIN:
                continue
            except Timeout:
                continue
            except NoNameservers:
                continue
<|endoftext|>