<commit_before>#
# Copyright (c) 2017 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# -*- coding: utf-8 -*-
import presets
import numpy as np
import pandas as pd
from os import path
import os
import glob
import shutil
import sys
import time
from logger import screen
from utils import list_all_classes_in_module, threaded_cmd_line_run, killed_processes
from subprocess import Popen
import signal
import argparse

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-p', '--preset',
                        help="(string) Name of a preset to run (as configured in presets.py)",
                        default=None,
                        type=str)
    parser.add_argument('-itf', '--ignore_tensorflow',
                        help="(flag) Don't test TensorFlow presets.",
                        action='store_true')
    parser.add_argument('-in', '--ignore_neon',
                        help="(flag) Don't test neon presets.",
                        action='store_true')

    args = parser.parse_args()
    if args.preset is not None:
        presets_lists = [args.preset]
    else:
        presets_lists = list_all_classes_in_module(presets)
    win_size = 10
    fail_count = 0
    test_count = 0
    read_csv_tries = 70

    # create a clean experiment directory
    test_name = '__test'
    test_path = os.path.join('./experiments', test_name)
    if path.exists(test_path):
        shutil.rmtree(test_path)

    for idx, preset_name in enumerate(presets_lists):
        preset = eval('presets.{}()'.format(preset_name))
        if preset.test:
            frameworks = []
            if preset.agent.tensorflow_support and not args.ignore_tensorflow:
                frameworks.append('tensorflow')
            if preset.agent.neon_support and not args.ignore_neon:
                frameworks.append('neon')

            for framework in frameworks:
                test_count += 1

                # run the experiment in a separate thread
                screen.log_title("Running test {} - {}".format(preset_name, framework))
                cmd = 'CUDA_VISIBLE_DEVICES='' python3 coach.py -p {} -f {} -e {} -n {} -cp "seed=0" &> test_log_{}_{}.txt '\
                    .format(preset_name, framework, test_name, preset.test_num_workers, preset_name, framework)
                p = Popen(cmd, shell=True, executable="/bin/bash", preexec_fn=os.setsid)

                # get the csv with the results
                csv_path = None
                csv_paths = []

                if preset.test_num_workers > 1:
                    # we have an evaluator
                    reward_str = 'Evaluation Reward'
                    filename_pattern = 'evaluator*.csv'
                else:
                    reward_str = 'Training Reward'
                    filename_pattern = 'worker*.csv'

                initialization_error = False
                test_passed = False

                tries_counter = 0
                while not csv_paths:
                    csv_paths = glob.glob(path.join(test_path, '*', filename_pattern))
                    if tries_counter > read_csv_tries:
                        break
                    tries_counter += 1
                    time.sleep(1)

                if csv_paths:
                    csv_path = csv_paths[0]

                    # verify results
                    csv = None
                    time.sleep(1)
                    averaged_rewards = [0]

                    last_num_episodes = 0
                    while csv is None or csv['Episode #'].values[-1] < preset.test_max_step_threshold:
                        try:
                            csv = pd.read_csv(csv_path)
                        except:
                            # sometimes the csv is being written at the same time we are
                            # trying to read it. no problem -> try again
                            continue

                        if reward_str not in csv.keys():
                            continue

                        rewards = csv[reward_str].values
                        rewards = rewards[~np.isnan(rewards)]

                        if len(rewards) >= win_size:
                            averaged_rewards = np.convolve(rewards, np.ones(win_size) / win_size, mode='valid')
                        else:
                            time.sleep(1)
                            continue

                        # print progress
                        percentage = int((100*last_num_episodes)/preset.test_max_step_threshold)
                        sys.stdout.write("\rReward: ({}/{})".format(round(averaged_rewards[-1], 1), preset.test_min_return_threshold))
                        sys.stdout.write(' Episode: ({}/{})'.format(last_num_episodes, preset.test_max_step_threshold))
                        sys.stdout.write(' {}%|{}{}|  '.format(percentage, '#'*int(percentage/10), ' '*(10-int(percentage/10))))
                        sys.stdout.flush()

                        if csv['Episode #'].shape[0] - last_num_episodes <= 0:
                            continue

                        last_num_episodes = csv['Episode #'].values[-1]

                        # check if reward is enough
                        if np.any(averaged_rewards > preset.test_min_return_threshold):
                            test_passed = True
                            break
                        time.sleep(1)

                # kill test and print result
                os.killpg(os.getpgid(p.pid), signal.SIGTERM)
                if test_passed:
                    screen.success("Passed successfully")
                else:
                    screen.error("Failed due to a mismatch with the golden", crash=False)
                    fail_count += 1
                shutil.rmtree(test_path)

    screen.separator()
    if fail_count == 0:
        screen.success(" Summary: " + str(test_count) + "/" + str(test_count) + " tests passed successfully")
    else:
        screen.error(" Summary: " + str(test_count - fail_count) + "/" + str(test_count) + " tests passed successfully")
<commit_msg>add options for more verbose test errors<commit_after>#
# Copyright (c) 2017 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# -*- coding: utf-8 -*-
import presets
import numpy as np
import pandas as pd
from os import path
import os
import glob
import shutil
import sys
import time
from logger import screen
from utils import list_all_classes_in_module, threaded_cmd_line_run, killed_processes
import subprocess
import signal
import argparse

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-p', '--preset',
                        help="(string) Name of a preset to run (as configured in presets.py)",
                        default=None,
                        type=str)
    parser.add_argument('-itf', '--ignore_tensorflow',
                        help="(flag) Don't test TensorFlow presets.",
                        action='store_true')
    parser.add_argument('-in', '--ignore_neon',
                        help="(flag) Don't test neon presets.",
                        action='store_true')
    parser.add_argument('-v', '--verbose',
                        help="(flag) display verbose logs in the event of an error",
                        action='store_true')
    parser.add_argument('--stop_after_first_failure',
                        help="(flag) stop executing tests after the first error",
                        action='store_true')

    args = parser.parse_args()
    if args.preset is not None:
        presets_lists = [args.preset]
    else:
        presets_lists = list_all_classes_in_module(presets)
    win_size = 10
    fail_count = 0
    test_count = 0
    read_csv_tries = 70

    # create a clean experiment directory
    test_name = '__test'
    test_path = os.path.join('./experiments', test_name)
    if path.exists(test_path):
        shutil.rmtree(test_path)

    for idx, preset_name in enumerate(presets_lists):
        preset = eval('presets.{}()'.format(preset_name))
        if preset.test:
            frameworks = []
            if preset.agent.tensorflow_support and not args.ignore_tensorflow:
                frameworks.append('tensorflow')
            if preset.agent.neon_support and not args.ignore_neon:
                frameworks.append('neon')

            for framework in frameworks:
                if args.stop_after_first_failure and fail_count > 0:
                    break

                test_count += 1

                # run the experiment in a separate thread
                screen.log_title("Running test {} - {}".format(preset_name, framework))
                log_file_name = 'test_log_{preset_name}_{framework}.txt'.format(
                    preset_name=preset_name,
                    framework=framework,
                )
                cmd = (
                    'CUDA_VISIBLE_DEVICES='' python3 coach.py '
                    '-p {preset_name} '
                    '-f {framework} '
                    '-e {test_name} '
                    '-n {num_workers} '
                    '-cp "seed=0" '
                    '&> {log_file_name} '
                ).format(
                    preset_name=preset_name,
                    framework=framework,
                    test_name=test_name,
                    num_workers=preset.test_num_workers,
                    log_file_name=log_file_name,
                )
                p = subprocess.Popen(cmd, shell=True, executable="/bin/bash", preexec_fn=os.setsid)

                # get the csv with the results
                csv_path = None
                csv_paths = []

                if preset.test_num_workers > 1:
                    # we have an evaluator
                    reward_str = 'Evaluation Reward'
                    filename_pattern = 'evaluator*.csv'
                else:
                    reward_str = 'Training Reward'
                    filename_pattern = 'worker*.csv'

                initialization_error = False
                test_passed = False

                tries_counter = 0
                while not csv_paths:
                    csv_paths = glob.glob(path.join(test_path, '*', filename_pattern))
                    if tries_counter > read_csv_tries:
                        break
                    tries_counter += 1
                    time.sleep(1)

                if csv_paths:
                    csv_path = csv_paths[0]

                    # verify results
                    csv = None
                    time.sleep(1)
                    averaged_rewards = [0]

                    last_num_episodes = 0
                    while csv is None or csv['Episode #'].values[-1] < preset.test_max_step_threshold:
                        try:
                            csv = pd.read_csv(csv_path)
                        except:
                            # sometimes the csv is being written at the same time we are
                            # trying to read it. no problem -> try again
                            continue

                        if reward_str not in csv.keys():
                            continue

                        rewards = csv[reward_str].values
                        rewards = rewards[~np.isnan(rewards)]

                        if len(rewards) >= win_size:
                            averaged_rewards = np.convolve(rewards, np.ones(win_size) / win_size, mode='valid')
                        else:
                            time.sleep(1)
                            continue

                        # print progress
                        percentage = int((100*last_num_episodes)/preset.test_max_step_threshold)
                        sys.stdout.write("\rReward: ({}/{})".format(round(averaged_rewards[-1], 1), preset.test_min_return_threshold))
                        sys.stdout.write(' Episode: ({}/{})'.format(last_num_episodes, preset.test_max_step_threshold))
                        sys.stdout.write(' {}%|{}{}|  '.format(percentage, '#'*int(percentage/10), ' '*(10-int(percentage/10))))
                        sys.stdout.flush()

                        if csv['Episode #'].shape[0] - last_num_episodes <= 0:
                            continue

                        last_num_episodes = csv['Episode #'].values[-1]

                        # check if reward is enough
                        if np.any(averaged_rewards > preset.test_min_return_threshold):
                            test_passed = True
                            break
                        time.sleep(1)

                # kill test and print result
                os.killpg(os.getpgid(p.pid), signal.SIGTERM)
                if test_passed:
                    screen.success("Passed successfully")
                else:
                    if csv_paths:
                        screen.error("Failed due to insufficient reward", crash=False)
                        screen.error("preset.test_max_step_threshold: {}".format(preset.test_max_step_threshold), crash=False)
                        screen.error("preset.test_min_return_threshold: {}".format(preset.test_min_return_threshold), crash=False)
                        screen.error("averaged_rewards: {}".format(averaged_rewards), crash=False)
                        screen.error("episode number: {}".format(csv['Episode #'].values[-1]), crash=False)
                    else:
                        screen.error("csv file never found", crash=False)
                        if args.verbose:
                            screen.error("command exitcode: {}".format(p.returncode), crash=False)
                            screen.error(open(log_file_name).read(), crash=False)

                    fail_count += 1
                shutil.rmtree(test_path)

    screen.separator()
    if fail_count == 0:
        screen.success(" Summary: " + str(test_count) + "/" + str(test_count) + " tests passed successfully")
    else:
        screen.error(" Summary: " + str(test_count - fail_count) + "/" + str(test_count) + " tests passed successfully")
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
gitolite-manager
Author: Tim Henderson
Contact: tim.tadh@gmail.com, tadh@case.edu
Copyright: 2013 All Rights Reserved, see LICENSE
'''

import urllib, re
import cgi
from logging import getLogger
log = getLogger('gm:view:user')

from pyramid.view import view_config
from pyramid import httpexceptions as httpexc
from pyramid.httpexceptions import HTTPFound, HTTPBadRequest
from pyramid.response import Response

from gitolite_manager import validate as v
from gitolite_manager.models.session import Session
from gitolite_manager.models.user import User
from gitolite_manager.controllers import key_controller, repo_controller

def get_user_name(user):
    email = user.email
    return email[:email.index('@')]

def tvars(request, extras):
    session = request.environ['gm_session']
    defaults = {
        'SITENAME' : 'Key Czar',
        'SITEURL' : request.application_url,
        'request' : request,
        'session' : session,
        'get_user_name' : get_user_name,
    }
    defaults.update(extras)
    print defaults
    return defaults

@view_config(
    route_name='user',
    request_method=['GET'],
    renderer='templates/user.html'
)
def user(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]
    return tvars(request, {
        'TITLE' : user + ' user',
        'user_name' : user,
        'keys_url': request.route_url('user/keys'),
        'add_key_url': request.route_url('user/addkey'),
        'partners_url': request.route_url('user/partners'),
    })

@view_config(
    route_name='user/keys',
    request_method=['GET'],
    renderer='templates/keys.html'
)
def keys(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]
    return tvars(request, {
        'TITLE' : 'keys for %s' % user,
        'user_name' : user,
    })

@view_config(
    route_name='user/addkey',
    request_method=['GET'],
    renderer='templates/addkey.html'
)
def addkey(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]
    return tvars(request, {
        'TITLE' : 'add key for %s' % user,
        'user_name' : user,
    })

addkey_schema = {
    'key': v.type_checker(cgi.FieldStorage),
    'csrf': v.type_checker(basestring)
}

@view_config(
    route_name='user/addkey',
    request_method=['POST'],
    renderer='templates/addkey.html'
)
def addkey_post(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.route_url("root"))

    email = session.user.email
    user = email[:email.index('@')]

    err, post = v.validate_dictionary(dict(request.POST), addkey_schema)
    if err:
        return tvars(request, {
            'TITLE' : 'add key for %s' % user,
            'user_name' : user,
            'errors': err,
        })
    elif not session.valid_csrf(post['csrf'], request.route_url('user/addkey')):
        return HTTPFound(request.route_url('root'))
    else:
        try:
            key = post['key'].file.read()
            key_controller.add_key(db, session.user, key)
        except Exception, e:
            log.exception(e)
            return tvars(request, {
                'TITLE' : 'add key for %s' % user,
                'user_name' : user,
                'errors': [e],
            })
        return HTTPFound(request.route_url('user/keys'))

rmkey_schema = {
    'keyid': v.type_checker(basestring) &
             v.format_checker(re.compile(r'[0-9]+')),
}

@view_config(
    route_name='user/rmkey',
    request_method=['GET'],
    renderer='templates/rmkey.html'
)
def rmkey(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]

    err, match = v.validate_dictionary(dict(request.matchdict), rmkey_schema)
    if err:
        return tvars(request, {
            'TITLE' : 'remove key for %s' % user,
            'user_name' : user,
            'errors': err,
        })
    try:
        key_controller.rm_key(db, session.user, int(match['keyid']))
    except Exception, e:
        return tvars(request, {
            'TITLE' : 'remove key for %s' % user,
            'user_name' : user,
            'errors': [str(e)],
        })
    return HTTPFound(request.route_url('user/keys'))

@view_config(
    route_name='user/partners',
    request_method=['GET'],
    renderer='templates/add-partner.html'
)
def partners(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]
    return tvars(request, {
        'TITLE' : 'partners for %s' % user,
        'user_name' : user,
    })

add_partner_schema = {
    'csrf': v.type_checker(basestring),
    'case_id': v.type_checker(basestring) &
               v.format_checker(re.compile(r'[a-z]{3}[0-9]*')),
    'repo_name': v.type_checker(basestring) &
                 v.format_checker(re.compile(r'[a-zA-Z][a-zA-Z0-9_-]*')),
}

@view_config(
    route_name='user/add-partner',
    request_method=['POST'],
    renderer='templates/add-partner.html'
)
def add_partners(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]

    err, post = v.validate_dictionary(dict(request.POST), add_partner_schema)
    if err:
        return tvars(request, {
            'TITLE' : 'add partner for %s' % user,
            'user_name' : user,
            'errors': err,
        })
    elif not session.valid_csrf(post['csrf'], request.route_url('user/add-partner')):
        return HTTPFound(request.route_url('root'))
    else:
        try:
            repo_controller.add_partner(db, session.user, post['case_id'],
                post['repo_name'])
        except Exception, e:
            return tvars(request, {
                'TITLE' : 'add partner for %s' % user,
                'user_name' : user,
                'errors': [e],
            })
        return HTTPFound(request.route_url('user/partners'))

rm_partner_schema = {
    'repo_id': v.type_checker(basestring) &
               v.format_checker(re.compile(r'[0-9]+')),
}

@view_config(
    route_name='user/rm-partner',
    request_method=['GET'],
    renderer='templates/rm-partner.html'
)
def rm_partner(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]

    err, match = v.validate_dictionary(dict(request.matchdict), rm_partner_schema)
    if err:
        return tvars(request, {
            'TITLE' : 'remove partner for %s' % user,
            'user_name' : user,
            'errors': err,
        })
    try:
        repo_controller.rm_partner(db, session.user, int(match['repo_id']))
    except Exception, e:
        return tvars(request, {
            'TITLE' : 'remove partner for %s' % user,
            'user_name' : user,
            'errors': [str(e)],
        })
    return HTTPFound(request.route_url('user/partners'))
<commit_msg>no print default<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
gitolite-manager
Author: Tim Henderson
Contact: tim.tadh@gmail.com, tadh@case.edu
Copyright: 2013 All Rights Reserved, see LICENSE
'''

import urllib, re
import cgi
from logging import getLogger
log = getLogger('gm:view:user')

from pyramid.view import view_config
from pyramid import httpexceptions as httpexc
from pyramid.httpexceptions import HTTPFound, HTTPBadRequest
from pyramid.response import Response

from gitolite_manager import validate as v
from gitolite_manager.models.session import Session
from gitolite_manager.models.user import User
from gitolite_manager.controllers import key_controller, repo_controller

def get_user_name(user):
    email = user.email
    return email[:email.index('@')]

def tvars(request, extras):
    session = request.environ['gm_session']
    defaults = {
        'SITENAME' : 'Key Czar',
        'SITEURL' : request.application_url,
        'request' : request,
        'session' : session,
        'get_user_name' : get_user_name,
    }
    defaults.update(extras)
    return defaults

@view_config(
    route_name='user',
    request_method=['GET'],
    renderer='templates/user.html'
)
def user(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]
    return tvars(request, {
        'TITLE' : user + ' user',
        'user_name' : user,
        'keys_url': request.route_url('user/keys'),
        'add_key_url': request.route_url('user/addkey'),
        'partners_url': request.route_url('user/partners'),
    })

@view_config(
    route_name='user/keys',
    request_method=['GET'],
    renderer='templates/keys.html'
)
def keys(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]
    return tvars(request, {
        'TITLE' : 'keys for %s' % user,
        'user_name' : user,
    })

@view_config(
    route_name='user/addkey',
    request_method=['GET'],
    renderer='templates/addkey.html'
)
def addkey(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]
    return tvars(request, {
        'TITLE' : 'add key for %s' % user,
        'user_name' : user,
    })

addkey_schema = {
    'key': v.type_checker(cgi.FieldStorage),
    'csrf': v.type_checker(basestring)
}

@view_config(
    route_name='user/addkey',
    request_method=['POST'],
    renderer='templates/addkey.html'
)
def addkey_post(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.route_url("root"))

    email = session.user.email
    user = email[:email.index('@')]

    err, post = v.validate_dictionary(dict(request.POST), addkey_schema)
    if err:
        return tvars(request, {
            'TITLE' : 'add key for %s' % user,
            'user_name' : user,
            'errors': err,
        })
    elif not session.valid_csrf(post['csrf'], request.route_url('user/addkey')):
        return HTTPFound(request.route_url('root'))
    else:
        try:
            key = post['key'].file.read()
            key_controller.add_key(db, session.user, key)
        except Exception, e:
            log.exception(e)
            return tvars(request, {
                'TITLE' : 'add key for %s' % user,
                'user_name' : user,
                'errors': [e],
            })
        return HTTPFound(request.route_url('user/keys'))

rmkey_schema = {
    'keyid': v.type_checker(basestring) &
             v.format_checker(re.compile(r'[0-9]+')),
}

@view_config(
    route_name='user/rmkey',
    request_method=['GET'],
    renderer='templates/rmkey.html'
)
def rmkey(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]

    err, match = v.validate_dictionary(dict(request.matchdict), rmkey_schema)
    if err:
        return tvars(request, {
            'TITLE' : 'remove key for %s' % user,
            'user_name' : user,
            'errors': err,
        })
    try:
        key_controller.rm_key(db, session.user, int(match['keyid']))
    except Exception, e:
        return tvars(request, {
            'TITLE' : 'remove key for %s' % user,
            'user_name' : user,
            'errors': [str(e)],
        })
    return HTTPFound(request.route_url('user/keys'))

@view_config(
    route_name='user/partners',
    request_method=['GET'],
    renderer='templates/add-partner.html'
)
def partners(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]
    return tvars(request, {
        'TITLE' : 'partners for %s' % user,
        'user_name' : user,
    })

add_partner_schema = {
    'csrf': v.type_checker(basestring),
    'case_id': v.type_checker(basestring) &
               v.format_checker(re.compile(r'[a-z]{3}[0-9]*')),
    'repo_name': v.type_checker(basestring) &
                 v.format_checker(re.compile(r'[a-zA-Z][a-zA-Z0-9_-]*')),
}

@view_config(
    route_name='user/add-partner',
    request_method=['POST'],
    renderer='templates/add-partner.html'
)
def add_partners(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]

    err, post = v.validate_dictionary(dict(request.POST), add_partner_schema)
    if err:
        return tvars(request, {
            'TITLE' : 'add partner for %s' % user,
            'user_name' : user,
            'errors': err,
        })
    elif not session.valid_csrf(post['csrf'], request.route_url('user/add-partner')):
        return HTTPFound(request.route_url('root'))
    else:
        try:
            repo_controller.add_partner(db, session.user, post['case_id'],
                post['repo_name'])
        except Exception, e:
            return tvars(request, {
                'TITLE' : 'add partner for %s' % user,
                'user_name' : user,
                'errors': [e],
            })
        return HTTPFound(request.route_url('user/partners'))

rm_partner_schema = {
    'repo_id': v.type_checker(basestring) &
               v.format_checker(re.compile(r'[0-9]+')),
}

@view_config(
    route_name='user/rm-partner',
    request_method=['GET'],
    renderer='templates/rm-partner.html'
)
def rm_partner(request):
    db = request.environ['db.session']
    session = request.environ['gm_session']
    if session.user is None:
        return HTTPFound(request.application_url)

    email = session.user.email
    user = email[:email.index('@')]

    err, match = v.validate_dictionary(dict(request.matchdict), rm_partner_schema)
    if err:
        return tvars(request, {
            'TITLE' : 'remove partner for %s' % user,
            'user_name' : user,
            'errors': err,
        })
    try:
        repo_controller.rm_partner(db, session.user, int(match['repo_id']))
    except Exception, e:
        return tvars(request, {
            'TITLE' : 'remove partner for %s' % user,
            'user_name' : user,
            'errors': [str(e)],
        })
    return HTTPFound(request.route_url('user/partners'))
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Unit Tests for remote procedure calls using queue
"""

import mox

from twisted.application.app import startApplication

from nova import context
from nova import exception
from nova import flags
from nova import rpc
from nova import test
from nova import service
from nova import manager

FLAGS = flags.FLAGS
flags.DEFINE_string("fake_manager", "nova.tests.service_unittest.FakeManager",
                    "Manager for testing")

class FakeManager(manager.Manager):
    """Fake manager for tests"""
    pass

class ServiceTestCase(test.BaseTestCase):
    """Test cases for rpc"""

    def setUp(self):  # pylint: disable=C0103
        super(ServiceTestCase, self).setUp()
        self.mox.StubOutWithMock(service, 'db')
        self.context = context.get_admin_context()

    def test_create(self):
        host = 'foo'
        binary = 'nova-fake'
        topic = 'fake'
        self.mox.StubOutWithMock(rpc,
                                 'AdapterConsumer',
                                 use_mock_anything=True)
        self.mox.StubOutWithMock(
                service.task, 'LoopingCall', use_mock_anything=True)
        rpc.AdapterConsumer(connection=mox.IgnoreArg(),
                            topic=topic,
                            proxy=mox.IsA(service.Service)).AndReturn(
                                    rpc.AdapterConsumer)

        rpc.AdapterConsumer(connection=mox.IgnoreArg(),
                            topic='%s.%s' % (topic, host),
                            proxy=mox.IsA(service.Service)).AndReturn(
                                    rpc.AdapterConsumer)

        rpc.AdapterConsumer.attach_to_twisted()
        rpc.AdapterConsumer.attach_to_twisted()

        # Stub out looping call a bit needlessly since we don't have an easy
        # way to cancel it (yet) when the tests finishes
        service.task.LoopingCall(mox.IgnoreArg()).AndReturn(
                        service.task.LoopingCall)
        service.task.LoopingCall.start(interval=mox.IgnoreArg(),
                                       now=mox.IgnoreArg())
        service.task.LoopingCall(mox.IgnoreArg()).AndReturn(
                        service.task.LoopingCall)
        service.task.LoopingCall.start(interval=mox.IgnoreArg(),
                                       now=mox.IgnoreArg())

        service_create = {'host': host,
                          'binary': binary,
                          'topic': topic,
                          'report_count': 0}
        service_ref = {'host': host,
                       'binary': binary,
                       'report_count': 0,
                       'id': 1}

        service.db.service_get_by_args(mox.IgnoreArg(),
                                       host,
                                       binary).AndRaise(exception.NotFound())
        service.db.service_create(mox.IgnoreArg(),
                                  service_create).AndReturn(service_ref)
        self.mox.ReplayAll()

        app = service.Service.create(host=host, binary=binary)
        startApplication(app, False)
        self.assert_(app)

    # We're testing sort of weird behavior in how report_state decides
    # whether it is disconnected, it looks for a variable on itself called
    # 'model_disconnected' and report_state doesn't really do much so this
    # these are mostly just for coverage
    def test_report_state(self):
        host = 'foo'
        binary = 'bar'
        service_ref = {'host': host,
                       'binary': binary,
                       'report_count': 0,
                       'id': 1}
        service.db.__getattr__('report_state')
        service.db.service_get_by_args(self.context,
                                       host,
                                       binary).AndReturn(service_ref)
        service.db.service_update(self.context, service_ref['id'],
                                  mox.ContainsKeyValue('report_count', 1))

        self.mox.ReplayAll()
        s = service.Service()
        rv = yield s.report_state(host, binary)

    def test_report_state_no_service(self):
        host = 'foo'
        binary = 'bar'
        service_create = {'host': host,
                          'binary': binary,
                          'report_count': 0}
        service_ref = {'host': host,
                       'binary': binary,
                       'report_count': 0,
                       'id': 1}

        service.db.__getattr__('report_state')
        service.db.service_get_by_args(self.context,
                                      host,
                                      binary).AndRaise(exception.NotFound())
        service.db.service_create(self.context,
                                  service_create).AndReturn(service_ref)
        service.db.service_get(self.context, service_ref['id']).AndReturn(service_ref)
        service.db.service_update(self.context, service_ref['id'],
                                  mox.ContainsKeyValue('report_count', 1))

        self.mox.ReplayAll()
        s = service.Service()
        rv = yield s.report_state(host, binary)

    def test_report_state_newly_disconnected(self):
        host = 'foo'
        binary = 'bar'
        service_ref = {'host': host,
                       'binary': binary,
                       'report_count': 0,
                       'id': 1}

        service.db.__getattr__('report_state')
        service.db.service_get_by_args(self.context,
                                       host,
                                       binary).AndRaise(Exception())

        self.mox.ReplayAll()
        s = service.Service()
        rv = yield s.report_state(host, binary)

        self.assert_(s.model_disconnected)

    def test_report_state_newly_connected(self):
        host = 'foo'
        binary = 'bar'
        service_ref = {'host': host,
                       'binary': binary,
                       'report_count': 0,
                       'id': 1}

        service.db.__getattr__('report_state')
        service.db.service_get_by_args(self.context,
                                       host,
                                       binary).AndReturn(service_ref)
        service.db.service_update(self.context, service_ref['id'],
                                  mox.ContainsKeyValue('report_count', 1))

        self.mox.ReplayAll()
        s = service.Service()
        s.model_disconnected = True
        rv = yield s.report_state(host, binary)

        self.assert_(not s.model_disconnected)
<commit_msg>Delays the creation of the looping calls that that check the queue until startService is called.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Unit Tests for remote procedure calls using queue
"""

import mox

from twisted.application.app import startApplication

from nova import context
from nova import exception
from nova import flags
from nova import rpc
from nova import test
from nova import service
from nova import manager

FLAGS = flags.FLAGS
flags.DEFINE_string("fake_manager", "nova.tests.service_unittest.FakeManager",
                    "Manager for testing")

class FakeManager(manager.Manager):
    """Fake manager for tests"""
    def test_method(self):
        return 'manager'

class ExtendedService(service.Service):
    def test_method(self):
        return 'service'

class ServiceManagerTestCase(test.BaseTestCase):
    """Test cases for Services"""

    def test_attribute_error_for_no_manager(self):
        serv = service.Service('test',
                               'test',
                               'test',
                               'nova.tests.service_unittest.FakeManager')
        self.assertRaises(AttributeError, getattr, serv, 'test_method')

    def test_message_gets_to_manager(self):
        serv = service.Service('test',
                               'test',
                               'test',
                               'nova.tests.service_unittest.FakeManager')
        serv.startService()
        self.assertEqual(serv.test_method(), 'manager')

    def test_override_manager_method(self):
        serv = ExtendedService('test',
                               'test',
                               'test',
                               'nova.tests.service_unittest.FakeManager')
        serv.startService()
        self.assertEqual(serv.test_method(), 'service')

class ServiceTestCase(test.BaseTestCase):
    """Test cases for Services"""

    def setUp(self):  # pylint: disable=C0103
        super(ServiceTestCase, self).setUp()
        self.mox.StubOutWithMock(service, 'db')
        self.context = context.get_admin_context()

    def test_create(self):
        host = 'foo'
        binary = 'nova-fake'
        topic = 'fake'

        # NOTE(vish): Create was moved out of mox replay to make sure that
        #             the looping calls are created in StartService.
        app = service.Service.create(host=host, binary=binary)

        self.mox.StubOutWithMock(rpc,
                                 'AdapterConsumer',
                                 use_mock_anything=True)
        self.mox.StubOutWithMock(
                service.task, 'LoopingCall', use_mock_anything=True)
        rpc.AdapterConsumer(connection=mox.IgnoreArg(),
                            topic=topic,
                            proxy=mox.IsA(service.Service)).AndReturn(
                                    rpc.AdapterConsumer)

        rpc.AdapterConsumer(connection=mox.IgnoreArg(),
                            topic='%s.%s' % (topic, host),
                            proxy=mox.IsA(service.Service)).AndReturn(
                                    rpc.AdapterConsumer)

        rpc.AdapterConsumer.attach_to_twisted()
        rpc.AdapterConsumer.attach_to_twisted()

        # Stub out looping call a bit needlessly since we don't have an easy
        # way to cancel it (yet) when the tests finishes
        service.task.LoopingCall(mox.IgnoreArg()).AndReturn(
                        service.task.LoopingCall)
        service.task.LoopingCall.start(interval=mox.IgnoreArg(),
                                       now=mox.IgnoreArg())
        service.task.LoopingCall(mox.IgnoreArg()).AndReturn(
                        service.task.LoopingCall)
        service.task.LoopingCall.start(interval=mox.IgnoreArg(),
                                       now=mox.IgnoreArg())

        service_create = {'host': host,
                          'binary': binary,
                          'topic': topic,
                          'report_count': 0}
        service_ref = {'host': host,
                       'binary': binary,
                       'report_count': 0,
                       'id': 1}

        service.db.service_get_by_args(mox.IgnoreArg(),
                                       host,
                                       binary).AndRaise(exception.NotFound())
        service.db.service_create(mox.IgnoreArg(),
                                  service_create).AndReturn(service_ref)
        self.mox.ReplayAll()

        startApplication(app, False)
        self.assert_(app)

    # We're testing sort of weird behavior in how report_state decides
    # whether it is disconnected, it looks for a variable on itself called
    # 'model_disconnected' and report_state doesn't really do much so this
    # these are mostly just for coverage
    def test_report_state(self):
        host = 'foo'
        binary = 'bar'
        service_ref = {'host': host,
                       'binary': binary,
                       'report_count': 0,
                       'id': 1}
        service.db.__getattr__('report_state')
        service.db.service_get_by_args(self.context,
                                       host,
                                       binary).AndReturn(service_ref)
        service.db.service_update(self.context, service_ref['id'],
                                  mox.ContainsKeyValue('report_count', 1))

        self.mox.ReplayAll()
        s = service.Service()
        rv = yield s.report_state(host, binary)

    def test_report_state_no_service(self):
        host = 'foo'
        binary = 'bar'
        service_create = {'host': host,
                          'binary': binary,
                          'report_count': 0}
        service_ref = {'host': host,
                       'binary': binary,
                       'report_count': 0,
                       'id': 1}

        service.db.__getattr__('report_state')
        service.db.service_get_by_args(self.context,
                                      host,
                                      binary).AndRaise(exception.NotFound())
        service.db.service_create(self.context,
                                  service_create).AndReturn(service_ref)
        service.db.service_get(self.context, service_ref['id']).AndReturn(service_ref)
        service.db.service_update(self.context, service_ref['id'],
                                  mox.ContainsKeyValue('report_count', 1))

        self.mox.ReplayAll()
        s = service.Service()
        rv = yield s.report_state(host, binary)

    def test_report_state_newly_disconnected(self):
        host = 'foo'
        binary = 'bar'
        service_ref = {'host': host,
                       'binary': binary,
                       'report_count': 0,
                       'id': 1}

        service.db.__getattr__('report_state')
        service.db.service_get_by_args(self.context,
                                       host,
                                       binary).AndRaise(Exception())

        self.mox.ReplayAll()
        s = service.Service()
        rv = yield s.report_state(host, binary)

        self.assert_(s.model_disconnected)

    def test_report_state_newly_connected(self):
        host = 'foo'
        binary = 'bar'
        service_ref = {'host': host,
                       'binary': binary,
                       'report_count': 0,
                       'id': 1}

        service.db.__getattr__('report_state')
        service.db.service_get_by_args(self.context,
                                       host,
                                       binary).AndReturn(service_ref)
        service.db.service_update(self.context, service_ref['id'],
                                  mox.ContainsKeyValue('report_count', 1))

        self.mox.ReplayAll()
        s = service.Service()
        s.model_disconnected = True
        rv = yield s.report_state(host, binary)

        self.assert_(not s.model_disconnected)

<|endoftext|>


<commit_before># coding=utf-8

from django.http import HttpResponseRedirect

# Create your views here.

from django.http import HttpResponse
import json
import os
import shutil
import uuid
import time

# patron_dir = "C:/etc/patron"
patron_dir = os.path.dirname(os.path.abspath(__file__)) + "/../etc/patron"
command_dir = os.path.dirname(os.path.abspath(__file__)) + "/../etc/commands"

admin_tenant_id = "admin"

def get_403_error():
    return "ERROR (Forbidden): Policy doesn't allow this operation to be performed. (HTTP 403) (Request-ID: req-%s)" % uuid.uuid4()

def get_404_error():
    return "ERROR (Unsupported): This operation is unsupported. (HTTP 404) (Request-ID: req-%s)" % uuid.uuid4()

def tenants(request):
    if request.method != "GET":
        return HttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

    response_data = []

    files = os.listdir(patron_dir + "/custom_policy")
    files.sort()
    i = 0
    display_names = {"admin": "Admins（云平台管理员）",
                     "tenant1": "企业A",
                     "tenant2": "企业B",
                     "tenant3": "企业C"}
    for f in files:
        print f
        tmp_tenant = {}
        tmp_tenant['id'] = f
        tmp_tenant['name'] = display_names[f]
        response_data.append(tmp_tenant)
        i += 1

    return HttpResponse(json.dumps(response_data), content_type="application/json")

def models(request, model_name):
    if not os.path.exists(patron_dir + "/model/" + model_name):
        return HttpResponse("The model doesn't exist, model = " + model_name, content_type="text/html")
    model_path = patron_dir + "/model/" + model_name

    if request.method == 'GET':
        print "method = " + request.method + ", file to read = " + model_path
        file_object = open(model_path, 'r')
        try:
            rules = []
            while 1:
                line = file_object.readline()
                if not line:
                    break
                rules.append(line.rstrip("\n").rstrip("\r"))

            response_data = rules
            # metadata_text = file_object.read()
        finally:
            file_object.close()

        return HttpResponse(json.dumps(response_data), content_type="application/json")
    else:
        print "Unsupported method = " + request.method
        return HttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

def metadata(request, tenant_id):
    if not os.path.exists(patron_dir + "/custom_policy/" + tenant_id):
        return HttpResponse("The tenant doesn't exist, tenant = " + tenant_id, content_type="text/html")
    metadata_path = patron_dir + "/custom_policy/" + tenant_id + "/metadata.json"

    if request.method == 'GET':
        print "method = " + request.method + ", file to read = " + metadata_path
        file_object = open(metadata_path, 'r')
        try:
            response_data = json.load(file_object)
            # metadata_text = file_object.read()
        finally:
            file_object.close()

        return HttpResponse(json.dumps(response_data), content_type="application/json")
    elif request.method == 'POST':
        print "method = " + request.method + ", file to write = " + metadata_path
        file_object = open(metadata_path, 'w')
        try:
            tmp = json.loads(request.body)
            jstr = json.dumps(tmp, ensure_ascii=False, indent=4)
            file_object.write(jstr.encode('utf-8'))
        finally:
            file_object.close()

            return HttpResponse("{\"status\":\"success\"}", content_type="text/html")
    else:
        print "Unsupported method = " + request.method
        return HttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

def policy(request, tenant_id, policy_name):
    if not os.path.exists(patron_dir + "/custom_policy/" + tenant_id):
        return HttpResponse("The tenant doesn't exist, tenant = " + tenant_id, content_type="text/html")

    if os.path.exists(patron_dir + "/custom_policy/" + tenant_id + "/" + policy_name):
        policy_path = patron_dir + "/custom_policy/" + tenant_id + "/" + policy_name
    elif os.path.exists(patron_dir + "/" + policy_name):
        policy_path = patron_dir + "/" + policy_name
    else:
        return HttpResponse("The policy doesn't exist, tenant = " + tenant_id + ", policy = " + policy_name,
                            content_type="text/html")

    if request.method == 'GET':
        print "method = " + request.method + ", file to read = " + policy_path
        file_object = open(policy_path, 'r')
        try:
            rules = []
            while 1:
                line = file_object.readline()
                if not line:
                    break
                rules.append(line.rstrip("\n").rstrip("\r"))

            response_data = rules
            # metadata_text = file_object.read()
        finally:
            file_object.close()

        return HttpResponse(json.dumps(response_data), content_type="application/json")
    elif request.method == 'POST':
        print "method = " + request.method + ", file to write = " + policy_path
        file_object = open(policy_path, 'w')
        try:
            tmp = json.loads(request.body)
            rules = ""
            for line in tmp:
                rules += line + "\n"
            rules = rules.rstrip("\n")
            file_object.write(rules)
        finally:
            file_object.close()

            return HttpResponse("{\"status\":\"success\"}", content_type="text/html")
    else:
        print "Unsupported method = " + request.method
        return HttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

def users(request, tenant_id):
    if request.method != "GET":
        return HttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

    if not os.path.exists(patron_dir + "/custom_policy/" + tenant_id):
        return HttpResponse("The tenant doesn't exist, tenant = " + tenant_id, content_type="text/html")

    userlist_path = patron_dir + "/custom_policy/" + tenant_id + "/" + "users.txt"
    print "method = " + request.method + ", file to read = " + userlist_path
    file_object = open(userlist_path, 'r')
    user_list = file_object.read().split(",")
    response_data = user_list

    return HttpResponse(json.dumps(response_data), content_type="application/json")

def get_short_command(command):
    word_list = command.split(" ")
    if len(word_list) < 2:
        return command
    else:
        return word_list[0] + " " + word_list[1]

def get_action(command):
    m = {"nova list": "compute:get_all",
         "nova service-list": "compute_extension:services",
         "nova boot": "compute:create",
         "nova show": "compute:get",
         "nova delete": "compute:delete"}
    if m.has_key(command):
        return m[command]
    else:
        return ""

def get_object(command):
    word_list = command.split(" ")
    if len(word_list) < 3:
        return ""
    else:
        return word_list[len(word_list) - 1]

def get_command_output(command):
    output_path = command_dir + "/" + command + ".txt"
    try:
        file_object = open(output_path, 'r')
    except:
        return ""
    return file_object.read()

def enforce_command(tenant_id, sub, obj, act):
    if tenant_id == admin_tenant_id:
        res = True
        print "sub = " + sub + ", obj = " + obj + ", act = " + act + ", res = " + str(res)
        return res

    if act == "compute_extension:services":
        res = False
        print "sub = " + sub + ", obj = " + obj + ", act = " + act + ", res = " + str(res)
        return res

    if tenant_id == "tenant3":
        res = False
        print "sub = " + sub + ", obj = " + obj + ", act = " + act + ", res = " + str(res)
        return res

    if sub == "admin":
        res = True
        print "sub = " + sub + ", obj = " + obj + ", act = " + act + ", res = " + str(res)
        return res

    policy_path = patron_dir + "/custom_policy/" + tenant_id + "/custom-policy.json"
    try:
        file_object = open(policy_path, 'r')
    except:
        res = True
        print "sub = " + sub + ", obj = " + obj + ", act = " + act + ", res = " + str(res)
        return res

    if obj != "":
        rule = '"%s": "target_id:%s and user_id:%s"' % (act, obj, sub)
    else:
        rule = '"%s": "user_id:%s"' % (act, sub)
    print "rule = " + rule
    rules = file_object.read()
    if rule in rules:
        res = True
    else:
        res = False

    print "sub = " + sub + ", obj = " + obj + ", act = " + act + ", res = " + str(res)
    return res

def commands(request, tenant_id, user_name):
    if request.method != "GET":
        return HttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

    response_data = ["nova list",
                     "nova service-list",
                     "nova boot --flavor m1.nano --image cirros --nic net-id=c4eb995e-748d-4684-a956-10d0ad0e73fd --security-group default vm1",
                     "nova show vm1",
                     "nova delete vm1"]
    return HttpResponse(json.dumps(response_data), content_type="application/json")

def command(request, tenant_id, user_name, command):
    return HttpResponse("test", content_type="text/plain")

    if request.method != "GET":
        return HttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

    if not os.path.exists(patron_dir + "/custom_policy/" + tenant_id):
        return HttpResponse("The tenant doesn't exist, tenant = " + tenant_id, content_type="text/html")

    userlist_path = patron_dir + "/custom_policy/" + tenant_id + "/" + "users.txt"
    print "method = " + request.method + ", file to read = " + userlist_path
    file_object = open(userlist_path, 'r')
    user_list = file_object.read().split(",")
    if user_name not in user_list:
        return HttpResponse("The user doesn't exist, tenant = " + tenant_id + ", user = " + user_name, content_type="text/html")

    print "method = " + request.method + ", tenant = " + tenant_id + ", user = " + user_name + ", command to run = " + command

    time.sleep(1)

    sub = user_name
    obj = get_object(command)
    act = get_action(get_short_command(command))
    if not enforce_command(tenant_id, sub, obj, act):
        response_data = get_403_error()
        return HttpResponse(response_data, content_type="text/plain")

    response_data = get_command_output(get_short_command(command))
    if response_data == "":
        response_data = get_404_error()
    return HttpResponse(response_data, content_type="text/plain")

def reset(request):
    shutil.rmtree(patron_dir)
    print "deleted " + patron_dir

    patron_copy_dir = patron_dir + "_copy"
    shutil.copytree(patron_copy_dir, patron_dir)
    print "copied from " + patron_copy_dir + " to " + patron_dir

    return HttpResponse("{\"status\":\"success\"}", content_type="application/json")

def redirect_handler(request, url_path):
    return HttpResponseRedirect('static/' + url_path + '.html')

def mainpage_handler(request):
    return HttpResponseRedirect('static/Portal.html')
<commit_msg>Return Access-Control-Allow-Origin header.<commit_after># coding=utf-8

from django.http import HttpResponseRedirect

# Create your views here.

from django.http import HttpResponse
import json
import os
import shutil
import uuid
import time

# patron_dir = "C:/etc/patron"
patron_dir = os.path.dirname(os.path.abspath(__file__)) + "/../etc/patron"
command_dir = os.path.dirname(os.path.abspath(__file__)) + "/../etc/commands"

admin_tenant_id = "admin"

def get_403_error():
    return "ERROR (Forbidden): Policy doesn't allow this operation to be performed. (HTTP 403) (Request-ID: req-%s)" % uuid.uuid4()

def get_404_error():
    return "ERROR (Unsupported): This operation is unsupported. (HTTP 404) (Request-ID: req-%s)" % uuid.uuid4()

def MyHttpResponse(*args, **kwargs):
    response = HttpResponse(*args, **kwargs)
    response["Access-Control-Allow-Origin"] = "*"
    response["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS"
    response["Access-Control-Max-Age"] = "1000"
    response["Access-Control-Allow-Headers"] = "*"
    return response

def tenants(request):
    if request.method != "GET":
        return MyHttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

    response_data = []

    files = os.listdir(patron_dir + "/custom_policy")
    files.sort()
    i = 0
    display_names = {"admin": "Admins（云平台管理员）",
                     "tenant1": "企业A",
                     "tenant2": "企业B",
                     "tenant3": "企业C"}
    for f in files:
        print f
        tmp_tenant = {}
        tmp_tenant['id'] = f
        tmp_tenant['name'] = display_names[f]
        response_data.append(tmp_tenant)
        i += 1

    return MyHttpResponse(json.dumps(response_data), content_type="application/json")

def models(request, model_name):
    if not os.path.exists(patron_dir + "/model/" + model_name):
        return MyHttpResponse("The model doesn't exist, model = " + model_name, content_type="text/html")
    model_path = patron_dir + "/model/" + model_name

    if request.method == 'GET':
        print "method = " + request.method + ", file to read = " + model_path
        file_object = open(model_path, 'r')
        try:
            rules = []
            while 1:
                line = file_object.readline()
                if not line:
                    break
                rules.append(line.rstrip("\n").rstrip("\r"))

            response_data = rules
            # metadata_text = file_object.read()
        finally:
            file_object.close()

        return MyHttpResponse(json.dumps(response_data), content_type="application/json")
    else:
        print "Unsupported method = " + request.method
        return MyHttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

def metadata(request, tenant_id):
    if not os.path.exists(patron_dir + "/custom_policy/" + tenant_id):
        return MyHttpResponse("The tenant doesn't exist, tenant = " + tenant_id, content_type="text/html")
    metadata_path = patron_dir + "/custom_policy/" + tenant_id + "/metadata.json"

    if request.method == 'GET':
        print "method = " + request.method + ", file to read = " + metadata_path
        file_object = open(metadata_path, 'r')
        try:
            response_data = json.load(file_object)
            # metadata_text = file_object.read()
        finally:
            file_object.close()

        return MyHttpResponse(json.dumps(response_data), content_type="application/json")
    elif request.method == 'POST':
        print "method = " + request.method + ", file to write = " + metadata_path
        file_object = open(metadata_path, 'w')
        try:
            tmp = json.loads(request.body)
            jstr = json.dumps(tmp, ensure_ascii=False, indent=4)
            file_object.write(jstr.encode('utf-8'))
        finally:
            file_object.close()

            return MyHttpResponse("{\"status\":\"success\"}", content_type="text/html")
    else:
        print "Unsupported method = " + request.method
        return MyHttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

def policy(request, tenant_id, policy_name):
    if not os.path.exists(patron_dir + "/custom_policy/" + tenant_id):
        return MyHttpResponse("The tenant doesn't exist, tenant = " + tenant_id, content_type="text/html")

    if os.path.exists(patron_dir + "/custom_policy/" + tenant_id + "/" + policy_name):
        policy_path = patron_dir + "/custom_policy/" + tenant_id + "/" + policy_name
    elif os.path.exists(patron_dir + "/" + policy_name):
        policy_path = patron_dir + "/" + policy_name
    else:
        return MyHttpResponse("The policy doesn't exist, tenant = " + tenant_id + ", policy = " + policy_name,
                            content_type="text/html")

    if request.method == 'GET':
        print "method = " + request.method + ", file to read = " + policy_path
        file_object = open(policy_path, 'r')
        try:
            rules = []
            while 1:
                line = file_object.readline()
                if not line:
                    break
                rules.append(line.rstrip("\n").rstrip("\r"))

            response_data = rules
            # metadata_text = file_object.read()
        finally:
            file_object.close()

        return MyHttpResponse(json.dumps(response_data), content_type="application/json")
    elif request.method == 'POST':
        print "method = " + request.method + ", file to write = " + policy_path
        file_object = open(policy_path, 'w')
        try:
            tmp = json.loads(request.body)
            rules = ""
            for line in tmp:
                rules += line + "\n"
            rules = rules.rstrip("\n")
            file_object.write(rules)
        finally:
            file_object.close()

            return MyHttpResponse("{\"status\":\"success\"}", content_type="text/html")
    else:
        print "Unsupported method = " + request.method
        return MyHttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

def users(request, tenant_id):
    if request.method != "GET":
        return MyHttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

    if not os.path.exists(patron_dir + "/custom_policy/" + tenant_id):
        return MyHttpResponse("The tenant doesn't exist, tenant = " + tenant_id, content_type="text/html")

    userlist_path = patron_dir + "/custom_policy/" + tenant_id + "/" + "users.txt"
    print "method = " + request.method + ", file to read = " + userlist_path
    file_object = open(userlist_path, 'r')
    user_list = file_object.read().split(",")
    response_data = user_list

    return MyHttpResponse(json.dumps(response_data), content_type="application/json")

def get_short_command(command):
    word_list = command.split(" ")
    if len(word_list) < 2:
        return command
    else:
        return word_list[0] + " " + word_list[1]

def get_action(command):
    m = {"nova list": "compute:get_all",
         "nova service-list": "compute_extension:services",
         "nova boot": "compute:create",
         "nova show": "compute:get",
         "nova delete": "compute:delete"}
    if m.has_key(command):
        return m[command]
    else:
        return ""

def get_object(command):
    word_list = command.split(" ")
    if len(word_list) < 3:
        return ""
    else:
        return word_list[len(word_list) - 1]

def get_command_output(command):
    output_path = command_dir + "/" + command + ".txt"
    try:
        file_object = open(output_path, 'r')
    except:
        return ""
    return file_object.read()

def enforce_command(tenant_id, sub, obj, act):
    if tenant_id == admin_tenant_id:
        res = True
        print "sub = " + sub + ", obj = " + obj + ", act = " + act + ", res = " + str(res)
        return res

    if act == "compute_extension:services":
        res = False
        print "sub = " + sub + ", obj = " + obj + ", act = " + act + ", res = " + str(res)
        return res

    if tenant_id == "tenant3":
        res = False
        print "sub = " + sub + ", obj = " + obj + ", act = " + act + ", res = " + str(res)
        return res

    if sub == "admin":
        res = True
        print "sub = " + sub + ", obj = " + obj + ", act = " + act + ", res = " + str(res)
        return res

    policy_path = patron_dir + "/custom_policy/" + tenant_id + "/custom-policy.json"
    try:
        file_object = open(policy_path, 'r')
    except:
        res = True
        print "sub = " + sub + ", obj = " + obj + ", act = " + act + ", res = " + str(res)
        return res

    if obj != "":
        rule = '"%s": "target_id:%s and user_id:%s"' % (act, obj, sub)
    else:
        rule = '"%s": "user_id:%s"' % (act, sub)
    print "rule = " + rule
    rules = file_object.read()
    if rule in rules:
        res = True
    else:
        res = False

    print "sub = " + sub + ", obj = " + obj + ", act = " + act + ", res = " + str(res)
    return res

def commands(request, tenant_id, user_name):
    if request.method != "GET":
        return MyHttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

    response_data = ["nova list",
                     "nova service-list",
                     "nova boot --flavor m1.nano --image cirros --nic net-id=c4eb995e-748d-4684-a956-10d0ad0e73fd --security-group default vm1",
                     "nova show vm1",
                     "nova delete vm1"]
    return MyHttpResponse(json.dumps(response_data), content_type="application/json")

def command(request, tenant_id, user_name, command):
    return MyHttpResponse("test", content_type="text/plain")

    if request.method != "GET":
        return MyHttpResponse("Unsupported HTTP method: " + request.method, content_type="text/html")

    if not os.path.exists(patron_dir + "/custom_policy/" + tenant_id):
        return MyHttpResponse("The tenant doesn't exist, tenant = " + tenant_id, content_type="text/html")

    userlist_path = patron_dir + "/custom_policy/" + tenant_id + "/" + "users.txt"
    print "method = " + request.method + ", file to read = " + userlist_path
    file_object = open(userlist_path, 'r')
    user_list = file_object.read().split(",")
    if user_name not in user_list:
        return MyHttpResponse("The user doesn't exist, tenant = " + tenant_id + ", user = " + user_name, content_type="text/html")

    print "method = " + request.method + ", tenant = " + tenant_id + ", user = " + user_name + ", command to run = " + command

    time.sleep(1)

    sub = user_name
    obj = get_object(command)
    act = get_action(get_short_command(command))
    if not enforce_command(tenant_id, sub, obj, act):
        response_data = get_403_error()
        return MyHttpResponse(response_data, content_type="text/plain")

    response_data = get_command_output(get_short_command(command))
    if response_data == "":
        response_data = get_404_error()
    return MyHttpResponse(response_data, content_type="text/plain")

def reset(request):
    shutil.rmtree(patron_dir)
    print "deleted " + patron_dir

    patron_copy_dir = patron_dir + "_copy"
    shutil.copytree(patron_copy_dir, patron_dir)
    print "copied from " + patron_copy_dir + " to " + patron_dir

    return MyHttpResponse("{\"status\":\"success\"}", content_type="application/json")

def redirect_handler(request, url_path):
    return HttpResponseRedirect('static/' + url_path + '.html')

def mainpage_handler(request):
    return HttpResponseRedirect('static/Portal.html')
<|endoftext|>


<commit_before>#!/usr/bin/env python
from will.main import WillBot

if __name__ == '__main__':
    bot = WillBot(plugins_dirs=["plugins",], template_dirs=["templates",])
    bot.bootstrap()
    <commit_msg>Cleans up run script<commit_after>#!/usr/bin/env python
from will.main import WillBot

if __name__ == '__main__':
    bot = WillBot()
    bot.bootstrap()
<|endoftext|>


<commit_before>from charms.reactive import when, when_not, set_state, remove_state, when_none
from charms.hadoop import get_hadoop_base
from charms.reactive.helpers import data_changed
from jujubigdata.handlers import YARN, HDFS
from jujubigdata import utils
from charmhelpers.core import hookenv, unitdata

@when('hadoop.installed')
@when_not('resourcemanager.configured')
def configure_resourcemanager():
    local_hostname = hookenv.local_unit().replace('/', '-')
    private_address = hookenv.unit_get('private-address')
    ip_addr = utils.resolve_private_address(private_address)
    hadoop = get_hadoop_base()
    yarn = YARN(hadoop)
    yarn.configure_resourcemanager()
    yarn.configure_jobhistory()
    yarn.start_jobhistory()
    hadoop.open_ports('resourcemanager')
    utils.update_kv_hosts({ip_addr: local_hostname})
    set_state('resourcemanager.configured')

@when('hdfs.related')
@when_not('nodemanager.related')
def blockednodemanager(hdfs):
    hookenv.status_set('blocked', 'Waiting for relation to NodeManager')

@when('nodemanager.related')
@when_not('hdfs.related')
def blockedhdfs(hdfs):
    hookenv.status_set('blocked', 'Waiting for relation to HDFS')

@when_none('nodemanager.related', 'hdfs.related')
def blockedboth():
    hookenv.status_set('blocked', 'Waiting for relation to NodeManager and HDFS')

@when('nodemanager.related', 'hdfs.ready')
@when_not('nodemanager.registered')
def waitingnodemanager(nodemanager, hdfs):
    hookenv.status_set('waiting', 'Waiting for NodeManager registration')

@when('nodemanager.registered')
@when_not('hdfs.ready')
def waitinghdfs(nodemanager):
    hookenv.status_set('waiting', 'Waiting for HDFS Ready')

@when('nodemanager.related', 'hdfs.related')
@when_none('hdfs.ready', 'nodemanager.registered')
def waitingboth(nodemanager, hdfs):
    hookenv.status_set('waiting', 'Waiting for HDFS Ready and NodeManager Registration')

@when('resourcemanager.started', 'nodemanager.related')
def send_info(nodemanager):
    hadoop = get_hadoop_base()
    local_hostname = hookenv.local_unit().replace('/', '-')
    port = hadoop.dist_config.port('resourcemanager')
    hs_http = hadoop.dist_config.port('jh_webapp_http')
    hs_ipc = hadoop.dist_config.port('jobhistory')

    utils.update_kv_hosts({node['ip']: node['host'] for node in nodemanager.nodes()})
    utils.manage_etc_hosts()

    nodemanager.send_spec(hadoop.spec())
    nodemanager.send_host(local_hostname)
    nodemanager.send_ports(port, hs_http, hs_ipc)
    nodemanager.send_ssh_key(utils.get_ssh_key('hdfs'))
    nodemanager.send_hosts_map(utils.get_kv_hosts())

@when('resourcemanager.started', 'nodemanager.registered')
def register_nodemanagers(nodemanager):
    hadoop = get_hadoop_base()
    yarn = YARN(hadoop)

    slaves = [node['host'] for node in nodemanager.nodes()]
    if data_changed('resourcemanager.slaves', slaves):
        unitdata.kv().set('resourcemanager.slaves', slaves)
        yarn.register_slaves(slaves)

    hookenv.status_set('active', 'Ready ({count} NodeManager{s})'.format(
        count=len(slaves),
        s='s' if len(slaves) > 1 else '',
    ))
    set_state('resourcemanager.ready')

@when('yarn.related', 'resourcemanager.ready')
def accept_clients(clients):
    hadoop = get_hadoop_base()
    private_address = hookenv.unit_get('private-address')
    ip_addr = utils.resolve_private_address(private_address)
    port = hadoop.dist_config.port('resourcemanager')
    hs_http = hadoop.dist_config.port('hs_http')
    hs_ipc = hadoop.dist_config.port('hs_ipc')

    clients.send_spec(hadoop.spec())
    clients.send_ip_addr(ip_addr)
    clients.send_ports(port, hs_http, hs_ipc)
    clients.send_ready(True)

@when('yarn.related')
@when_not('resourcemanager.ready')
def reject_clients(clients):
    clients.send_ready(False)

@when('hdfs.ready')
@when_not('resourcemanager.started')
def configure_hdfs(hdfs_rel):
    hadoop = get_hadoop_base()
    hdfs = HDFS(hadoop)
    yarn = YARN(hadoop)
    # FIX HARDCODED HOSTNAME 'NAMENODE'
    utils.update_kv_hosts({hdfs_rel.ip_addr(): 'namenode'})
    utils.manage_etc_hosts()
    # FIX THE NEXT LINE ONCE CORY FIXES HIS FIX
    #hdfs.configure_client('namenode', hdfs_rel.port())
    hdfs.configure_hdfs_base('namenode', hdfs_rel.port())
    yarn.start_resourcemanager()
    set_state('resourcemanager.started')

@when('resourcemanager.started')
@when_not('hdfs.ready')
def hdfs_departed():
    hadoop = get_hadoop_base()
    yarn = YARN(hadoop)
    hadoop.close_ports('resourcemanager')
    yarn.stop_resourcemanager()
    remove_state('resourcemanager.started')
    remove_state('resourcemanager.ready')

@when('resourcemanager.started', 'nodemanager.departing')
def unregister_nodemanager(nodemanager):
    hadoop = get_hadoop_base()
    yarn = YARN(hadoop)
    nodes_leaving = nodemanager.nodes()  # only returns nodes in "leaving" state

    slaves = unitdata.kv().get('nodemanager.slaves')
    slaves_leaving = [node['host'] for node in nodes_leaving]
    hookenv.log('Slaves leaving: {}'.format(slaves_leaving))

    slaves_remaining = list(set(slaves) ^ set(slaves_leaving))
    unitdata.kv().set('resourcemanager.slaves', slaves_remaining)
    yarn.register_slaves(slaves_remaining)

    utils.remove_kv_hosts({node['ip']: node['host'] for node in nodes_leaving})
    utils.manage_etc_hosts()

    if not slaves_remaining:
        remove_state('resourcemanager.ready')
<commit_msg>Fixed issue with -broken not working as expected<commit_after>from charms.reactive import when, when_not, set_state, remove_state, when_none
from charms.hadoop import get_hadoop_base
from charms.reactive.helpers import data_changed
from jujubigdata.handlers import YARN, HDFS
from jujubigdata import utils
from charmhelpers.core import hookenv, unitdata

@when('hadoop.installed')
@when_not('resourcemanager.configured')
def configure_resourcemanager():
    local_hostname = hookenv.local_unit().replace('/', '-')
    private_address = hookenv.unit_get('private-address')
    ip_addr = utils.resolve_private_address(private_address)
    hadoop = get_hadoop_base()
    yarn = YARN(hadoop)
    yarn.configure_resourcemanager()
    yarn.configure_jobhistory()
    yarn.start_jobhistory()
    hadoop.open_ports('resourcemanager')
    utils.update_kv_hosts({ip_addr: local_hostname})
    set_state('resourcemanager.configured')

@when('hdfs.related')
@when_not('nodemanager.related')
def blockednodemanager(hdfs):
    hookenv.status_set('blocked', 'Waiting for relation to NodeManager')

@when('nodemanager.related')
@when_not('hdfs.related')
def blockedhdfs(hdfs):
    hookenv.status_set('blocked', 'Waiting for relation to HDFS')

@when_none('nodemanager.related', 'hdfs.related')
def blockedboth():
    hookenv.status_set('blocked', 'Waiting for relation to NodeManager and HDFS')

@when('nodemanager.related', 'hdfs.ready')
@when_not('nodemanager.registered')
def waitingnodemanager(nodemanager, hdfs):
    hookenv.status_set('waiting', 'Waiting for NodeManager registration')

@when('nodemanager.registered', 'hdfs.related')
@when_not('hdfs.ready')
def waitinghdfs(nodemanager):
    hookenv.status_set('waiting', 'Waiting for HDFS Ready')

@when('nodemanager.related', 'hdfs.related')
@when_none('hdfs.ready', 'nodemanager.registered')
def waitingboth(nodemanager, hdfs):
    hookenv.status_set('waiting', 'Waiting for HDFS Ready and NodeManager Registration')

@when('resourcemanager.started', 'nodemanager.related')
def send_info(nodemanager):
    hadoop = get_hadoop_base()
    local_hostname = hookenv.local_unit().replace('/', '-')
    port = hadoop.dist_config.port('resourcemanager')
    hs_http = hadoop.dist_config.port('jh_webapp_http')
    hs_ipc = hadoop.dist_config.port('jobhistory')

    utils.update_kv_hosts({node['ip']: node['host'] for node in nodemanager.nodes()})
    utils.manage_etc_hosts()

    nodemanager.send_spec(hadoop.spec())
    nodemanager.send_host(local_hostname)
    nodemanager.send_ports(port, hs_http, hs_ipc)
    nodemanager.send_ssh_key(utils.get_ssh_key('hdfs'))
    nodemanager.send_hosts_map(utils.get_kv_hosts())

@when('resourcemanager.started', 'nodemanager.registered')
def register_nodemanagers(nodemanager):
    hadoop = get_hadoop_base()
    yarn = YARN(hadoop)

    slaves = [node['host'] for node in nodemanager.nodes()]
    if data_changed('resourcemanager.slaves', slaves):
        unitdata.kv().set('resourcemanager.slaves', slaves)
        yarn.register_slaves(slaves)

    hookenv.status_set('active', 'Ready ({count} NodeManager{s})'.format(
        count=len(slaves),
        s='s' if len(slaves) > 1 else '',
    ))
    set_state('resourcemanager.ready')

@when('yarn.related', 'resourcemanager.ready')
def accept_clients(clients):
    hadoop = get_hadoop_base()
    private_address = hookenv.unit_get('private-address')
    ip_addr = utils.resolve_private_address(private_address)
    port = hadoop.dist_config.port('resourcemanager')
    hs_http = hadoop.dist_config.port('hs_http')
    hs_ipc = hadoop.dist_config.port('hs_ipc')

    clients.send_spec(hadoop.spec())
    clients.send_ip_addr(ip_addr)
    clients.send_ports(port, hs_http, hs_ipc)
    clients.send_ready(True)

@when('yarn.related')
@when_not('resourcemanager.ready')
def reject_clients(clients):
    clients.send_ready(False)

@when('hdfs.ready')
@when_not('resourcemanager.started')
def configure_hdfs(hdfs_rel):
    hadoop = get_hadoop_base()
    hdfs = HDFS(hadoop)
    yarn = YARN(hadoop)
    # FIX HARDCODED HOSTNAME 'NAMENODE'
    utils.update_kv_hosts({hdfs_rel.ip_addr(): 'namenode'})
    utils.manage_etc_hosts()
    # FIX THE NEXT LINE ONCE CORY FIXES HIS FIX
    #hdfs.configure_client('namenode', hdfs_rel.port())
    hdfs.configure_hdfs_base('namenode', hdfs_rel.port())
    yarn.start_resourcemanager()
    set_state('resourcemanager.started')

@when('resourcemanager.started')
@when_not('hdfs.ready')
def hdfs_departed():
    hadoop = get_hadoop_base()
    yarn = YARN(hadoop)
    hadoop.close_ports('resourcemanager')
    yarn.stop_resourcemanager()
    remove_state('resourcemanager.started')
    remove_state('resourcemanager.ready')

@when('resourcemanager.started', 'nodemanager.departing')
def unregister_nodemanager(nodemanager):
    hadoop = get_hadoop_base()
    yarn = YARN(hadoop)
    nodes_leaving = nodemanager.nodes()  # only returns nodes in "leaving" state

    slaves = unitdata.kv().get('nodemanager.slaves', [])
    slaves_leaving = [node['host'] for node in nodes_leaving]
    hookenv.log('Slaves leaving: {}'.format(slaves_leaving))

    slaves_remaining = list(set(slaves) ^ set(slaves_leaving))
    unitdata.kv().set('resourcemanager.slaves', slaves_remaining)
    yarn.register_slaves(slaves_remaining)

    utils.remove_kv_hosts(slaves_leaving)
    utils.manage_etc_hosts()

    if not slaves_remaining:
        remove_state('resourcemanager.ready')

    nodemanager.dismiss()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys

import django
from django.conf import settings

settings.configure(
    DEBUG=True,
    DATABASES={
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
        }
    },
    ROOT_URLCONF='urls',
    INSTALLED_APPS=(
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.sessions',
        'django.contrib.admin',
        'arcutils',
    ),
    MIDDLEWARE_CLASSES=[],
    LDAP={
        "default": {
            "host": "ldap://ldap-login.oit.pdx.edu",
            "username": "",
            "password": "",
            "search_dn": "ou=people,dc=pdx,dc=edu",
        }
    },
    LOGGING_CONFIG='arcutils.logging.basic',
)

if django.VERSION[:2] >= (1, 7):
    from django import setup
else:
    setup = lambda: None

from django.test.simple import DjangoTestSuiteRunner
from django.test.utils import setup_test_environment

setup()
setup_test_environment()
test_runner = DjangoTestSuiteRunner(verbosity=1)

# Django 1.6 doesn't have the app loader, so you have to manually call ready()
if django.VERSION[:2] < (1, 7):
    from arcutils.apps import ARCUtilsConfig
    ARCUtilsConfig().ready()

failures = test_runner.run_tests(['arcutils', ])
if failures:
    sys.exit(failures)
<commit_msg>Replace DjangoTestSuiteRunner w/ DiscoverRunner<commit_after>#!/usr/bin/env python
import sys

import django
from django.conf import settings

settings.configure(
    DEBUG=True,
    DATABASES={
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
        }
    },
    ROOT_URLCONF='urls',
    INSTALLED_APPS=(
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.sessions',
        'django.contrib.admin',
        'arcutils',
    ),
    MIDDLEWARE_CLASSES=[],
    LDAP={
        "default": {
            "host": "ldap://ldap-login.oit.pdx.edu",
            "username": "",
            "password": "",
            "search_dn": "ou=people,dc=pdx,dc=edu",
        }
    },
    LOGGING_CONFIG='arcutils.logging.basic',
)

if django.VERSION[:2] >= (1, 7):
    from django import setup
else:
    setup = lambda: None

from django.test.runner import DiscoverRunner
from django.test.utils import setup_test_environment

setup()
setup_test_environment()
test_runner = DiscoverRunner(verbosity=1)

# Django 1.6 doesn't have the app loader, so you have to manually call ready()
if django.VERSION[:2] < (1, 7):
    from arcutils.apps import ARCUtilsConfig
    ARCUtilsConfig().ready()

failures = test_runner.run_tests(['arcutils', ])
if failures:
    sys.exit(failures)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Utilities for the builds app."""

from __future__ import (
    absolute_import, division, print_function, unicode_literals)

import re

GH_REGEXS = [
    re.compile('github.com/(.+)/(.+)(?:\.git){1}'),
    re.compile('github.com/(.+)/(.+)'),
    re.compile('github.com:(.+)/(.+).git'),
]

BB_REGEXS = [
    re.compile('bitbucket.org/(.+)/(.+)/'),
    re.compile('bitbucket.org/(.+)/(.+)'),
    re.compile('bitbucket.org:(.+)/(.+)\.git'),
]

# TODO: I think this can be different than `gitlab.com`
# self.adapter.provider_base_url
GL_REGEXS = [
    re.compile('gitlab.com/(.+)/(.+)(?:\.git){1}'),
    re.compile('gitlab.com/(.+)/(.+)'),
    re.compile('gitlab.com:(.+)/(.+)\.git'),
]

def get_github_username_repo(url):
    if 'github' in url:
        for regex in GH_REGEXS:
            match = regex.search(url)
            if match:
                return match.groups()
    return (None, None)

def get_bitbucket_username_repo(url=None):
    if 'bitbucket' in url:
        for regex in BB_REGEXS:
            match = regex.search(url)
            if match:
                return match.groups()
    return (None, None)

def get_gitlab_username_repo(url=None):
    if 'gitlab' in url:
        for regex in GL_REGEXS:
            match = regex.search(url)
            if match:
                return match.groups()
    return (None, None)
<commit_msg>Fix regex for getting project and user<commit_after># -*- coding: utf-8 -*-
"""Utilities for the builds app."""

from __future__ import (
    absolute_import, division, print_function, unicode_literals)

import re

GH_REGEXS = [
    re.compile('github.com/(.+)/(.+)(?:\.git){1}$'),
    re.compile('github.com/(.+)/(.+)'),
    re.compile('github.com:(.+)/(.+).git$'),
]

BB_REGEXS = [
    re.compile('bitbucket.org/(.+)/(.+)/'),
    re.compile('bitbucket.org/(.+)/(.+)'),
    re.compile('bitbucket.org:(.+)/(.+)\.git$'),
]

# TODO: I think this can be different than `gitlab.com`
# self.adapter.provider_base_url
GL_REGEXS = [
    re.compile('gitlab.com/(.+)/(.+)(?:\.git){1}$'),
    re.compile('gitlab.com/(.+)/(.+)'),
    re.compile('gitlab.com:(.+)/(.+)\.git$'),
]

def get_github_username_repo(url):
    if 'github' in url:
        for regex in GH_REGEXS:
            match = regex.search(url)
            if match:
                return match.groups()
    return (None, None)

def get_bitbucket_username_repo(url=None):
    if 'bitbucket' in url:
        for regex in BB_REGEXS:
            match = regex.search(url)
            if match:
                return match.groups()
    return (None, None)

def get_gitlab_username_repo(url=None):
    if 'gitlab' in url:
        for regex in GL_REGEXS:
            match = regex.search(url)
            if match:
                return match.groups()
    return (None, None)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import os
import shutil

from django.conf import settings, global_settings
from django.core.management import execute_from_command_line

WAGTAIL_ROOT = os.path.dirname(__file__)
STATIC_ROOT = os.path.join(WAGTAIL_ROOT, 'test-static')

if not settings.configured:
    settings.configure(
        DATABASES={
            'default': {
                'ENGINE': 'django.db.backends.postgresql_psycopg2',
                'NAME': 'wagtaildemo',
                'USER': 'postgres',
            }
        },
        ROOT_URLCONF='wagtail.tests.urls',
        STATIC_URL='/static/',
        STATIC_ROOT=STATIC_ROOT,
        STATICFILES_FINDERS=(
            'django.contrib.staticfiles.finders.AppDirectoriesFinder',
            'compressor.finders.CompressorFinder',
        ),
        TEMPLATE_CONTEXT_PROCESSORS=global_settings.TEMPLATE_CONTEXT_PROCESSORS + (
            'django.core.context_processors.request',
        ),
        MIDDLEWARE_CLASSES=(
            'django.middleware.common.CommonMiddleware',
            'django.contrib.sessions.middleware.SessionMiddleware',
            'django.middleware.csrf.CsrfViewMiddleware',
            'django.contrib.auth.middleware.AuthenticationMiddleware',
            'django.contrib.messages.middleware.MessageMiddleware',
            'django.middleware.clickjacking.XFrameOptionsMiddleware',

            'wagtail.wagtailcore.middleware.SiteMiddleware',

            'wagtail.wagtailredirects.middleware.RedirectMiddleware',
        ),
        INSTALLED_APPS=[
            'django.contrib.contenttypes',
            'django.contrib.sessions',
            'django.contrib.auth',
            'django.contrib.messages',
            'django.contrib.staticfiles',

            'taggit',
            'south',
            'compressor',

            'wagtail.wagtailcore',
            'wagtail.wagtailadmin',
            'wagtail.wagtaildocs',
            'wagtail.wagtailsnippets',
            'wagtail.wagtailusers',
            'wagtail.wagtailimages',
            'wagtail.wagtailembeds',
            'wagtail.wagtailsearch',
            'wagtail.wagtailredirects',
        ]
    )

def runtests():
    argv = sys.argv[:1] + ['test'] + sys.argv[1:]
    try:
        execute_from_command_line(argv)
    finally:
        shutil.rmtree(STATIC_ROOT, ignore_errors=True)

if __name__ == '__main__':
    runtests()
<commit_msg>Add django.contrib.admin to test settings<commit_after>#!/usr/bin/env python
import sys
import os
import shutil

from django.conf import settings, global_settings
from django.core.management import execute_from_command_line

WAGTAIL_ROOT = os.path.dirname(__file__)
STATIC_ROOT = os.path.join(WAGTAIL_ROOT, 'test-static')

if not settings.configured:
    settings.configure(
        DATABASES={
            'default': {
                'ENGINE': 'django.db.backends.postgresql_psycopg2',
                'NAME': 'wagtaildemo',
                'USER': 'postgres',
            }
        },
        ROOT_URLCONF='wagtail.tests.urls',
        STATIC_URL='/static/',
        STATIC_ROOT=STATIC_ROOT,
        STATICFILES_FINDERS=(
            'django.contrib.staticfiles.finders.AppDirectoriesFinder',
            'compressor.finders.CompressorFinder',
        ),
        TEMPLATE_CONTEXT_PROCESSORS=global_settings.TEMPLATE_CONTEXT_PROCESSORS + (
            'django.core.context_processors.request',
        ),
        MIDDLEWARE_CLASSES=(
            'django.middleware.common.CommonMiddleware',
            'django.contrib.sessions.middleware.SessionMiddleware',
            'django.middleware.csrf.CsrfViewMiddleware',
            'django.contrib.auth.middleware.AuthenticationMiddleware',
            'django.contrib.messages.middleware.MessageMiddleware',
            'django.middleware.clickjacking.XFrameOptionsMiddleware',

            'wagtail.wagtailcore.middleware.SiteMiddleware',

            'wagtail.wagtailredirects.middleware.RedirectMiddleware',
        ),
        INSTALLED_APPS=[
            'django.contrib.contenttypes',
            'django.contrib.sessions',
            'django.contrib.auth',
            'django.contrib.messages',
            'django.contrib.staticfiles',
            'django.contrib.admin',

            'taggit',
            'south',
            'compressor',

            'wagtail.wagtailcore',
            'wagtail.wagtailadmin',
            'wagtail.wagtaildocs',
            'wagtail.wagtailsnippets',
            'wagtail.wagtailusers',
            'wagtail.wagtailimages',
            'wagtail.wagtailembeds',
            'wagtail.wagtailsearch',
            'wagtail.wagtailredirects',
        ]
    )

def runtests():
    argv = sys.argv[:1] + ['test'] + sys.argv[1:]
    try:
        execute_from_command_line(argv)
    finally:
        shutil.rmtree(STATIC_ROOT, ignore_errors=True)

if __name__ == '__main__':
    runtests()
<|endoftext|>


<commit_before>import logging

from django.apps import apps
from django_elasticsearch_dsl.registries import registry

from readthedocs.worker import app

log = logging.getLogger(__name__)

def _get_index(indices, index_name):
    """
    Get Index from all the indices

    :param indices: DED indices list
    :param index_name: Name of the index
    :return: DED Index
    """
    for index in indices:
        if str(index) == index_name:
            return index

def _get_document(model, document_class):
    """
    Get DED document class object from the model and name of document class

    :param model: The model class to find the document
    :param document_class: the name of the document class.
    :return: DED DocType object
    """
    documents = registry.get_documents(models=[model])

    for document in documents:
        if str(document) == document_class:
            return document

@app.task(queue='web')
def create_new_es_index(app_label, model_name, index_name, new_index_name):
    model = apps.get_model(app_label, model_name)
    indices = registry.get_indices(models=[model])
    old_index = _get_index(indices=indices, index_name=index_name)
    new_index = old_index.clone(name=new_index_name)
    new_index.create()

@app.task(queue='web')
def switch_es_index(app_label, model_name, index_name, new_index_name):
    model = apps.get_model(app_label, model_name)
    indices = registry.get_indices(models=[model])
    old_index = _get_index(indices=indices, index_name=index_name)
    new_index = old_index.clone(name=new_index_name)
    old_index_actual_name = None

    if old_index.exists():
        # Alias can not be used to delete an index.
        # https://www.elastic.co/guide/en/elasticsearch/reference/6.0/indices-delete-index.html
        # So get the index actual name to delete it
        old_index_info = old_index.get()
        # The info is a dictionary and the key is the actual name of the index
        old_index_actual_name = old_index_info.keys()[0]

    # Put alias into the new index name and delete the old index if its exist
    new_index.put_alias(name=index_name)
    if old_index_actual_name:
        old_index.connection.indices.delete(index=old_index_actual_name)

@app.task(queue='web')
def index_objects_to_es(app_label, model_name, document_class, index_name, chunk):
    # Chunk is a tuple with start and end index of queryset
    start = chunk[0]
    end = chunk[1]
    model = apps.get_model(app_label, model_name)
    document = _get_document(model=model, document_class=document_class)

    # Use queryset from model as the ids are specific
    queryset = model.objects.all()[start:end]
    log.info("Indexing model: {}, from:'{}' to '{}'".format(model.__name__, start, end))
    document().update(queryset.iterator(), index_name=index_name)

@app.task(queue='web')
def index_missing_objects(app_label, model_name, document_class, index_generation_time):
    """
    Task to insure that none of the object is missed from indexing.

    The object ids are sent to `index_objects_to_es` task for indexing.
    While the task is running, new objects can be created/deleted in database
    and they will not be in the tasks for indexing into ES.
    This task will index all the objects that got into DB after the `latest_indexed` timestamp
    to ensure that everything is in ES index.
    """
    model = apps.get_model(app_label, model_name)
    document = _get_document(model=model, document_class=document_class)
    queryset = document().get_queryset().exclude(modified_date__lte=index_generation_time)
    document().update(queryset.iterator())

    log.info("Indexed {} missing objects from model: {}'".format(queryset.count(), model.__name__))

    # TODO: Figure out how to remove the objects from ES index that has been deleted
<commit_msg>fixup<commit_after>import logging

from django.apps import apps
from django_elasticsearch_dsl.registries import registry

from readthedocs.worker import app

log = logging.getLogger(__name__)

def _get_index(indices, index_name):
    """
    Get Index from all the indices

    :param indices: DED indices list
    :param index_name: Name of the index
    :return: DED Index
    """
    for index in indices:
        if str(index) == index_name:
            return index

def _get_document(model, document_class):
    """
    Get DED document class object from the model and name of document class

    :param model: The model class to find the document
    :param document_class: the name of the document class.
    :return: DED DocType object
    """
    documents = registry.get_documents(models=[model])

    for document in documents:
        if str(document) == document_class:
            return document

@app.task(queue='web')
def create_new_es_index(app_label, model_name, index_name, new_index_name):
    model = apps.get_model(app_label, model_name)
    indices = registry.get_indices(models=[model])
    old_index = _get_index(indices=indices, index_name=index_name)
    new_index = old_index.clone(name=new_index_name)
    new_index.create()

@app.task(queue='web')
def switch_es_index(app_label, model_name, index_name, new_index_name):
    model = apps.get_model(app_label, model_name)
    indices = registry.get_indices(models=[model])
    old_index = _get_index(indices=indices, index_name=index_name)
    new_index = old_index.clone(name=new_index_name)
    old_index_actual_name = None

    if old_index.exists():
        # Alias can not be used to delete an index.
        # https://www.elastic.co/guide/en/elasticsearch/reference/6.0/indices-delete-index.html
        # So get the index actual name to delete it
        old_index_info = old_index.get()
        # The info is a dictionary and the key is the actual name of the index
        old_index_actual_name = old_index_info.keys()[0]

    # Put alias into the new index name and delete the old index if its exist
    new_index.put_alias(name=index_name)
    if old_index_actual_name:
        old_index.connection.indices.delete(index=old_index_actual_name)

@app.task(queue='web')
def index_objects_to_es(app_label, model_name, document_class, index_name,
                        chunk=None, objects_id=None):

    assert not chunk and objects_id, "You can not pass both chunk and objects_id"

    model = apps.get_model(app_label, model_name)
    document = _get_document(model=model, document_class=document_class)

    # Use queryset from model as the ids are specific
    queryset = model.objects.all()
    if chunk:
        # Chunk is a tuple with start and end index of queryset
        start = chunk[0]
        end = chunk[1]
        queryset = queryset[start:end]
    elif objects_id:
        queryset = queryset.filter(id__in=objects_id)

    log.info("Indexing model: {}, '{}' objects".format(model.__name__, queryset.count()))
    document().update(queryset.iterator(), index_name=index_name)

@app.task(queue='web')
def index_missing_objects(app_label, model_name, document_class, index_generation_time):
    """
    Task to insure that none of the object is missed from indexing.

    The object ids are sent to `index_objects_to_es` task for indexing.
    While the task is running, new objects can be created/deleted in database
    and they will not be in the tasks for indexing into ES.
    This task will index all the objects that got into DB after the `latest_indexed` timestamp
    to ensure that everything is in ES index.
    """
    model = apps.get_model(app_label, model_name)
    document = _get_document(model=model, document_class=document_class)
    queryset = document().get_queryset().exclude(modified_date__lte=index_generation_time)
    document().update(queryset.iterator())

    log.info("Indexed {} missing objects from model: {}'".format(queryset.count(), model.__name__))

    # TODO: Figure out how to remove the objects from ES index that has been deleted
<|endoftext|>