<commit_before>from radar.validation.core import Validation, Field, pass_call, pass_context, ValidationError
from radar.validation.validators import optional, required, not_in_future, in_, none_if_blank
from radar.validation.patient_number_validators import nhs_no, ukrdc_no, chi_no
from radar.models.patients import GENDERS
from radar.permissions import has_permission_for_organisation
from radar.organisations import is_chi_organisation, is_nhs_organisation, is_radar_organisation
from radar.models.organisations import ORGANISATION_TYPE_OTHER
from radar.validation.patient_number_validators import NUMBER_VALIDATORS

class RecruitPatientSearchValidation(Validation):
    first_name = Field([required()])
    last_name = Field([required()])
    date_of_birth = Field([required()])
    number = Field([required()])
    number_organisation = Field([required()])

    def validate_number_organisation(self, number_organisation):
        if not is_chi_organisation(number_organisation) and not is_nhs_organisation(number_organisation):
            raise ValidationError("Not a valid organisation.")

        return number_organisation

    @pass_call
    def validate(self, call, obj):
        number_organisation = obj['number_organisation']

        if number_organisation.type == ORGANISATION_TYPE_OTHER:
            number_validators = NUMBER_VALIDATORS.get(number_organisation.code)

            if number_validators is not None:
                call.validators_for_field(number_validators, obj, self.number)

        return obj

# TODO validate patient numbers
class RecruitPatientValidation(Validation):
    first_name = Field([none_if_blank(), optional()])
    last_name = Field([none_if_blank(), optional()])
    date_of_birth = Field([optional(), not_in_future()])
    gender = Field([optional(), in_(GENDERS.keys())])
    ethnicity_code = Field([optional()])
    recruited_by_organisation = Field([required()])
    cohort = Field([required()])

    @pass_context
    def validate_recruited_by_organisation(self, ctx, recruited_by_organisation):
        current_user = ctx['user']

        if not has_permission_for_organisation(current_user, recruited_by_organisation, 'has_recruit_patient_permission'):
            raise ValidationError('Permission denied!')

        return recruited_by_organisation

    @pass_call
    def validate(self, call, obj):
        radar_id = None

        for x in obj['patient_numbers']:
            if is_radar_organisation(x['organisation']):
                radar_id = int(number)
                break

        if radar_id is None:
            call.validators_for_field([required()], obj, self.first_name)
            call.validators_for_field([required()], obj, self.last_name)
            call.validators_for_field([required()], obj, self.date_of_birth)
            call.validators_for_field([required()], obj, self.gender)

        return obj
<commit_msg>Unused imports<commit_after>from radar.validation.core import Validation, Field, pass_call, pass_context, ValidationError
from radar.validation.validators import optional, required, not_in_future, in_, none_if_blank
from radar.models.patients import GENDERS
from radar.permissions import has_permission_for_organisation
from radar.organisations import is_chi_organisation, is_nhs_organisation, is_radar_organisation
from radar.models.organisations import ORGANISATION_TYPE_OTHER
from radar.validation.patient_number_validators import NUMBER_VALIDATORS

class RecruitPatientSearchValidation(Validation):
    first_name = Field([required()])
    last_name = Field([required()])
    date_of_birth = Field([required()])
    number = Field([required()])
    number_organisation = Field([required()])

    def validate_number_organisation(self, number_organisation):
        if not is_chi_organisation(number_organisation) and not is_nhs_organisation(number_organisation):
            raise ValidationError("Not a valid organisation.")

        return number_organisation

    @pass_call
    def validate(self, call, obj):
        number_organisation = obj['number_organisation']

        if number_organisation.type == ORGANISATION_TYPE_OTHER:
            number_validators = NUMBER_VALIDATORS.get(number_organisation.code)

            if number_validators is not None:
                call.validators_for_field(number_validators, obj, self.number)

        return obj

def get_radar_id(obj):
    for x in obj['patient_numbers']:
        if is_radar_organisation(x['organisation']):
            return int(x['number'])

    return None

# TODO validate patient numbers
class RecruitPatientValidation(Validation):
    first_name = Field([none_if_blank(), optional()])
    last_name = Field([none_if_blank(), optional()])
    date_of_birth = Field([optional(), not_in_future()])
    gender = Field([optional(), in_(GENDERS.keys())])
    ethnicity_code = Field([optional()])
    recruited_by_organisation = Field([required()])
    cohort = Field([required()])

    @pass_context
    def validate_recruited_by_organisation(self, ctx, recruited_by_organisation):
        current_user = ctx['user']

        if not has_permission_for_organisation(current_user, recruited_by_organisation, 'has_recruit_patient_permission'):
            raise ValidationError('Permission denied!')

        return recruited_by_organisation

    @pass_call
    def validate(self, call, obj):
        radar_id = get_radar_id(obj)

        if radar_id is None:
            call.validators_for_field([required()], obj, self.first_name)
            call.validators_for_field([required()], obj, self.last_name)
            call.validators_for_field([required()], obj, self.date_of_birth)
            call.validators_for_field([required()], obj, self.gender)

        return obj
<|endoftext|>


<commit_before>###############################################################################
# Copyright (c) 2012 Evgeny Proydakov <lord.tiran@gmail.com>
###############################################################################

import os
import sys
import xml.dom.minidom

###############################################################################
#                                 constant                                    #
###############################################################################

directory           = '.'
model_directory     = '.'
test_expansion      = '.rtestx'
trace_expansion     = '.trc'
result_expansion    = '.pmv'
build_dir_substr    = 'build'
rdo_ex_substr       = 'rdo'
rdo_test_ex_substr  = 'rdo_test'

null_file = 'null_temp_file'

if sys.platform == 'win32':
    rdo_ex_substr      = 'rdo.exe'
    rdo_test_ex_substr = 'rdo_test.exe'

dividing_line = '-------------------------------------------------------------------------------'

TARGET_CONSOLE   = 'CONSOLE'
TAGRET_CONVERTER = 'CONVERTOR'
TARGET_GUI       = 'GUI'

APP_CODE_TERMINATION_NORMAL = 0
APP_CODE_TERMINATION_ERROR  = 1

EXIT_CODE_TERMINATION_NORMAL	             = 0
EXIT_CODE_TERMINATION_ERROR_FILE_NOT_FOUND   = 1
EXIT_CODE_TERMINATION_ERROR_INVALID_INPUT    = 2
EXIT_CODE_TERMINATION_ERROR_RESULT           = 3
EXIT_CODE_TERMINATION_ERROR_TRACE            = 4
EXIT_CODE_TERMINATION_ERROR_RESULT_AND_TRACE = 5

# global exit code variable
G_EXIT_CODE = APP_CODE_TERMINATION_NORMAL

###############################################################################
#                                 functions                                   #
###############################################################################

def get_files_list(dir):
    dirs  = []
    nfile = []
    files = []
    for dirname, dirnames, filenames in os.walk(dir):
        dirs.append(dirname)
        for subdirname in dirnames:
            dirs.append(os.path.join(dirname, subdirname))
        for filename in filenames:
            files.append(os.path.join(dirname, filename))
    return files

def get_test_files(dir):
    files = get_files_list(dir)
    nfile = filter(lambda x: x.endswith(test_expansion), files)
    return nfile

def get_executables(dir):
    # get rdo_studio_console executable and test_rdo_studio_console executable

    files = get_files_list(dir)
    rdo_ex = filter(lambda x: x.endswith(rdo_ex_substr), files)[0]
    rdo_test_ex = filter(lambda x: x.endswith(rdo_test_ex_substr), files)[0]

    return rdo_ex, rdo_test_ex

def get_text_from_node_list(nodelist):
    rc = []
    for node in nodelist:
        if node.nodeType == node.TEXT_NODE:
            rc.append(node.data)
    return ''.join(rc)

###############################################################################
#                                 main code                                   #
###############################################################################

print "STARTED SCRIPT :", sys.argv[0]

# search rdo and rdo_test executables
executables = get_executables(directory)

rdo_ex      = executables[0]
rdo_test_ex = executables[1]

if not os.path.exists(rdo_ex) or not os.path.exists(rdo_test_ex):
    print 'Build app not found. Critical error !!!'
    exit(EXIT_CODE_TERMINATION_ERROR)

# search .rtestx files
files = get_test_files(model_directory)
files.sort()

print '\nDEBUG INFO\n'

print 'Find RDO executables    : ', executables, '\n'
print 'Find test project files : ', files, '\n'

# parse xml and start tests
print 'STARTED TEST CYCLE'

for task in files:
    cycle_exit_code = EXIT_CODE_TERMINATION_NORMAL

    print dividing_line

    dom = xml.dom.minidom.parse(task)

    dirname = os.path.dirname(task) + '/'

    model_name_with_ex = get_text_from_node_list(dom.getElementsByTagName('model')[0].childNodes)

    target = get_text_from_node_list(dom.getElementsByTagName('target')[0].childNodes)
    exit_code = get_text_from_node_list(dom.getElementsByTagName('exit_code')[0].childNodes)
    etalon_trace_name = get_text_from_node_list(dom.getElementsByTagName('trace')[0].childNodes)
    etalon_result_name =  get_text_from_node_list(dom.getElementsByTagName('result')[0].childNodes)

    print 'Project     :', task
    print 'Model file  :', model_name_with_ex
    print 'Target      :', target
    print 'Exit code   :', exit_code
    print 'Trace file  :', etalon_trace_name
    print 'Result file :', etalon_result_name
    print ''

    model = dirname + model_name_with_ex
    etalon_trace = dirname + etalon_trace_name
    etalon_result =  dirname + etalon_result_name

    model_name = model_name_with_ex.partition('.')[0]

    simulation_trace = dirname + model_name + trace_expansion
    simulation_result = dirname + model_name + result_expansion

    if target == TARGET_CONSOLE:
        simulation_code = os.system(rdo_ex + ' -i ' + model
                                           + ' >> ' + null_file)

        print "SIMYLATION EXIT CODE :", simulation_code

        simulation_exit_code_string = ''
        if cmp(simulation_code, exit_code):
            simulation_exit_code_string = "OK"
        else:
            simulation_exit_code_string = "ERROR"

        print "CHECK SIM EXIT CODE  :", simulation_exit_code_string

        check_exit_code_string = ''
        if simulation_code == EXIT_CODE_TERMINATION_NORMAL:
            test_code = os.system(rdo_test_ex + ' -T ' + etalon_trace
                                              + ' -R ' + etalon_result
                                              + ' -t ' + simulation_trace
                                              + ' -r ' + simulation_result
                                              + ' >> ' + null_file)

            cycle_exit_code = APP_CODE_TERMINATION_ERROR

            if test_code == EXIT_CODE_TERMINATION_NORMAL:
                cycle_exit_code = APP_CODE_TERMINATION_NORMAL
                check_exit_code_string = 'OK'

            elif test_code == EXIT_CODE_TERMINATION_ERROR_FILE_NOT_FOUND:
                check_exit_code_string = 'ERROR FILE NOT FOUND'

            elif test_code == EXIT_CODE_TERMINATION_ERROR_INVALID_INPUT:
                check_exit_code_string = 'ERROR INVALID INPUT'

            elif test_code == EXIT_CODE_TERMINATION_ERROR_RESULT:
                check_exit_code_string = 'ERROR RESULT'

            elif test_code == EXIT_CODE_TERMINATION_ERROR_TRACE:
                check_exit_code_string = 'ERROR TRACE'

            elif test_code == EXIT_CODE_TERMINATION_ERROR_RESULT_AND_TRACE:
                check_exit_code_string = 'ERROR RESULT AND TRACE'

            else:
                check_exit_code_string = 'ERROR UNKNOWN'

        print "TEST EXIT CODE       :", test_code
        print "CHECK TEST CODE      :", check_exit_code_string

    else:
        print 'INVALID TARGET'

    # remove temp file
    #os.remove(simulation_trace)
    #os.remove(simulation_result)

    if not cycle_exit_code == EXIT_CODE_TERMINATION_NORMAL:
        G_EXIT_CODE = APP_CODE_TERMINATION_ERROR

print

os.remove(null_file)

sys.exit(G_EXIT_CODE)<commit_msg> - updated python test system script : fixed incorrect return code after calling application testing<commit_after>###############################################################################
# Copyright (c) 2012 Evgeny Proydakov <lord.tiran@gmail.com>
###############################################################################

import os
import sys
import subprocess
import xml.dom.minidom

###############################################################################
#                                 constant                                    #
###############################################################################

directory           = '.'
model_directory     = '.'
test_expansion      = '.rtestx'
trace_expansion     = '.trc'
result_expansion    = '.pmv'
build_dir_substr    = 'build'
rdo_ex_substr       = 'rdo'
rdo_test_ex_substr  = 'rdo_test'

null_file = 'null_temp_file'

if sys.platform == 'win32':
    rdo_ex_substr      = 'rdo.exe'
    rdo_test_ex_substr = 'rdo_test.exe'

dividing_line = '-------------------------------------------------------------------------------'

TARGET_CONSOLE   = 'CONSOLE'
TAGRET_CONVERTER = 'CONVERTOR'
TARGET_GUI       = 'GUI'

APP_CODE_TERMINATION_NORMAL = 0
APP_CODE_TERMINATION_ERROR  = 1

EXIT_CODE_TERMINATION_NORMAL                 = 0
EXIT_CODE_TERMINATION_ERROR_FILE_NOT_FOUND   = 1
EXIT_CODE_TERMINATION_ERROR_INVALID_INPUT    = 2
EXIT_CODE_TERMINATION_ERROR_RESULT           = 3
EXIT_CODE_TERMINATION_ERROR_TRACE            = 4
EXIT_CODE_TERMINATION_ERROR_RESULT_AND_TRACE = 5

# global exit code variable
G_EXIT_CODE = APP_CODE_TERMINATION_NORMAL

###############################################################################
#                                 functions                                   #
###############################################################################

def get_files_list(dir):
    dirs  = []
    nfile = []
    files = []
    for dirname, dirnames, filenames in os.walk(dir):
        dirs.append(dirname)
        for subdirname in dirnames:
            dirs.append(os.path.join(dirname, subdirname))
        for filename in filenames:
            files.append(os.path.join(dirname, filename))
    return files

def get_test_files(dir):
    files = get_files_list(dir)
    nfile = filter(lambda x: x.endswith(test_expansion), files)
    return nfile

def get_executables(dir):
    # get rdo_studio_console executable and test_rdo_studio_console executable

    files = get_files_list(dir)
    rdo_ex = filter(lambda x: x.endswith(rdo_ex_substr), files)[0]
    rdo_test_ex = filter(lambda x: x.endswith(rdo_test_ex_substr), files)[0]

    return rdo_ex, rdo_test_ex

def get_text_from_node_list(nodelist):
    rc = []
    for node in nodelist:
        if node.nodeType == node.TEXT_NODE:
            rc.append(node.data)
    return ''.join(rc)

###############################################################################
#                                 main code                                   #
###############################################################################

print "STARTED SCRIPT :", sys.argv[0]

# search rdo and rdo_test executables
executables = get_executables(directory)

rdo_ex      = executables[0]
rdo_test_ex = executables[1]

if not os.path.exists(rdo_ex) or not os.path.exists(rdo_test_ex):
    print 'Build app not found. Critical error !!!'
    exit(EXIT_CODE_TERMINATION_ERROR)

# search .rtestx files
files = get_test_files(model_directory)
files.sort()

print '\nDEBUG INFO\n'

print 'Find RDO executables    : ', executables, '\n'
print 'Find test project files : ', files, '\n'

# parse xml and start tests
print 'STARTED TEST CYCLE'

for task in files:
    cycle_exit_code = EXIT_CODE_TERMINATION_NORMAL

    print dividing_line

    dom = xml.dom.minidom.parse(task)

    dirname = os.path.dirname(task) + '/'

    model_name_with_ex = get_text_from_node_list(dom.getElementsByTagName('model')[0].childNodes)

    target = get_text_from_node_list(dom.getElementsByTagName('target')[0].childNodes)
    exit_code = get_text_from_node_list(dom.getElementsByTagName('exit_code')[0].childNodes)
    etalon_trace_name = get_text_from_node_list(dom.getElementsByTagName('trace')[0].childNodes)
    etalon_result_name =  get_text_from_node_list(dom.getElementsByTagName('result')[0].childNodes)

    print 'Project     :', task
    print 'Model file  :', model_name_with_ex
    print 'Target      :', target
    print 'Exit code   :', exit_code
    print 'Trace file  :', etalon_trace_name
    print 'Result file :', etalon_result_name
    print ''

    model = dirname + model_name_with_ex
    etalon_trace = dirname + etalon_trace_name
    etalon_result =  dirname + etalon_result_name

    model_name = model_name_with_ex.partition('.')[0]

    simulation_trace = dirname + model_name + trace_expansion
    simulation_result = dirname + model_name + result_expansion

    if target == TARGET_CONSOLE:
        simulation_code = os.system(rdo_ex + ' -i ' + model
                                           + ' >> ' + null_file)

        print "SIMYLATION EXIT CODE :", simulation_code

        simulation_exit_code_string = ''
        if cmp(simulation_code, exit_code):
            simulation_exit_code_string = "OK"
        else:
            simulation_exit_code_string = "ERROR"

        print "CHECK SIM EXIT CODE  :", simulation_exit_code_string

        check_exit_code_string = ''
        if simulation_code == EXIT_CODE_TERMINATION_NORMAL:
            command = (rdo_test_ex + ' -T ' + etalon_trace
                                              + ' -R ' + etalon_result
                                              + ' -t ' + simulation_trace
                                              + ' -r ' + simulation_result
                                              + ' >> ' + null_file)
            #print command

            test_code = subprocess.call(command, shell=True)

            cycle_exit_code = APP_CODE_TERMINATION_ERROR

            if test_code == EXIT_CODE_TERMINATION_NORMAL:
                cycle_exit_code = APP_CODE_TERMINATION_NORMAL
                check_exit_code_string = 'OK'

            elif test_code == EXIT_CODE_TERMINATION_ERROR_FILE_NOT_FOUND:
                check_exit_code_string = 'ERROR FILE NOT FOUND'

            elif test_code == EXIT_CODE_TERMINATION_ERROR_INVALID_INPUT:
                check_exit_code_string = 'ERROR INVALID INPUT'

            elif test_code == EXIT_CODE_TERMINATION_ERROR_RESULT:
                check_exit_code_string = 'ERROR RESULT'

            elif test_code == EXIT_CODE_TERMINATION_ERROR_TRACE:
                check_exit_code_string = 'ERROR TRACE'

            elif test_code == EXIT_CODE_TERMINATION_ERROR_RESULT_AND_TRACE:
                check_exit_code_string = 'ERROR RESULT AND TRACE'

            else:
                check_exit_code_string = 'ERROR UNKNOWN'

        print "TEST EXIT CODE       :", test_code
        print "CHECK TEST CODE      :", check_exit_code_string

    else:
        print 'INVALID TARGET'

    # remove temp file
    #os.remove(simulation_trace)
    #os.remove(simulation_result)

    if not cycle_exit_code == EXIT_CODE_TERMINATION_NORMAL:
        G_EXIT_CODE = APP_CODE_TERMINATION_ERROR

os.remove(null_file)

print "\n", "PYTHON EXIT CODE :", G_EXIT_CODE, "\n"

sys.exit(G_EXIT_CODE)<|endoftext|>


<commit_before>##########################################################################################
## to use in binning pandakiller assembled samples into seqfilter parsed tag directories############################################################################################

import sys
from Bio import SeqIO

names = []
tags = open(sys.argv[1], 'rU')
for record in SeqIO.parse(tags, "fasta"):
	names.append(record.id.split(' ')[0])
#print names

ids = []
reads = []
assemb = open(sys.argv[2], 'rU')
for line in SeqIO.parse(assemb, "fastq"):
	lexemes = line.id.rsplit(':', 1)
	ids.append(lexemes[0])
	reads.append(line.seq)

dict_assemb = dict(zip(ids, reads))

output = open(sys.argv[3], 'w')
drop = open(sys.argv[4], 'w')
for item in names:
	if item in dict_assemb:
		output.write('>%s\n' % (item))
		output.write('%s\n' % (dict_assemb[item]))
	else:
		drop. write('%s\n' % (item))

<commit_msg>modified with os.walk<commit_after>##########################################################################################
## to use in binning pandakiller assembled samples into seqfilter parsed tag directories############################################################################################
import os
import sys
from Bio import SeqIO
import string

ids = []
reads = []
assemb = open(sys.argv[1], 'rU')
for line in SeqIO.parse(assemb, "fastq"):
        lexemes = line.id.rsplit(':', 1)
	ids.append(lexemes[0])
        reads.append(line.seq)
dict_assemb = dict(zip(ids, reads))

rootdir = "."
lfiles = []
for topdir, subdir, filelist in os.walk(rootdir):
	for files in filelist:
		if files.endswith("trimmed.fasta"):
			lfiles.append(files)

for each in lfiles:
	fout = '%s_assem.fasta' % each.strip()
        fp = open(fout, 'w')
	f = open(each, 'rU')
	l = []
	for records in SeqIO.parse(f, 'fasta'):
		name = records.id.split(' ')[0]
		if name in dict_assemb:
			#print each + ':' + name
			l.append(name)
#	print len(l)
	for item in l:
		fp.write('>%s\n' % item)
		fp.write('%s\n' % dict_assemb[item])
#	for files in filelist:
#		if files.endswith("trimmed.fasta"):
#			lfiles.append(files)
#			for each in lfiles:
#				outfiles_keep = open(each + '_' + 'assem' + '.fasta', 'w')
				#outfiles_drop = open(each + '_' + 'dropped' + 'reads' + '.txt', 'w')
#				f = open(each, 'rU')
#				names = []
#				for records in SeqIO.parse(f, 'fasta'):
#					names.append(records.id.split(' ')[0])
#				for item in names:
#					if item in dict_assemb:
#						outfiles_keep.write('>%s\n' % (item))
#						outfiles_keep.write('%s\n' % (dict_assemb[item]))
					#else:
					#	outfiles_drop. write('%s\n' % (item))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Endpoint to generate footer HTML."""

from __future__ import (
    absolute_import, division, print_function, unicode_literals)

import six
from django.conf import settings
from django.shortcuts import get_object_or_404
from django.template import loader as template_loader
from rest_framework import decorators, permissions
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from rest_framework_jsonp.renderers import JSONPRenderer

from readthedocs.builds.constants import LATEST, TAG
from readthedocs.builds.models import Version
from readthedocs.projects.models import Project
from readthedocs.projects.version_handling import (
    highest_version, parse_version_failsafe)
from readthedocs.restapi.signals import footer_response

def get_version_compare_data(project, base_version=None):
    """
    Retrieve metadata about the highest version available for this project.

    :param base_version: We assert whether or not the base_version is also the
                         highest version in the resulting "is_highest" value.
    """
    versions_qs = project.versions.public().filter(active=True)

    # Take preferences over tags only if the project has at least one tag
    if versions_qs.filter(type=TAG).exists():
        versions_qs = versions_qs.filter(type=TAG)

    highest_version_obj, highest_version_comparable = highest_version(
        versions_qs)
    ret_val = {
        'project': six.text_type(highest_version_obj),
        'version': six.text_type(highest_version_comparable),
        'is_highest': True,
    }
    if highest_version_obj:
        ret_val['url'] = highest_version_obj.get_absolute_url()
        ret_val['slug'] = highest_version_obj.slug
    if base_version and base_version.slug != LATEST:
        try:
            base_version_comparable = parse_version_failsafe(
                base_version.verbose_name)
            if base_version_comparable:
                # This is only place where is_highest can get set. All error
                # cases will be set to True, for non- standard versions.
                ret_val['is_highest'] = (
                    base_version_comparable >= highest_version_comparable)
            else:
                ret_val['is_highest'] = True
        except (Version.DoesNotExist, TypeError):
            ret_val['is_highest'] = True
    return ret_val

@decorators.api_view(['GET'])
@decorators.permission_classes((permissions.AllowAny,))
@decorators.renderer_classes((JSONRenderer, JSONPRenderer))
def footer_html(request):
    """Render and return footer markup."""
    # TODO refactor this function
    # pylint: disable=too-many-locals
    project_slug = request.GET.get('project', None)
    version_slug = request.GET.get('version', None)
    page_slug = request.GET.get('page', None)
    theme = request.GET.get('theme', False)
    docroot = request.GET.get('docroot', '')
    subproject = request.GET.get('subproject', False)
    source_suffix = request.GET.get('source_suffix', '.rst')

    new_theme = (theme == 'sphinx_rtd_theme')
    using_theme = (theme == 'default')
    project = get_object_or_404(Project, slug=project_slug)
    version = get_object_or_404(
        Version.objects.public(
            request.user, project=project, only_active=False),
        slug__iexact=version_slug)
    main_project = project.main_language_project or project

    if page_slug and page_slug != 'index':
        if (main_project.documentation_type == 'sphinx_htmldir' or
                main_project.documentation_type == 'mkdocs'):
            path = page_slug + '/'
        elif main_project.documentation_type == 'sphinx_singlehtml':
            path = 'index.html#document-' + page_slug
        else:
            path = page_slug + '.html'
    else:
        path = ''

    version_compare_data = get_version_compare_data(project, version)

    context = {
        'project': project,
        'version': version,
        'path': path,
        'downloads': version.get_downloads(pretty=True),
        'current_version': version.verbose_name,
        'versions': project.ordered_active_versions(user=request.user),
        'main_project': main_project,
        'translations': main_project.translations.all(),
        'current_language': project.language,
        'using_theme': using_theme,
        'new_theme': new_theme,
        'settings': settings,
        'subproject': subproject,
        'github_edit_url': version.get_github_url(
            docroot,
            page_slug,
            source_suffix,
            'edit',
        ),
        'github_view_url': version.get_github_url(
            docroot,
            page_slug,
            source_suffix,
            'view',
        ),
        'gitlab_edit_url': version.get_gitlab_url(
            docroot,
            page_slug,
            source_suffix,
            'edit',
        ),
        'gitlab_view_url': version.get_gitlab_url(
            docroot,
            page_slug,
            source_suffix,
            'view',
        ),
        'bitbucket_url': version.get_bitbucket_url(
            docroot,
            page_slug,
            source_suffix,
        ),
        'theme': theme,
    }

    html = template_loader.get_template('restapi/footer.html').render(
        context,
        request,
    )
    resp_data = {
        'html': html,
        'version_active': version.active,
        'version_compare': version_compare_data,
        'version_supported': version.supported,
    }

    # Allow folks to hook onto the footer response for various information
    # collection, or to modify the resp_data.
    footer_response.send(
        sender=None,
        request=request,
        context=context,
        resp_data=resp_data,
    )

    return Response(resp_data)
<commit_msg>I guess it really is supposed to be a tuple<commit_after># -*- coding: utf-8 -*-
"""Endpoint to generate footer HTML."""

from __future__ import (
    absolute_import, division, print_function, unicode_literals)

import six
from django.conf import settings
from django.shortcuts import get_object_or_404
from django.template import loader as template_loader
from rest_framework import decorators, permissions
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from rest_framework_jsonp.renderers import JSONPRenderer

from readthedocs.builds.constants import LATEST, TAG
from readthedocs.builds.models import Version
from readthedocs.projects.models import Project
from readthedocs.projects.version_handling import (
    highest_version, parse_version_failsafe)
from readthedocs.restapi.signals import footer_response

def get_version_compare_data(project, base_version=None):
    """
    Retrieve metadata about the highest version available for this project.

    :param base_version: We assert whether or not the base_version is also the
                         highest version in the resulting "is_highest" value.
    """
    versions_qs = project.versions.public().filter(active=True)

    # Take preferences over tags only if the project has at least one tag
    if versions_qs.filter(type=TAG).exists():
        versions_qs = versions_qs.filter(type=TAG)

    highest_version_obj, highest_version_comparable = highest_version(
        versions_qs)
    ret_val = {
        'project': six.text_type(highest_version_obj),
        'version': six.text_type(highest_version_comparable),
        'is_highest': True,
    }
    if highest_version_obj:
        ret_val['url'] = highest_version_obj.get_absolute_url()
        ret_val['slug'] = (highest_version_obj.slug,)
    if base_version and base_version.slug != LATEST:
        try:
            base_version_comparable = parse_version_failsafe(
                base_version.verbose_name)
            if base_version_comparable:
                # This is only place where is_highest can get set. All error
                # cases will be set to True, for non- standard versions.
                ret_val['is_highest'] = (
                    base_version_comparable >= highest_version_comparable)
            else:
                ret_val['is_highest'] = True
        except (Version.DoesNotExist, TypeError):
            ret_val['is_highest'] = True
    return ret_val

@decorators.api_view(['GET'])
@decorators.permission_classes((permissions.AllowAny,))
@decorators.renderer_classes((JSONRenderer, JSONPRenderer))
def footer_html(request):
    """Render and return footer markup."""
    # TODO refactor this function
    # pylint: disable=too-many-locals
    project_slug = request.GET.get('project', None)
    version_slug = request.GET.get('version', None)
    page_slug = request.GET.get('page', None)
    theme = request.GET.get('theme', False)
    docroot = request.GET.get('docroot', '')
    subproject = request.GET.get('subproject', False)
    source_suffix = request.GET.get('source_suffix', '.rst')

    new_theme = (theme == 'sphinx_rtd_theme')
    using_theme = (theme == 'default')
    project = get_object_or_404(Project, slug=project_slug)
    version = get_object_or_404(
        Version.objects.public(
            request.user, project=project, only_active=False),
        slug__iexact=version_slug)
    main_project = project.main_language_project or project

    if page_slug and page_slug != 'index':
        if (main_project.documentation_type == 'sphinx_htmldir' or
                main_project.documentation_type == 'mkdocs'):
            path = page_slug + '/'
        elif main_project.documentation_type == 'sphinx_singlehtml':
            path = 'index.html#document-' + page_slug
        else:
            path = page_slug + '.html'
    else:
        path = ''

    version_compare_data = get_version_compare_data(project, version)

    context = {
        'project': project,
        'version': version,
        'path': path,
        'downloads': version.get_downloads(pretty=True),
        'current_version': version.verbose_name,
        'versions': project.ordered_active_versions(user=request.user),
        'main_project': main_project,
        'translations': main_project.translations.all(),
        'current_language': project.language,
        'using_theme': using_theme,
        'new_theme': new_theme,
        'settings': settings,
        'subproject': subproject,
        'github_edit_url': version.get_github_url(
            docroot,
            page_slug,
            source_suffix,
            'edit',
        ),
        'github_view_url': version.get_github_url(
            docroot,
            page_slug,
            source_suffix,
            'view',
        ),
        'gitlab_edit_url': version.get_gitlab_url(
            docroot,
            page_slug,
            source_suffix,
            'edit',
        ),
        'gitlab_view_url': version.get_gitlab_url(
            docroot,
            page_slug,
            source_suffix,
            'view',
        ),
        'bitbucket_url': version.get_bitbucket_url(
            docroot,
            page_slug,
            source_suffix,
        ),
        'theme': theme,
    }

    html = template_loader.get_template('restapi/footer.html').render(
        context,
        request,
    )
    resp_data = {
        'html': html,
        'version_active': version.active,
        'version_compare': version_compare_data,
        'version_supported': version.supported,
    }

    # Allow folks to hook onto the footer response for various information
    # collection, or to modify the resp_data.
    footer_response.send(
        sender=None,
        request=request,
        context=context,
        resp_data=resp_data,
    )

    return Response(resp_data)
<|endoftext|>


<commit_before>#!/usr/bin/python
#License#
#bitHopper by Colin Rice is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License.
#Based on a work at github.com.

import json
import work
import diff
import stats
import pool
import speed
import database
import scheduler
import website
import getwork_store
import request_store
import data

import sys
import exceptions
import optparse
import time
import lp
import os.path
import os

from twisted.web import server, resource
from client import Agent
from _newclient import Request
from twisted.internet import reactor, defer
from twisted.internet.defer import Deferred
from twisted.internet.task import LoopingCall
from twisted.python import log, failure
from scheduler import Scheduler
import twisted.web.client

class BitHopper():
    def __init__(self):
        self.json_agent = twisted.web.client.Agent(reactor)
        self.lp_agent = Agent(reactor, persistent=True)
        self.new_server = Deferred()
        self.stats_file = None
        self.options = None
        self.reactor = reactor
        self.difficulty = diff.Difficulty(self)
        self.pool = pool.Pool(self)
        self.db = database.Database(self)
        self.lp = lp.LongPoll(self)
        self.speed = speed.Speed(self)
        self.stats = stats.Statistics(self)
        self.scheduler = scheduler.Scheduler(self)
        self.getwork_store = getwork_store.Getwork_store(self)
        self.request_store = request_store.Request_store(self)
        self.data = data.Data(self)
        self.pool.setup(self)

    def reject_callback(self,server,data):
        self.data.reject_callback(server,data)

    def data_callback(self,server,data, user, password):
        self.data.data_callback(server, data, user, password)

    def update_payout(self,server,payout):
        self.db.set_payout(server,float(payout))
        self.pool.servers[server]['payout'] = float(payout)

    def lp_callback(self, work):
        if work == None:
            return
        reactor.callLater(0.1,self.new_server.callback,work)
        self.new_server = Deferred()

    def get_json_agent(self):
        return self.json_agent

    def get_lp_agent(self):
        return self.lp_agent

    def get_options(self):
        return self.options

    def log_msg(self, msg, **kwargs):
        if kwargs and kwargs.get('cat'):
            print time.strftime("[%H:%M:%S] ") + '[' + kwargs.get('cat') + '] ' + str(msg)
        elif self.get_options() == None:
            print time.strftime("[%H:%M:%S] ") +str(msg)
            sys.stdout.flush()
        elif self.get_options().debug == True:
            log.msg(msg)
            sys.stdout.flush()
        else:
            print time.strftime("[%H:%M:%S] ") +str(msg)
            sys.stdout.flush()

    def log_dbg(self, msg, **kwargs):
        if self.get_options().debug == True and kwargs and kwargs.get('cat'):
            log.err('['+kwargs.get('cat')+"] "+msg)
            sys.stderr.flush()
        elif self.get_options() == None:
            log.err(msg)
            sys.stderr.flush()
        elif self.get_options().debug == True:
            log.err(msg)
            sys.stderr.flush()
        return

    def get_server(self, ):
        return self.pool.get_current()

    def select_best_server(self, ):
        server_name = None
        server_name = self.scheduler.select_best_server()
        if server_name == None:
            self.log_msg('FATAL Error, scheduler did not return any pool!')
            os._exit(-1)

        if self.pool.get_current() != server_name:
            self.pool.set_current(server_name)
            self.log_msg("Server change to " + str(self.pool.get_current()))

        return

    def get_new_server(self, server):
        self.pool.get_entry(server)['lag'] = True
        self.log_dbg('Lagging. :' + server)
        if server == self.pool.get_current():
            self.select_best_server()
        return self.pool.get_current()

    def server_update(self, ):
        if self.scheduler.server_update():
            self.select_best_server()

    @defer.inlineCallbacks
    def delag_server(self ):
        self.log_dbg('Running Delager')
        for server in self.pool.get_servers():
            info = self.pool.servers[server]
            if info['lag'] == True:
                data = yield work.jsonrpc_call(self.json_agent, server,[], self)
                self.log_dbg('Got' + server + ":" + str(data))
                if data != None:
                    info['lag'] = False
                    self.log_dbg('Delagging')
                else:
                    self.log_dbg('Not delagging')

    def bitHopper_Post(self,request):
        self.request_store.add(request)
        if not self.options.noLP:
            request.setHeader('X-Long-Polling', '/LP')
        rpc_request = json.loads(request.content.read())
        #check if they are sending a valid message
        if rpc_request['method'] != "getwork":
            return json.dumps({'result':None, 'error':'Not supported', 'id':rpc_request['id']})

        #Check for data to be validated
        current = self.pool.get_current()

        data = rpc_request['params']
        j_id = rpc_request['id']
        if data != []:
            new_server = self.getwork_store.get_server(data[0][72:136])
            if new_server != None:
                current = new_server

        work.jsonrpc_getwork(self.json_agent, current, data, j_id, request, self)

        if self.options.debug:
            self.log_msg('RPC request ' + str(data) + " submitted to " + current)
        else:
            if data == []:
                #If request contains no data, tell the user which remote procedure was called instead
                rep = rpc_request['method']
            else:
                rep = str(data[0][155:163])
            self.log_msg('RPC request [' + rep + "] submitted to " + current)

        if data != []:
            self.data_callback(current,data, request.getUser(), request.getPassword())
        return server.NOT_DONE_YET

    def bitHopperLP(self, value, *methodArgs):
        try:
            self.log_msg('LP triggered serving miner')
            request = methodArgs[0]
            #Duplicated from above because its a little less of a hack
            #But apparently people expect well formed json-rpc back but won't actually make the call
            try:
                json_request = request.content.read()
            except Exception,e:
                self.log_dbg( 'reading request content failed')
                json_request = None
            try:
                rpc_request = json.loads(json_request)
            except Exception, e:
                self.log_dbg('Loading the request failed')
                rpc_request = {'params':[],'id':1}

            j_id = rpc_request['id']

            response = json.dumps({"result":value,'error':None,'id':j_id})
            if self.request_store.closed(request):
                return value
            request.write(response)
            request.finish()

        except Exception, e:
            self.log_msg('Error Caught in bitHopperLP')
            self.log_dbg(str(e))
            try:
                request.finish()
            except Exception, e:
                self.log_dbg( "Client already disconnected Urgh.")

        return value

def parse_server_disable(option, opt, value, parser):
    setattr(parser.values, option.dest, value.split(','))

def select_scheduler(option, opt, value, parser):
    pass

bithopper_global = BitHopper()

def main():
    parser = optparse.OptionParser(description='bitHopper')
    parser.add_option('--noLP', action = 'store_true' ,default=False, help='turns off client side longpolling')
    parser.add_option('--debug', action= 'store_true', default = False, help='Use twisted output')
    parser.add_option('--listschedulers', action='store_true', default = False, help='List alternate schedulers available')
    parser.add_option('--list', action= 'store_true', default = False, help='List servers')
    parser.add_option('--disable', type=str, default = None, action='callback', callback=parse_server_disable, help='Servers to disable. Get name from --list. Servera,Serverb,Serverc')
    parser.add_option('--port', type = int, default=8337, help='Port to listen on')
    parser.add_option('--scheduler', type=str, default=None, help='Select an alternate scheduler')
    parser.add_option('--threshold', type=float, default=None, help='Override difficulty threshold (default 0.43)')
    parser.add_option('--altslicesize', type=int, default=900, help='Override Default AltSliceScheduler Slice Size of 900')
    parser.add_option('--altminslicesize', type=int, default=60, help='Override Default Minimum Pool Slice Size of 60 (AltSliceScheduler only)')
    parser.add_option('--altslicejitter', type=int, default=0, help='Add some random variance to slice size, disabled by default (AltSliceScheduler only)')
    parser.add_option('--startLP', action= 'store_true', default = True, help='Seeds the LP module with known pools. Must use it for LP based hopping with deepbit, True by default')
    parser.add_option('--ip', type = str, default='', help='IP to listen on')
    args, rest = parser.parse_args()
    options = args
    bithopper_global.options = args

    if options.list:
        for k in bithopper_global.pool.get_servers():
            print k
        return

    if options.listschedulers:
        schedulers = None
        for s in Scheduler.__subclasses__():
            if schedulers != None: schedulers = schedulers + ", " + s.__name__
            else: schedulers = s.__name__
        print "Available Schedulers: " + schedulers
        return

    if options.scheduler:
        bithopper_global.log_msg("Selecting scheduler: " + options.scheduler)
        foundScheduler = False
        for s in Scheduler.__subclasses__():
            if s.__name__ == options.scheduler:
                bithopper_global.scheduler = s(bithopper_global)
                foundScheduler = True
                break
        if foundScheduler == False:
            bithopper_global.log_msg("Error couldn't find: " + options.scheduler + ". Using default scheduler.")
            bithopper_global.scheduler = scheduler.DefaultScheduler(bithopper_global)
    else:
        bithopper_global.log_msg("Using default scheduler.")
        bithopper_global.scheduler = scheduler.DefaultScheduler(bithopper_global)

    bithopper_global.select_best_server()

    if options.disable != None:
        for k in options.disable:
            if k in bithopper_global.pool.get_servers():
                if bithopper_global.pool.get_servers()[k]['role'] == 'backup':
                    bithopper_global.log_msg("You just disabled the backup pool. I hope you know what you are doing")
                bithopper_global.pool.get_servers()[k]['role'] = 'disable'
            else:
                bithopper_global.log_msg(k + " Not a valid server")

    if options.debug: log.startLogging(sys.stdout)

    if options.startLP:
        bithopper_global.log_msg( 'Starting LP')
        startlp = LoopingCall(bithopper_global.lp.start_lp)
        startlp.start(60*30)

    site = server.Site(website.bitSite(bithopper_global))
    reactor.listenTCP(options.port, site,5, options.ip)
    reactor.callLater(0, bithopper_global.pool.update_api_servers, bithopper_global)
    delag_call = LoopingCall(bithopper_global.delag_server)
    delag_call.start(10)
    reactor.run()
    bithopper_global.db.close()

if __name__ == "__main__":
    main()
<commit_msg>Update agent to use timeout if possible<commit_after>#!/usr/bin/python
#License#
#bitHopper by Colin Rice is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License.
#Based on a work at github.com.

import json
import work
import diff
import stats
import pool
import speed
import database
import scheduler
import website
import getwork_store
import request_store
import data

import sys
import exceptions
import optparse
import time
import lp
import os.path
import os

from twisted.web import server, resource
from client import Agent
from _newclient import Request
from twisted.internet import reactor, defer
from twisted.internet.defer import Deferred
from twisted.internet.task import LoopingCall
from twisted.python import log, failure
from scheduler import Scheduler
import twisted.web.client

class BitHopper():
    def __init__(self):
        try:
            self.json_agent = twisted.web.client.Agent(reactor, connectTimeout=5)
        except:
            self.json_agent = twisted.web.client.Agent(reactor)
        self.lp_agent = Agent(reactor, persistent=True)
        self.new_server = Deferred()
        self.stats_file = None
        self.options = None
        self.reactor = reactor
        self.difficulty = diff.Difficulty(self)
        self.pool = pool.Pool(self)
        self.db = database.Database(self)
        self.lp = lp.LongPoll(self)
        self.speed = speed.Speed(self)
        self.stats = stats.Statistics(self)
        self.scheduler = scheduler.Scheduler(self)
        self.getwork_store = getwork_store.Getwork_store(self)
        self.request_store = request_store.Request_store(self)
        self.data = data.Data(self)
        self.pool.setup(self)

    def reject_callback(self,server,data):
        self.data.reject_callback(server,data)

    def data_callback(self,server,data, user, password):
        self.data.data_callback(server, data, user, password)

    def update_payout(self,server,payout):
        self.db.set_payout(server,float(payout))
        self.pool.servers[server]['payout'] = float(payout)

    def lp_callback(self, work):
        if work == None:
            return
        reactor.callLater(0.1,self.new_server.callback,work)
        self.new_server = Deferred()

    def get_json_agent(self):
        return self.json_agent

    def get_lp_agent(self):
        return self.lp_agent

    def get_options(self):
        return self.options

    def log_msg(self, msg, **kwargs):
        if kwargs and kwargs.get('cat'):
            print time.strftime("[%H:%M:%S] ") + '[' + kwargs.get('cat') + '] ' + str(msg)
        elif self.get_options() == None:
            print time.strftime("[%H:%M:%S] ") +str(msg)
            sys.stdout.flush()
        elif self.get_options().debug == True:
            log.msg(msg)
            sys.stdout.flush()
        else:
            print time.strftime("[%H:%M:%S] ") +str(msg)
            sys.stdout.flush()

    def log_dbg(self, msg, **kwargs):
        if self.get_options().debug == True and kwargs and kwargs.get('cat'):
            log.err('['+kwargs.get('cat')+"] "+msg)
            sys.stderr.flush()
        elif self.get_options() == None:
            log.err(msg)
            sys.stderr.flush()
        elif self.get_options().debug == True:
            log.err(msg)
            sys.stderr.flush()
        return

    def get_server(self, ):
        return self.pool.get_current()

    def select_best_server(self, ):
        server_name = None
        server_name = self.scheduler.select_best_server()
        if server_name == None:
            self.log_msg('FATAL Error, scheduler did not return any pool!')
            os._exit(-1)

        if self.pool.get_current() != server_name:
            self.pool.set_current(server_name)
            self.log_msg("Server change to " + str(self.pool.get_current()))

        return

    def get_new_server(self, server):
        self.pool.get_entry(server)['lag'] = True
        self.log_dbg('Lagging. :' + server)
        if server == self.pool.get_current():
            self.select_best_server()
        return self.pool.get_current()

    def server_update(self, ):
        if self.scheduler.server_update():
            self.select_best_server()

    @defer.inlineCallbacks
    def delag_server(self ):
        self.log_dbg('Running Delager')
        for server in self.pool.get_servers():
            info = self.pool.servers[server]
            if info['lag'] == True:
                data = yield work.jsonrpc_call(self.json_agent, server,[], self)
                self.log_dbg('Got' + server + ":" + str(data))
                if data != None:
                    info['lag'] = False
                    self.log_dbg('Delagging')
                else:
                    self.log_dbg('Not delagging')

    def bitHopper_Post(self,request):
        self.request_store.add(request)
        if not self.options.noLP:
            request.setHeader('X-Long-Polling', '/LP')
        rpc_request = json.loads(request.content.read())
        #check if they are sending a valid message
        if rpc_request['method'] != "getwork":
            return json.dumps({'result':None, 'error':'Not supported', 'id':rpc_request['id']})

        #Check for data to be validated
        current = self.pool.get_current()

        data = rpc_request['params']
        j_id = rpc_request['id']
        if data != []:
            new_server = self.getwork_store.get_server(data[0][72:136])
            if new_server != None:
                current = new_server

        work.jsonrpc_getwork(self.json_agent, current, data, j_id, request, self)

        if self.options.debug:
            self.log_msg('RPC request ' + str(data) + " submitted to " + current)
        else:
            if data == []:
                #If request contains no data, tell the user which remote procedure was called instead
                rep = rpc_request['method']
            else:
                rep = str(data[0][155:163])
            self.log_msg('RPC request [' + rep + "] submitted to " + current)

        if data != []:
            self.data_callback(current,data, request.getUser(), request.getPassword())
        return server.NOT_DONE_YET

    def bitHopperLP(self, value, *methodArgs):
        try:
            self.log_msg('LP triggered serving miner')
            request = methodArgs[0]
            #Duplicated from above because its a little less of a hack
            #But apparently people expect well formed json-rpc back but won't actually make the call
            try:
                json_request = request.content.read()
            except Exception,e:
                self.log_dbg( 'reading request content failed')
                json_request = None
            try:
                rpc_request = json.loads(json_request)
            except Exception, e:
                self.log_dbg('Loading the request failed')
                rpc_request = {'params':[],'id':1}

            j_id = rpc_request['id']

            response = json.dumps({"result":value,'error':None,'id':j_id})
            if self.request_store.closed(request):
                return value
            request.write(response)
            request.finish()

        except Exception, e:
            self.log_msg('Error Caught in bitHopperLP')
            self.log_dbg(str(e))
            try:
                request.finish()
            except Exception, e:
                self.log_dbg( "Client already disconnected Urgh.")

        return value

def parse_server_disable(option, opt, value, parser):
    setattr(parser.values, option.dest, value.split(','))

def select_scheduler(option, opt, value, parser):
    pass

bithopper_global = BitHopper()

def main():
    parser = optparse.OptionParser(description='bitHopper')
    parser.add_option('--noLP', action = 'store_true' ,default=False, help='turns off client side longpolling')
    parser.add_option('--debug', action= 'store_true', default = False, help='Use twisted output')
    parser.add_option('--listschedulers', action='store_true', default = False, help='List alternate schedulers available')
    parser.add_option('--list', action= 'store_true', default = False, help='List servers')
    parser.add_option('--disable', type=str, default = None, action='callback', callback=parse_server_disable, help='Servers to disable. Get name from --list. Servera,Serverb,Serverc')
    parser.add_option('--port', type = int, default=8337, help='Port to listen on')
    parser.add_option('--scheduler', type=str, default=None, help='Select an alternate scheduler')
    parser.add_option('--threshold', type=float, default=None, help='Override difficulty threshold (default 0.43)')
    parser.add_option('--altslicesize', type=int, default=900, help='Override Default AltSliceScheduler Slice Size of 900')
    parser.add_option('--altminslicesize', type=int, default=60, help='Override Default Minimum Pool Slice Size of 60 (AltSliceScheduler only)')
    parser.add_option('--altslicejitter', type=int, default=0, help='Add some random variance to slice size, disabled by default (AltSliceScheduler only)')
    parser.add_option('--startLP', action= 'store_true', default = True, help='Seeds the LP module with known pools. Must use it for LP based hopping with deepbit, True by default')
    parser.add_option('--ip', type = str, default='', help='IP to listen on')
    args, rest = parser.parse_args()
    options = args
    bithopper_global.options = args

    if options.list:
        for k in bithopper_global.pool.get_servers():
            print k
        return

    if options.listschedulers:
        schedulers = None
        for s in Scheduler.__subclasses__():
            if schedulers != None: schedulers = schedulers + ", " + s.__name__
            else: schedulers = s.__name__
        print "Available Schedulers: " + schedulers
        return

    if options.scheduler:
        bithopper_global.log_msg("Selecting scheduler: " + options.scheduler)
        foundScheduler = False
        for s in Scheduler.__subclasses__():
            if s.__name__ == options.scheduler:
                bithopper_global.scheduler = s(bithopper_global)
                foundScheduler = True
                break
        if foundScheduler == False:
            bithopper_global.log_msg("Error couldn't find: " + options.scheduler + ". Using default scheduler.")
            bithopper_global.scheduler = scheduler.DefaultScheduler(bithopper_global)
    else:
        bithopper_global.log_msg("Using default scheduler.")
        bithopper_global.scheduler = scheduler.DefaultScheduler(bithopper_global)

    bithopper_global.select_best_server()

    if options.disable != None:
        for k in options.disable:
            if k in bithopper_global.pool.get_servers():
                if bithopper_global.pool.get_servers()[k]['role'] == 'backup':
                    bithopper_global.log_msg("You just disabled the backup pool. I hope you know what you are doing")
                bithopper_global.pool.get_servers()[k]['role'] = 'disable'
            else:
                bithopper_global.log_msg(k + " Not a valid server")

    if options.debug: log.startLogging(sys.stdout)

    if options.startLP:
        bithopper_global.log_msg( 'Starting LP')
        startlp = LoopingCall(bithopper_global.lp.start_lp)
        startlp.start(60*30)

    site = server.Site(website.bitSite(bithopper_global))
    reactor.listenTCP(options.port, site,5, options.ip)
    reactor.callLater(0, bithopper_global.pool.update_api_servers, bithopper_global)
    delag_call = LoopingCall(bithopper_global.delag_server)
    delag_call.start(10)
    reactor.run()
    bithopper_global.db.close()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from __future__ import unicode_literals

__author__ = 'Julio Marquez'
__copyright__ = 'Copyright (c) 2016 Julio Marquez'
__license__ = 'Apache License 2.0'
__version__ = '0.2.0b'

default_app_config = 'knowledgebase.apps.DjangoKnowledgebaseConfig'
<commit_msg>Update version<commit_after>from __future__ import unicode_literals

__author__ = 'Julio Marquez'
__copyright__ = 'Copyright (c) 2016 Julio Marquez'
__license__ = 'Apache License 2.0'
__version__ = '0.2.0'

default_app_config = 'knowledgebase.apps.DjangoKnowledgebaseConfig'
<|endoftext|>


<commit_before>import httplib2
import json
import os
import re

from kong import openstack
from kong import tests

class TestImagesThroughCompute(tests.FunctionalTest):

    def setUp(self):
        super(TestImagesThroughCompute, self).setUp()
        self.os = openstack.Manager(self.nova)

    def _assert_image_links(self, image):
        image_id = str(image['id'])

        mgmt_url = self.os.nova.management_url
        mgmt_url = re.sub(r'1//', r'1/', mgmt_url) # TODO: is this a bug in Nova?
        bmk_url = re.sub(r'v1.1\/', r'', mgmt_url)
        self_link = {'rel': 'self',
                     'href': os.path.join(mgmt_url, 'images', image_id)}
        bookmark_link = {'rel': 'bookmark',
                         'href': os.path.join(bmk_url, 'images', image_id)}

        self.assertIn(bookmark_link, image['links'])
        self.assertIn(self_link, image['links'])

    def _assert_image_entity_basic(self, image):
        actual_keys = set(image.keys())
        expected_keys = set((
            'id',
            'name',
            'links',
        ))
        self.assertEqual(actual_keys, expected_keys)

        self._assert_image_links(image)

    def _assert_image_entity_detailed(self, image):
        keys = image.keys()
        if 'server' in keys:
            keys.remove('server')
        actual_keys = set(keys)
        expected_keys = set((
            'created',
            'id',
            'links',
            'metadata',
            'minDisk',
            'minRam',
            'name',
            'progress',
            'status',
            'updated',
        ))
        self.assertEqual(actual_keys, expected_keys)

        self._assert_image_links(image)

    def test_index(self):
        """List all images"""

        response, body = self.os.nova.request('GET', '/images')

        self.assertEqual(response['status'], '200')
        resp_body = json.loads(body)
        self.assertEqual(resp_body.keys(), ['images'])

        for image in resp_body['images']:
            self._assert_image_entity_basic(image)
    test_index.tags = ['nova', 'glance']

    def test_detail(self):
        """List all images in detail"""

        response, body = self.os.nova.request('GET', '/images/detail')
        self.assertEqual(response['status'], '200')
        resp_body = json.loads(body)
        self.assertEqual(resp_body.keys(), ['images'])

        for image in resp_body['images']:
            self._assert_image_entity_detailed(image)
    test_detail.tags = ['nova', 'glance']

class TestGlanceAPI(tests.FunctionalTest):

    def setUp(self):
        super(TestGlanceAPI, self).setUp()
        self.base_url = "http://%s:%s/%s/images" % (self.glance['host'],
		                                    self.glance['port'],
                                                    self.glance['apiver'])

    def test_upload_ami_style_image(self):
        """Uploads a three-part ami-style image"""
        aki_location = self.config['environment']['aki_location']
        headers = {'x-image-meta-is-public': 'true',
                   'x-image-meta-name': 'test-kernel',
                   'x-image-meta-disk-format': 'aki',
                   'x-image-meta-container-format': 'aki',
                   'Content-Length': '%d' % os.path.getsize(aki_location),
                   'Content-Type': 'application/octet-stream'}
        image_file = open(aki_location, "rb")
        http = httplib2.Http()
        response, content = http.request(self.base_url, 'POST',
                                         headers=headers,body=image_file)
        image_file.close()
        self.assertEqual(201, response.status)
        data = json.loads(content)
        self.assertEqual(data['image']['name'], "test-kernel")
        self.assertEqual(data['image']['checksum'],
                         self._md5sum_file(aki_location))
        kernel_id = data['image']['id']

        ari_location = self.config['environment'].get('ari_location')
        if ari_location:
            headers = {'x-image-meta-is-public': 'true',
                       'x-image-meta-name': 'test-ramdisk',
                       'x-image-meta-disk-format': 'ari',
                       'x-image-meta-container-format': 'ari',
                       'Content-Length': '%d' % os.path.getsize(ari_location),
                       'Content-Type': 'application/octet-stream'}
            image_file = open(ari_location, "rb")
            http = httplib2.Http()
            response, content = http.request(self.base_url, 'POST',
                                             headers=headers, body=image_file)
            image_file.close()
            self.assertEqual(201, response.status)
            data = json.loads(content)
            self.assertEqual(data['image']['name'], "test-ramdisk")
            self.assertEqual(data['image']['checksum'],
                             self._md5sum_file(ari_location))
            ramdisk_id = data['image']['id']
        else:
            ramdisk_id = None

        ami_location = self.config['environment']['ami_location']
        upload_data = ""
        for chunk in self._read_in_chunks(ami_location):
            upload_data += chunk
        headers = {'x-image-meta-is-public': 'true',
                   'x-image-meta-name': 'test-image',
                   'x-image-meta-disk-format': 'ami',
                   'x-image-meta-container-format': 'ami',
                   'x-image-meta-property-kernel_id': kernel_id,
                   'Content-Length': '%d' % os.path.getsize(ami_location),
                   'Content-Type': 'application/octet-stream'}

        if ari_location:
            headers['x-image-meta-property-ramdisk_id'] = ramdisk_id

        http = httplib2.Http()
        response, content = http.request(self.base_url, 'POST',
                                         headers=headers, body=upload_data)
        self.assertEqual(201, response.status)
        data = json.loads(content)
        self.assertEqual(data['image']['name'], "test-image")
        self.assertEqual(data['image']['checksum'],
                         self._md5sum_file(ami_location))
        machine_id = data['image']['id']

        # now ensure we can modify the image properties
        headers = {'X-Image-Meta-Property-distro': 'Ubuntu',
                   'X-Image-Meta-Property-arch': 'x86_64',
                   'X-Image-Meta-Property-kernel_id': kernel_id}
        if ari_location:
            headers['X-Image-Meta-Property-ramdisk_id'] = ramdisk_id

        http = httplib2.Http()
        url = '%s/%s' % (self.base_url, machine_id)
        response, content = http.request(url, 'PUT', headers=headers)
        self.assertEqual(response.status, 200)
        data = json.loads(content)
        properties = data['image']['properties']
        self.assertEqual(properties['arch'], "x86_64")
        self.assertEqual(properties['distro'], "Ubuntu")
        self.assertEqual(properties['kernel_id'], kernel_id)
        if ari_location:
            self.assertEqual(properties['ramdisk_id'], ramdisk_id)

        # list the metadata to ensure the new values stuck
        http = httplib2.Http()
        response, content = http.request(url, 'HEAD')
        self.assertEqual(response.status, 200)
        self.assertEqual(response['x-image-meta-name'], "test-image")
        self.assertEqual(response['x-image-meta-checksum'],
                         self._md5sum_file(ami_location))
        self.assertEqual(response['x-image-meta-container_format'], "ami")
        self.assertEqual(response['x-image-meta-disk_format'], "ami")
        self.assertEqual(response['x-image-meta-property-arch'], "x86_64")
        self.assertEqual(response['x-image-meta-property-distro'], "Ubuntu")
        self.assertEqual(response['x-image-meta-property-kernel_id'],
                         kernel_id)
        if ari_location:
            self.assertEqual(response['x-image-meta-property-ramdisk_id'],
                             ramdisk_id)

        # delete images for which we have non-None ids
        delete_ids = filter(lambda x: x, (kernel_id, ramdisk_id, machine_id))
        for image_id in delete_ids:
            http = httplib2.Http()
            url = '%s/%s' % (self.base_url, image_id)
            response, content = http.request(url, 'DELETE')

    test_upload_ami_style_image.tags = ['glance']
<commit_msg>Removing link doctoring in test_images<commit_after>import httplib2
import json
import os
import re

from kong import openstack
from kong import tests

class TestImagesThroughCompute(tests.FunctionalTest):

    def setUp(self):
        super(TestImagesThroughCompute, self).setUp()
        self.os = openstack.Manager(self.nova)

    def _assert_image_links(self, image):
        image_id = str(image['id'])

        mgmt_url = self.os.nova.management_url
        bmk_url = re.sub(r'v1.1\/', r'', mgmt_url)
        self_link = {'rel': 'self',
                     'href': os.path.join(mgmt_url, 'images', image_id)}
        bookmark_link = {'rel': 'bookmark',
                         'href': os.path.join(bmk_url, 'images', image_id)}

        self.assertIn(bookmark_link, image['links'])
        self.assertIn(self_link, image['links'])

    def _assert_image_entity_basic(self, image):
        actual_keys = set(image.keys())
        expected_keys = set((
            'id',
            'name',
            'links',
        ))
        self.assertEqual(actual_keys, expected_keys)

        self._assert_image_links(image)

    def _assert_image_entity_detailed(self, image):
        keys = image.keys()
        if 'server' in keys:
            keys.remove('server')
        actual_keys = set(keys)
        expected_keys = set((
            'created',
            'id',
            'links',
            'metadata',
            'minDisk',
            'minRam',
            'name',
            'progress',
            'status',
            'updated',
        ))
        self.assertEqual(actual_keys, expected_keys)

        self._assert_image_links(image)

    def test_index(self):
        """List all images"""

        response, body = self.os.nova.request('GET', '/images')

        self.assertEqual(response['status'], '200')
        resp_body = json.loads(body)
        self.assertEqual(resp_body.keys(), ['images'])

        for image in resp_body['images']:
            self._assert_image_entity_basic(image)
    test_index.tags = ['nova', 'glance']

    def test_detail(self):
        """List all images in detail"""

        response, body = self.os.nova.request('GET', '/images/detail')
        self.assertEqual(response['status'], '200')
        resp_body = json.loads(body)
        self.assertEqual(resp_body.keys(), ['images'])

        for image in resp_body['images']:
            self._assert_image_entity_detailed(image)
    test_detail.tags = ['nova', 'glance']

class TestGlanceAPI(tests.FunctionalTest):

    def setUp(self):
        super(TestGlanceAPI, self).setUp()
        self.base_url = "http://%s:%s/%s/images" % (self.glance['host'],
		                                    self.glance['port'],
                                                    self.glance['apiver'])

    def test_upload_ami_style_image(self):
        """Uploads a three-part ami-style image"""
        aki_location = self.config['environment']['aki_location']
        headers = {'x-image-meta-is-public': 'true',
                   'x-image-meta-name': 'test-kernel',
                   'x-image-meta-disk-format': 'aki',
                   'x-image-meta-container-format': 'aki',
                   'Content-Length': '%d' % os.path.getsize(aki_location),
                   'Content-Type': 'application/octet-stream'}
        image_file = open(aki_location, "rb")
        http = httplib2.Http()
        response, content = http.request(self.base_url, 'POST',
                                         headers=headers,body=image_file)
        image_file.close()
        self.assertEqual(201, response.status)
        data = json.loads(content)
        self.assertEqual(data['image']['name'], "test-kernel")
        self.assertEqual(data['image']['checksum'],
                         self._md5sum_file(aki_location))
        kernel_id = data['image']['id']

        ari_location = self.config['environment'].get('ari_location')
        if ari_location:
            headers = {'x-image-meta-is-public': 'true',
                       'x-image-meta-name': 'test-ramdisk',
                       'x-image-meta-disk-format': 'ari',
                       'x-image-meta-container-format': 'ari',
                       'Content-Length': '%d' % os.path.getsize(ari_location),
                       'Content-Type': 'application/octet-stream'}
            image_file = open(ari_location, "rb")
            http = httplib2.Http()
            response, content = http.request(self.base_url, 'POST',
                                             headers=headers, body=image_file)
            image_file.close()
            self.assertEqual(201, response.status)
            data = json.loads(content)
            self.assertEqual(data['image']['name'], "test-ramdisk")
            self.assertEqual(data['image']['checksum'],
                             self._md5sum_file(ari_location))
            ramdisk_id = data['image']['id']
        else:
            ramdisk_id = None

        ami_location = self.config['environment']['ami_location']
        upload_data = ""
        for chunk in self._read_in_chunks(ami_location):
            upload_data += chunk
        headers = {'x-image-meta-is-public': 'true',
                   'x-image-meta-name': 'test-image',
                   'x-image-meta-disk-format': 'ami',
                   'x-image-meta-container-format': 'ami',
                   'x-image-meta-property-kernel_id': kernel_id,
                   'Content-Length': '%d' % os.path.getsize(ami_location),
                   'Content-Type': 'application/octet-stream'}

        if ari_location:
            headers['x-image-meta-property-ramdisk_id'] = ramdisk_id

        http = httplib2.Http()
        response, content = http.request(self.base_url, 'POST',
                                         headers=headers, body=upload_data)
        self.assertEqual(201, response.status)
        data = json.loads(content)
        self.assertEqual(data['image']['name'], "test-image")
        self.assertEqual(data['image']['checksum'],
                         self._md5sum_file(ami_location))
        machine_id = data['image']['id']

        # now ensure we can modify the image properties
        headers = {'X-Image-Meta-Property-distro': 'Ubuntu',
                   'X-Image-Meta-Property-arch': 'x86_64',
                   'X-Image-Meta-Property-kernel_id': kernel_id}
        if ari_location:
            headers['X-Image-Meta-Property-ramdisk_id'] = ramdisk_id

        http = httplib2.Http()
        url = '%s/%s' % (self.base_url, machine_id)
        response, content = http.request(url, 'PUT', headers=headers)
        self.assertEqual(response.status, 200)
        data = json.loads(content)
        properties = data['image']['properties']
        self.assertEqual(properties['arch'], "x86_64")
        self.assertEqual(properties['distro'], "Ubuntu")
        self.assertEqual(properties['kernel_id'], kernel_id)
        if ari_location:
            self.assertEqual(properties['ramdisk_id'], ramdisk_id)

        # list the metadata to ensure the new values stuck
        http = httplib2.Http()
        response, content = http.request(url, 'HEAD')
        self.assertEqual(response.status, 200)
        self.assertEqual(response['x-image-meta-name'], "test-image")
        self.assertEqual(response['x-image-meta-checksum'],
                         self._md5sum_file(ami_location))
        self.assertEqual(response['x-image-meta-container_format'], "ami")
        self.assertEqual(response['x-image-meta-disk_format'], "ami")
        self.assertEqual(response['x-image-meta-property-arch'], "x86_64")
        self.assertEqual(response['x-image-meta-property-distro'], "Ubuntu")
        self.assertEqual(response['x-image-meta-property-kernel_id'],
                         kernel_id)
        if ari_location:
            self.assertEqual(response['x-image-meta-property-ramdisk_id'],
                             ramdisk_id)

        # delete images for which we have non-None ids
        delete_ids = filter(lambda x: x, (kernel_id, ramdisk_id, machine_id))
        for image_id in delete_ids:
            http = httplib2.Http()
            url = '%s/%s' % (self.base_url, image_id)
            response, content = http.request(url, 'DELETE')

    test_upload_ami_style_image.tags = ['glance']
<|endoftext|>


<commit_before># coding: utf-8
import re

from bson import ObjectId
from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext as _

from kpi.constants import NESTED_MONGO_RESERVED_ATTRIBUTES
from kpi.utils.strings import base64_encodestring

class MongoHelper:
    """
    Mongo's helper.

    Mix of KoBoCAT's onadata.apps.api.mongo_helper.MongoHelper
    and KoBoCAT's ParseInstance class to query mongo.
    """

    KEY_WHITELIST = ['$or', '$and', '$exists', '$in', '$gt', '$gte',
                     '$lt', '$lte', '$regex', '$options', '$all']

    ENCODING_SUBSTITUTIONS = [
        (re.compile(r'^\$'), base64_encodestring('$').strip()),
        (re.compile(r'\.'), base64_encodestring('.').strip()),
    ]

    DECODING_SUBSTITUTIONS = [
        (re.compile(r'^' + base64_encodestring('$').strip()), '$'),
        (re.compile(base64_encodestring('.').strip()), '.'),
    ]

    # Match KoBoCAT's variables of ParsedInstance class
    USERFORM_ID = '_userform_id'
    DEFAULT_BATCHSIZE = 1000

    @classmethod
    def decode(cls, key):
        """
        Replace base64-encoded characters not allowed in Mongo keys with their
        original representations

        :param key: string
        :return: string
        """
        for pattern, repl in cls.DECODING_SUBSTITUTIONS:
            key = re.sub(pattern, repl, key)
        return key

    @classmethod
    def encode(cls, key):
        """
        Replace characters not allowed in Mongo keys with their base64-encoded
        representations

        :param key: string
        :return: string
        """
        for pattern, repl in cls.ENCODING_SUBSTITUTIONS:
            key = re.sub(pattern, repl, key)
        return key

    @classmethod
    def get_count(
            cls, mongo_userform_id, hide_deleted=True, query=None, instance_ids=None,
            permission_filters=None):

        _, total_count = cls._get_cursor_and_count(
            mongo_userform_id,
            hide_deleted=hide_deleted,
            fields={'_id': 1},
            query=query,
            instance_ids=instance_ids,
            permission_filters=permission_filters)

        return total_count

    @classmethod
    def get_instances(
            cls, mongo_userform_id, hide_deleted=True, start=None, limit=None,
            sort=None, fields=None, query=None, instance_ids=None,
            permission_filters=None
    ):
        cursor, total_count = cls._get_cursor_and_count(
            mongo_userform_id,
            hide_deleted=hide_deleted,
            fields=fields,
            query=query,
            instance_ids=instance_ids,
            permission_filters=permission_filters)

        cursor.skip(start)
        if limit is not None:
            cursor.limit(limit)

        if len(sort) == 1:
            sort = MongoHelper.to_safe_dict(sort, reading=True)
            sort_key = list(sort.keys())[0]
            sort_dir = int(sort[sort_key])  # -1 for desc, 1 for asc
            cursor.sort(sort_key, sort_dir)

        # set batch size
        cursor.batch_size = cls.DEFAULT_BATCHSIZE

        return cursor, total_count

    @classmethod
    def is_attribute_invalid(cls, key):
        """
        Checks if an attribute can't be passed to Mongo as is.
        :param key:
        :return:
        """
        return key not in cls.KEY_WHITELIST and\
               (key.startswith('$') or key.count('.') > 0)

    @classmethod
    def to_readable_dict(cls, d):
        """
        Updates encoded attributes of a dict with human-readable attributes.
        For example:
        { "myLg==attribute": True } => { "my.attribute": True }

        :param d: dict
        :return: dict
        """

        for key, value in list(d.items()):
            if type(value) == list:
                value = [cls.to_readable_dict(e)
                         if type(e) == dict else e for e in value]
            elif type(value) == dict:
                value = cls.to_readable_dict(value)

            if cls._is_attribute_encoded(key):
                del d[key]
                d[cls.decode(key)] = value

        return d

    @classmethod
    def to_safe_dict(cls, d, reading=False):
        """
        Updates invalid attributes of a dict by encoding disallowed characters
        and, when `reading=False`, expanding dotted keys into nested dicts for
        `NESTED_MONGO_RESERVED_ATTRIBUTES`

        :param d: dict
        :param reading: boolean.
        :return: dict

        Example:

            >>> d = {
                    '_validation_status.other.nested': 'lorem',
                    '_validation_status.uid': 'approved',
                    'my.string.with.dots': 'yes'
                }
            >>> MongoHelper.to_safe_dict(d)
                {
                    'myLg==stringLg==withLg==dots': 'yes',
                    '_validation_status': {
                        'other': {
                            'nested': 'lorem'
                        },
                        'uid': 'approved'
                    }
                }
            >>> MongoHelper.to_safe_dict(d, reading=True)
                {
                    'myLg==stringLg==withLg==dots': 'yes',
                    '_validation_status.other.nested': 'lorem',
                    '_validation_status.uid': 'approved'
                }
        """
        for key, value in list(d.items()):
            if type(value) == list:
                value = [cls.to_safe_dict(e, reading=reading)
                         if type(e) == dict else e for e in value]
            elif type(value) == dict:
                value = cls.to_safe_dict(value, reading=reading)
            elif key == '_id':
                try:
                    d[key] = int(value)
                except ValueError:
                    # if it is not an int don't convert it
                    pass

            if cls._is_nested_reserved_attribute(key):
                # If we want to write into Mongo, we need to transform the dot delimited string into a dict
                # Otherwise, for reading, Mongo query engine reads dot delimited string as a nested object.
                # Drawback, if a user uses a reserved property with dots, it will be converted as well.
                if not reading and key.count(".") > 0:
                    tree = {}
                    t = tree
                    parts = key.split(".")
                    last_index = len(parts) - 1
                    for index, part in enumerate(parts):
                        v = value if index == last_index else {}
                        t = t.setdefault(part, v)
                    del d[key]
                    first_part = parts[0]
                    if first_part not in d:
                        d[first_part] = {}

                    # We update the main dict with new dict.
                    # We use dict_for_mongo again on the dict to ensure, no invalid characters are children
                    # elements
                    d[first_part].update(cls.to_safe_dict(tree[first_part]))

            elif cls.is_attribute_invalid(key):
                del d[key]
                d[cls.encode(key)] = value

        return d

    @classmethod
    def encode(cls, key):
        """
        Replace characters not allowed in Mongo keys with their base64-encoded
        representations

        :param key: string
        :return: string
        """
        for pattern, repl in cls.ENCODING_SUBSTITUTIONS:
            key = re.sub(pattern, repl, key)
        return key

    @classmethod
    def decode(cls, key):
        """
        Replace base64-encoded characters not allowed in Mongo keys with their
        original representations

        :param key: string
        :return: string
        """
        for pattern, repl in cls.DECODING_SUBSTITUTIONS:
            key = re.sub(pattern, repl, key)
        return key

    @classmethod
    def is_attribute_invalid(cls, key):
        """
        Checks if an attribute can't be passed to Mongo as is.
        :param key:
        :return:
        """
        return key not in \
               cls.KEY_WHITELIST and (key.startswith('$') or key.count('.') > 0)

    @classmethod
    def _get_cursor_and_count(cls, mongo_userform_id, hide_deleted=True,
                              fields=None, query=None, instance_ids=None,
                              permission_filters=None):

        if len(instance_ids) > 0:
            query.update({
                '_id': {'$in': instance_ids}
            })

        query.update({cls.USERFORM_ID: mongo_userform_id})

        # Narrow down query
        if permission_filters is not None:
            permission_filters_query = {'$or': []}
            for permission_filter in permission_filters:
                permission_filters_query['$or'].append(permission_filter)

            query = {'$and': [query, permission_filters_query]}

        if hide_deleted:
            # display only active elements
            deleted_at_query = {
                '$or': [{'_deleted_at': {'$exists': False}},
                        {'_deleted_at': None}]}
            # join existing query with deleted_at_query on an $and
            query = {'$and': [query, deleted_at_query]}

        query = cls.to_safe_dict(query, reading=True)

        if len(fields) > 0:
            # Retrieve only specified fields from Mongo. Remove
            # `cls.USERFORM_ID` from those fields in case users try to add it.
            if cls.USERFORM_ID in fields:
                fields.remove(cls.USERFORM_ID)
            fields_to_select = dict(
                [(cls.encode(field), 1) for field in fields])
        else:
            # Retrieve all fields except `cls.USERFORM_ID`
            fields_to_select = {cls.USERFORM_ID: 0}

        cursor = settings.MONGO_DB.instances.find(
            query,
            fields_to_select,
            max_time_ms=MONGO_DB_MAX_TIME_MS
        )
        return cursor, cursor.count()

    @classmethod
    def _is_attribute_encoded(cls, key):
        """
        Checks if an attribute has been encoded when saved in Mongo.

        :param key: string
        :return: string
        """
        return key not in cls.KEY_WHITELIST and (key.startswith('JA==') or
                                                 key.count('Lg==') > 0)

    @staticmethod
    def _is_nested_reserved_attribute(key):
        """
        Checks if key starts with one of variables values declared in NESTED_MONGO_RESERVED_ATTRIBUTES

        :param key: string
        :return: boolean
        """
        for reserved_attribute in NESTED_MONGO_RESERVED_ATTRIBUTES:
            if key.startswith("{}.".format(reserved_attribute)):
                return True
        return False
<commit_msg>edited variable path<commit_after># coding: utf-8
import re

from bson import ObjectId
from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext as _

from kpi.constants import NESTED_MONGO_RESERVED_ATTRIBUTES
from kpi.utils.strings import base64_encodestring

class MongoHelper:
    """
    Mongo's helper.

    Mix of KoBoCAT's onadata.apps.api.mongo_helper.MongoHelper
    and KoBoCAT's ParseInstance class to query mongo.
    """

    KEY_WHITELIST = ['$or', '$and', '$exists', '$in', '$gt', '$gte',
                     '$lt', '$lte', '$regex', '$options', '$all']

    ENCODING_SUBSTITUTIONS = [
        (re.compile(r'^\$'), base64_encodestring('$').strip()),
        (re.compile(r'\.'), base64_encodestring('.').strip()),
    ]

    DECODING_SUBSTITUTIONS = [
        (re.compile(r'^' + base64_encodestring('$').strip()), '$'),
        (re.compile(base64_encodestring('.').strip()), '.'),
    ]

    # Match KoBoCAT's variables of ParsedInstance class
    USERFORM_ID = '_userform_id'
    DEFAULT_BATCHSIZE = 1000

    @classmethod
    def decode(cls, key):
        """
        Replace base64-encoded characters not allowed in Mongo keys with their
        original representations

        :param key: string
        :return: string
        """
        for pattern, repl in cls.DECODING_SUBSTITUTIONS:
            key = re.sub(pattern, repl, key)
        return key

    @classmethod
    def encode(cls, key):
        """
        Replace characters not allowed in Mongo keys with their base64-encoded
        representations

        :param key: string
        :return: string
        """
        for pattern, repl in cls.ENCODING_SUBSTITUTIONS:
            key = re.sub(pattern, repl, key)
        return key

    @classmethod
    def get_count(
            cls, mongo_userform_id, hide_deleted=True, query=None, instance_ids=None,
            permission_filters=None):

        _, total_count = cls._get_cursor_and_count(
            mongo_userform_id,
            hide_deleted=hide_deleted,
            fields={'_id': 1},
            query=query,
            instance_ids=instance_ids,
            permission_filters=permission_filters)

        return total_count

    @classmethod
    def get_instances(
            cls, mongo_userform_id, hide_deleted=True, start=None, limit=None,
            sort=None, fields=None, query=None, instance_ids=None,
            permission_filters=None
    ):
        cursor, total_count = cls._get_cursor_and_count(
            mongo_userform_id,
            hide_deleted=hide_deleted,
            fields=fields,
            query=query,
            instance_ids=instance_ids,
            permission_filters=permission_filters)

        cursor.skip(start)
        if limit is not None:
            cursor.limit(limit)

        if len(sort) == 1:
            sort = MongoHelper.to_safe_dict(sort, reading=True)
            sort_key = list(sort.keys())[0]
            sort_dir = int(sort[sort_key])  # -1 for desc, 1 for asc
            cursor.sort(sort_key, sort_dir)

        # set batch size
        cursor.batch_size = cls.DEFAULT_BATCHSIZE

        return cursor, total_count

    @classmethod
    def is_attribute_invalid(cls, key):
        """
        Checks if an attribute can't be passed to Mongo as is.
        :param key:
        :return:
        """
        return key not in cls.KEY_WHITELIST and\
               (key.startswith('$') or key.count('.') > 0)

    @classmethod
    def to_readable_dict(cls, d):
        """
        Updates encoded attributes of a dict with human-readable attributes.
        For example:
        { "myLg==attribute": True } => { "my.attribute": True }

        :param d: dict
        :return: dict
        """

        for key, value in list(d.items()):
            if type(value) == list:
                value = [cls.to_readable_dict(e)
                         if type(e) == dict else e for e in value]
            elif type(value) == dict:
                value = cls.to_readable_dict(value)

            if cls._is_attribute_encoded(key):
                del d[key]
                d[cls.decode(key)] = value

        return d

    @classmethod
    def to_safe_dict(cls, d, reading=False):
        """
        Updates invalid attributes of a dict by encoding disallowed characters
        and, when `reading=False`, expanding dotted keys into nested dicts for
        `NESTED_MONGO_RESERVED_ATTRIBUTES`

        :param d: dict
        :param reading: boolean.
        :return: dict

        Example:

            >>> d = {
                    '_validation_status.other.nested': 'lorem',
                    '_validation_status.uid': 'approved',
                    'my.string.with.dots': 'yes'
                }
            >>> MongoHelper.to_safe_dict(d)
                {
                    'myLg==stringLg==withLg==dots': 'yes',
                    '_validation_status': {
                        'other': {
                            'nested': 'lorem'
                        },
                        'uid': 'approved'
                    }
                }
            >>> MongoHelper.to_safe_dict(d, reading=True)
                {
                    'myLg==stringLg==withLg==dots': 'yes',
                    '_validation_status.other.nested': 'lorem',
                    '_validation_status.uid': 'approved'
                }
        """
        for key, value in list(d.items()):
            if type(value) == list:
                value = [cls.to_safe_dict(e, reading=reading)
                         if type(e) == dict else e for e in value]
            elif type(value) == dict:
                value = cls.to_safe_dict(value, reading=reading)
            elif key == '_id':
                try:
                    d[key] = int(value)
                except ValueError:
                    # if it is not an int don't convert it
                    pass

            if cls._is_nested_reserved_attribute(key):
                # If we want to write into Mongo, we need to transform the dot delimited string into a dict
                # Otherwise, for reading, Mongo query engine reads dot delimited string as a nested object.
                # Drawback, if a user uses a reserved property with dots, it will be converted as well.
                if not reading and key.count(".") > 0:
                    tree = {}
                    t = tree
                    parts = key.split(".")
                    last_index = len(parts) - 1
                    for index, part in enumerate(parts):
                        v = value if index == last_index else {}
                        t = t.setdefault(part, v)
                    del d[key]
                    first_part = parts[0]
                    if first_part not in d:
                        d[first_part] = {}

                    # We update the main dict with new dict.
                    # We use dict_for_mongo again on the dict to ensure, no invalid characters are children
                    # elements
                    d[first_part].update(cls.to_safe_dict(tree[first_part]))

            elif cls.is_attribute_invalid(key):
                del d[key]
                d[cls.encode(key)] = value

        return d

    @classmethod
    def encode(cls, key):
        """
        Replace characters not allowed in Mongo keys with their base64-encoded
        representations

        :param key: string
        :return: string
        """
        for pattern, repl in cls.ENCODING_SUBSTITUTIONS:
            key = re.sub(pattern, repl, key)
        return key

    @classmethod
    def decode(cls, key):
        """
        Replace base64-encoded characters not allowed in Mongo keys with their
        original representations

        :param key: string
        :return: string
        """
        for pattern, repl in cls.DECODING_SUBSTITUTIONS:
            key = re.sub(pattern, repl, key)
        return key

    @classmethod
    def is_attribute_invalid(cls, key):
        """
        Checks if an attribute can't be passed to Mongo as is.
        :param key:
        :return:
        """
        return key not in \
               cls.KEY_WHITELIST and (key.startswith('$') or key.count('.') > 0)

    @classmethod
    def _get_cursor_and_count(cls, mongo_userform_id, hide_deleted=True,
                              fields=None, query=None, instance_ids=None,
                              permission_filters=None):

        if len(instance_ids) > 0:
            query.update({
                '_id': {'$in': instance_ids}
            })

        query.update({cls.USERFORM_ID: mongo_userform_id})

        # Narrow down query
        if permission_filters is not None:
            permission_filters_query = {'$or': []}
            for permission_filter in permission_filters:
                permission_filters_query['$or'].append(permission_filter)

            query = {'$and': [query, permission_filters_query]}

        if hide_deleted:
            # display only active elements
            deleted_at_query = {
                '$or': [{'_deleted_at': {'$exists': False}},
                        {'_deleted_at': None}]}
            # join existing query with deleted_at_query on an $and
            query = {'$and': [query, deleted_at_query]}

        query = cls.to_safe_dict(query, reading=True)

        if len(fields) > 0:
            # Retrieve only specified fields from Mongo. Remove
            # `cls.USERFORM_ID` from those fields in case users try to add it.
            if cls.USERFORM_ID in fields:
                fields.remove(cls.USERFORM_ID)
            fields_to_select = dict(
                [(cls.encode(field), 1) for field in fields])
        else:
            # Retrieve all fields except `cls.USERFORM_ID`
            fields_to_select = {cls.USERFORM_ID: 0}

        cursor = settings.MONGO_DB.instances.find(
            query,
            fields_to_select,
            max_time_ms=settings.MONGO_DB_MAX_TIME_MS
        )
        return cursor, cursor.count()

    @classmethod
    def _is_attribute_encoded(cls, key):
        """
        Checks if an attribute has been encoded when saved in Mongo.

        :param key: string
        :return: string
        """
        return key not in cls.KEY_WHITELIST and (key.startswith('JA==') or
                                                 key.count('Lg==') > 0)

    @staticmethod
    def _is_nested_reserved_attribute(key):
        """
        Checks if key starts with one of variables values declared in NESTED_MONGO_RESERVED_ATTRIBUTES

        :param key: string
        :return: boolean
        """
        for reserved_attribute in NESTED_MONGO_RESERVED_ATTRIBUTES:
            if key.startswith("{}.".format(reserved_attribute)):
                return True
        return False
<|endoftext|>