<commit_before>from functools import reduce
from operator import add
import re
from typing import Any, List, Dict, Set, Tuple

from werkzeug.datastructures import ImmutableOrderedMultiDict

EMAIL_REGEX = r'^[^@^\s]+@[^@^\.^\s]+(\.[^@^\.^\s]+)+$'

def get_validator(framework, content, answers):
    """
    Retrieves a validator by slug contained in the framework dictionary.
    """
    if framework is None:
        raise ValueError("a framework dictionary must be provided")
    if framework is not None:
        validator_cls = VALIDATORS.get(framework['slug'], SharedValidator)
        return validator_cls(content, answers)

class DeclarationValidator(object):
    email_validation_fields = []
    number_string_fields = []
    character_limit = None
    word_limit = None
    optional_fields = set([])

    def __init__(self, content, answers):
        self.content = content
        self.answers = answers

    def get_error_messages_for_page(self, section) -> ImmutableOrderedMultiDict:
        all_errors = self.get_error_messages()
        page_ids = section.get_question_ids()
        page_errors = ImmutableOrderedMultiDict(filter(lambda err: err[0] in page_ids, all_errors))
        return page_errors

    def get_error_messages(self) -> List[Tuple[str, dict]]:
        raw_errors_map = self.errors()
        errors_map = list()
        for question_id in self.all_fields():
            if question_id in raw_errors_map:
                question_number = self.content.get_question(question_id).get('number')
                validation_message = self.get_error_message(question_id, raw_errors_map[question_id])
                errors_map.append((question_id, {
                    'input_name': question_id,
                    'question': "Question {}".format(question_number)
                    if question_number else self.content.get_question(question_id).get('question'),
                    'message': validation_message,
                }))

        return errors_map

    def get_error_message(self, question_id: str, message_key: str) -> str:
        for validation in self.content.get_question(question_id).get('validations', []):
            if validation['name'] == message_key:
                return validation['message']
        default_messages = {
            'answer_required': 'You need to answer this question.',
            'under_character_limit': 'Your answer must be no more than {} characters.'.format(self.character_limit),
            'invalid_format': 'You must enter a valid email address.',
        }
        return default_messages.get(
            message_key, 'There was a problem with the answer to this question')

    def all_fields(self) -> List[str]:
        return reduce(add, (section.get_question_ids() for section in self.content))

    def fields_with_values(self) -> Set[str]:
        return set(key for key, value in self.answers.items()
                   if value is not None and (not isinstance(value, str) or len(value) > 0))

    def errors(self) -> Dict[str, str]:
        errors_map = {}
        errors_map.update(self.character_limit_errors())
        errors_map.update(self.word_limit_errors())
        errors_map.update(self.formatting_errors(self.answers))
        errors_map.update(self.answer_required_errors())
        return errors_map

    def answer_required_errors(self) -> Dict[str, str]:
        req_fields = self.get_required_fields()
        filled_fields = self.fields_with_values()
        errors_map = {}

        for field in req_fields - filled_fields:
            errors_map[field] = 'answer_required'

        return errors_map

    def character_limit_errors(self) -> Dict[str, str]:
        errors_map = {}
        for question_id in self.all_fields():
            if self.content.get_question(question_id).get('type') in ['text', 'textbox_large']:
                answer = self.answers.get(question_id) or ''
                if self.character_limit is not None and len(answer) > self.character_limit:
                    errors_map[question_id] = "under_character_limit"

        return errors_map

    def word_limit_errors(self) -> Dict[str, str]:
        errors_map = {}
        for question_id in self.all_fields():
            question = self.content.get_question(question_id)
            if question.get('type') in ['text', 'textbox_large']:
                # Get word limit from question content, fall back to class attribute
                word_limit = question.get('max_length_in_words', self.word_limit)
                answer = self.answers.get(question_id) or ''
                if word_limit is not None and len(answer.split()) > word_limit:
                    errors_map[question_id] = "under_word_limit"

        return errors_map

    def formatting_errors(self, answers) -> Dict[str, str]:
        errors_map = {}
        if self.email_validation_fields is not None and len(self.email_validation_fields) > 0:
            for field in self.email_validation_fields:
                if self.answers.get(field) is None or not re.match(EMAIL_REGEX, self.answers.get(field, '')):
                    errors_map[field] = 'invalid_format'

        if self.number_string_fields is not None and len(self.number_string_fields) > 0:
            for field, length in self.number_string_fields:
                if self.answers.get(field) is None or not re.match(
                        r'^\d{{{0}}}$'.format(length), self.answers.get(field, '')
                ):
                    errors_map[field] = 'invalid_format'
        return errors_map

    def get_required_fields(self) -> Set[str]:
        try:
            req_fields = self.required_fields
        except AttributeError:
            req_fields = set(self.all_fields())

        #  Remove optional fields
        if self.optional_fields is not None:
            req_fields -= set(self.optional_fields)

        return req_fields

class G7Validator(DeclarationValidator):
    """
    Validator for G-Cloud 7.
    """
    optional_fields = {
        "SQ1-1p-i", "SQ1-1p-ii", "SQ1-1p-iii", "SQ1-1p-iv",
        "SQ1-1q-i", "SQ1-1q-ii", "SQ1-1q-iii", "SQ1-1q-iv", "SQ1-1cii", "SQ1-1i-ii",
        "SQ1-1j-i", "SQ1-1j-ii", "SQ4-1c", "SQ3-1k", "SQ1-1i-i"
    }
    email_validation_fields = {'SQ1-1o', 'SQ1-2b'}
    character_limit = 5000

    def get_required_fields(self) -> Set[str]:
        req_fields = super(G7Validator, self).get_required_fields()

        #  If you answered other to question 19 (trading status)
        if self.answers.get('SQ1-1ci') == 'other (please specify)':
            req_fields.add('SQ1-1cii')

        #  If you answered yes to question 27 (non-UK business registered in EU)
        if self.answers.get('SQ1-1i-i', False):
            req_fields.add('SQ1-1i-ii')

        #  If you answered 'licensed' or 'a member of a relevant organisation' in question 29
        answer_29 = self.answers.get('SQ1-1j-i', [])
        if answer_29 and len(answer_29) > 0 and (
            'licensed' in answer_29
            or 'a member of a relevant organisation' in answer_29
        ):
            req_fields.add('SQ1-1j-ii')

        # If you answered yes to either question 53 or 54 (tax returns)
        if self.answers.get('SQ4-1a', False) or self.answers.get('SQ4-1b', False):
            req_fields.add('SQ4-1c')

        # If you answered Yes to questions 39 - 51 (discretionary exclusion)
        dependent_fields = [
            'SQ2-2a', 'SQ3-1a', 'SQ3-1b', 'SQ3-1c', 'SQ3-1d', 'SQ3-1e', 'SQ3-1f', 'SQ3-1g',
            'SQ3-1h-i', 'SQ3-1h-ii', 'SQ3-1i-i', 'SQ3-1i-ii', 'SQ3-1j'
        ]
        if any(self.answers.get(field) for field in dependent_fields):
            req_fields.add('SQ3-1k')

        # If you answered No to question 26 (established in the UK)
        if 'SQ5-2a' in self.answers and not self.answers['SQ5-2a']:
            req_fields.add('SQ1-1i-i')
            req_fields.add('SQ1-1j-i')

        return req_fields

class DOSValidator(DeclarationValidator):
    optional_fields = {
        "mitigatingFactors", "mitigatingFactors2", "mitigatingFactors3", "tradingStatusOther",
        "modernSlaveryStatement", "modernSlaveryStatementOptional", "modernSlaveryReportingRequirements",
        # Registered in UK = no
        "appropriateTradeRegisters", "appropriateTradeRegistersNumber",
        "licenceOrMemberRequired", "licenceOrMemberRequiredDetails",
    }

    dependent_fields = {
        # If you responded yes to any of questions 22 to 34
        "mitigatingFactors": [
            'misleadingInformation', 'confidentialInformation', 'influencedContractingAuthority',
            'witheldSupportingDocuments', 'seriousMisrepresentation', 'significantOrPersistentDeficiencies',
            'distortedCompetition', 'conflictOfInterest', 'distortingCompetition', 'graveProfessionalMisconduct',
            'bankrupt', 'environmentalSocialLabourLaw', 'taxEvasion'
        ],
        # If you responded yes to either 36 or 37
        "mitigatingFactors2": [
            "unspentTaxConvictions", "GAAR"
        ],
        # If you responded yes to 50 (Modern Slavery)
        "modernSlaveryStatement": [
            "modernSlaveryTurnover",
        ],
        "modernSlaveryReportingRequirements": [
            "modernSlaveryTurnover"
        ],
    }

    email_validation_fields = {"contactEmailContractNotice", "primaryContactEmail"}
    character_limit = 5000

    def get_required_fields(self) -> Set[str]:
        req_fields = super(DOSValidator, self).get_required_fields()

        for target_field, fields in self.dependent_fields.items():
            if any(self.answers.get(field) for field in fields):
                req_fields.add(target_field)

        # Describe your trading status
        if self.answers.get('tradingStatus') == "other (please specify)":
            req_fields.add('tradingStatusOther')

        # If your company was not established in the UK
        if self.answers.get('establishedInTheUK') is False:
            req_fields.add('appropriateTradeRegisters')
            # If yes to appropriate trade registers
            if self.answers.get('appropriateTradeRegisters') is True:
                req_fields.add('appropriateTradeRegistersNumber')

            req_fields.add('licenceOrMemberRequired')
            # If not 'none of the above' to licenceOrMemberRequired
            if self.answers.get('licenceOrMemberRequired') in ['licensed', 'a member of a relevant organisation']:
                req_fields.add('licenceOrMemberRequiredDetails')

        # If supplier doesn't meet the Modern Slavery reporting requirements, they don't need to upload a statement
        # but must have mitigatingFactors3 explanation
        if self.answers.get('modernSlaveryReportingRequirements') is False:
            req_fields.add('mitigatingFactors3')
            req_fields.remove('modernSlaveryStatement')

        return req_fields

class SharedValidator(DOSValidator):
    # From DOS2 and G8 onwards, validate DUNS number length
    number_string_fields = [('dunsNumber', 9)]
    word_limit = 500

class G12Validator(SharedValidator):

    def errors(self) -> Dict[str, str]:

        errors_map = super().errors()

        q1_answer = self.answers.get('servicesHaveOrSupportCloudHostingCloudSoftware')
        q2_answer = self.answers.get('servicesHaveOrSupportCloudSupport')

        q1_negative = "My organisation isn't submitting cloud hosting (lot 1) or cloud software (lot 2) services"
        q2_negative = "My organisation isn't submitting cloud support (lot 3) services"

        if q1_answer == q1_negative and q2_answer == q2_negative:
            errors_map.update({
                'servicesHaveOrSupportCloudHostingCloudSoftware': 'dependent_question_error',
                'servicesHaveOrSupportCloudSupport': 'dependent_question_error',
            })

        return errors_map

def is_valid_percentage(value: Any) -> bool:
    # Design System guidance is that we should allow users to provide answers with or without units.
    if isinstance(value, str):
        value = value.rstrip('%')

    try:
        number = float(value)
    except ValueError:
        return False
    else:
        return 0 <= number <= 100

class DOS5Validator(SharedValidator):
    """Following an accessibility review, a number of questions and answers were changed for DOS 5"""
    percentage_fields = ["subcontractingInvoicesPaid"]

    optional_fields = SharedValidator.optional_fields.union({
        "subcontracting30DayPayments",
        "subcontractingInvoicesPaid"}
    )

    def get_required_fields(self) -> Set[str]:
        req_fields = super(DOS5Validator, self).get_required_fields()

        # as per subcontracting configuration on digitalmarketplace-frameworks
        if self.answers.get("subcontracting") in [
            "as a prime contractor, using third parties (subcontractors) to provide some services",
            "as part of a consortium or special purpose vehicle, using third parties (subcontractors) to provide some "
            "services"
        ]:
            req_fields.add("subcontracting30DayPayments")
            req_fields.add("subcontractingInvoicesPaid")

        return req_fields

    def formatting_errors(self, answers) -> Dict[str, str]:
        error_map = super(DOS5Validator, self).formatting_errors(answers)

        for field in self.percentage_fields:
            value = self.answers.get(field)
            if value is not None and not is_valid_percentage(value):
                error_map[field] = 'not_a_number'

        return error_map

VALIDATORS = {
    "g-cloud-7": G7Validator,
    "g-cloud-8": SharedValidator,
    "digital-outcomes-and-specialists": DOSValidator,
    "digital-outcomes-and-specialists-2": SharedValidator,
    "g-cloud-9": SharedValidator,
    "g-cloud-10": SharedValidator,
    "digital-outcomes-and-specialists-3": SharedValidator,
    "g-cloud-11": SharedValidator,
    "digital-outcomes-and-specialists-4": SharedValidator,
    "g-cloud-12": G12Validator,
    "digital-outcomes-and-specialists-5": DOS5Validator,
}
<commit_msg>Fix or ignore validation type errors<commit_after>from functools import reduce
from operator import add
import re
from typing import Any, List, Dict, Set, Tuple, Optional

from werkzeug.datastructures import ImmutableOrderedMultiDict

EMAIL_REGEX = r'^[^@^\s]+@[^@^\.^\s]+(\.[^@^\.^\s]+)+$'

def get_validator(framework, content, answers):
    """
    Retrieves a validator by slug contained in the framework dictionary.
    """
    if framework is None:
        raise ValueError("a framework dictionary must be provided")
    if framework is not None:
        validator_cls = VALIDATORS.get(framework['slug'], SharedValidator)
        return validator_cls(content, answers)

class DeclarationValidator(object):
    email_validation_fields: Set[str] = set()
    number_string_fields: List[Tuple[str, int]] = []
    character_limit: Optional[int] = None
    word_limit: Optional[int] = None
    optional_fields: Set[str] = set()

    def __init__(self, content, answers):
        self.content = content
        self.answers = answers

    def get_error_messages_for_page(self, section) -> ImmutableOrderedMultiDict:
        all_errors = self.get_error_messages()
        page_ids = section.get_question_ids()
        return ImmutableOrderedMultiDict(filter(lambda err: err[0] in page_ids, all_errors))

    def get_error_messages(self) -> List[Tuple[str, dict]]:
        raw_errors_map = self.errors()
        errors_map = list()
        for question_id in self.all_fields():
            if question_id in raw_errors_map:
                question_number = self.content.get_question(question_id).get('number')
                validation_message = self.get_error_message(question_id, raw_errors_map[question_id])
                errors_map.append((question_id, {
                    'input_name': question_id,
                    'question': "Question {}".format(question_number)
                    if question_number else self.content.get_question(question_id).get('question'),
                    'message': validation_message,
                }))

        return errors_map

    def get_error_message(self, question_id: str, message_key: str) -> str:
        for validation in self.content.get_question(question_id).get('validations', []):
            if validation['name'] == message_key:
                return validation['message']  # type: ignore
        default_messages = {
            'answer_required': 'You need to answer this question.',
            'under_character_limit': 'Your answer must be no more than {} characters.'.format(self.character_limit),
            'invalid_format': 'You must enter a valid email address.',
        }
        return default_messages.get(
            message_key, 'There was a problem with the answer to this question')

    def all_fields(self) -> List[str]:
        return reduce(add, (section.get_question_ids() for section in self.content))

    def fields_with_values(self) -> Set[str]:
        return set(key for key, value in self.answers.items()
                   if value is not None and (not isinstance(value, str) or len(value) > 0))

    def errors(self) -> Dict[str, str]:
        errors_map = {}
        errors_map.update(self.character_limit_errors())
        errors_map.update(self.word_limit_errors())
        errors_map.update(self.formatting_errors(self.answers))
        errors_map.update(self.answer_required_errors())
        return errors_map

    def answer_required_errors(self) -> Dict[str, str]:
        req_fields = self.get_required_fields()
        filled_fields = self.fields_with_values()
        errors_map = {}

        for field in req_fields - filled_fields:
            errors_map[field] = 'answer_required'

        return errors_map

    def character_limit_errors(self) -> Dict[str, str]:
        errors_map = {}
        for question_id in self.all_fields():
            if self.content.get_question(question_id).get('type') in ['text', 'textbox_large']:
                answer = self.answers.get(question_id) or ''
                if self.character_limit is not None and len(answer) > self.character_limit:
                    errors_map[question_id] = "under_character_limit"

        return errors_map

    def word_limit_errors(self) -> Dict[str, str]:
        errors_map = {}
        for question_id in self.all_fields():
            question = self.content.get_question(question_id)
            if question.get('type') in ['text', 'textbox_large']:
                # Get word limit from question content, fall back to class attribute
                word_limit = question.get('max_length_in_words', self.word_limit)
                answer = self.answers.get(question_id) or ''
                if word_limit is not None and len(answer.split()) > word_limit:
                    errors_map[question_id] = "under_word_limit"

        return errors_map

    def formatting_errors(self, answers) -> Dict[str, str]:
        errors_map = {}
        if self.email_validation_fields is not None and len(self.email_validation_fields) > 0:
            for field in self.email_validation_fields:
                if self.answers.get(field) is None or not re.match(EMAIL_REGEX, self.answers.get(field, '')):
                    errors_map[field] = 'invalid_format'

        if self.number_string_fields is not None and len(self.number_string_fields) > 0:
            for field, length in self.number_string_fields:
                if self.answers.get(field) is None or not re.match(
                        r'^\d{{{0}}}$'.format(length), self.answers.get(field, '')
                ):
                    errors_map[field] = 'invalid_format'
        return errors_map

    def get_required_fields(self) -> Set[str]:
        try:
            req_fields = self.required_fields  # type: ignore
        except AttributeError:
            req_fields = set(self.all_fields())

        #  Remove optional fields
        if self.optional_fields is not None:
            req_fields -= set(self.optional_fields)

        return req_fields  # type: ignore

class G7Validator(DeclarationValidator):
    """
    Validator for G-Cloud 7.
    """
    optional_fields = {
        "SQ1-1p-i", "SQ1-1p-ii", "SQ1-1p-iii", "SQ1-1p-iv",
        "SQ1-1q-i", "SQ1-1q-ii", "SQ1-1q-iii", "SQ1-1q-iv", "SQ1-1cii", "SQ1-1i-ii",
        "SQ1-1j-i", "SQ1-1j-ii", "SQ4-1c", "SQ3-1k", "SQ1-1i-i"
    }
    email_validation_fields = {'SQ1-1o', 'SQ1-2b'}
    character_limit = 5000

    def get_required_fields(self) -> Set[str]:
        req_fields = super(G7Validator, self).get_required_fields()

        #  If you answered other to question 19 (trading status)
        if self.answers.get('SQ1-1ci') == 'other (please specify)':
            req_fields.add('SQ1-1cii')

        #  If you answered yes to question 27 (non-UK business registered in EU)
        if self.answers.get('SQ1-1i-i', False):
            req_fields.add('SQ1-1i-ii')

        #  If you answered 'licensed' or 'a member of a relevant organisation' in question 29
        answer_29 = self.answers.get('SQ1-1j-i', [])
        if answer_29 and len(answer_29) > 0 and (
            'licensed' in answer_29
            or 'a member of a relevant organisation' in answer_29
        ):
            req_fields.add('SQ1-1j-ii')

        # If you answered yes to either question 53 or 54 (tax returns)
        if self.answers.get('SQ4-1a', False) or self.answers.get('SQ4-1b', False):
            req_fields.add('SQ4-1c')

        # If you answered Yes to questions 39 - 51 (discretionary exclusion)
        dependent_fields = [
            'SQ2-2a', 'SQ3-1a', 'SQ3-1b', 'SQ3-1c', 'SQ3-1d', 'SQ3-1e', 'SQ3-1f', 'SQ3-1g',
            'SQ3-1h-i', 'SQ3-1h-ii', 'SQ3-1i-i', 'SQ3-1i-ii', 'SQ3-1j'
        ]
        if any(self.answers.get(field) for field in dependent_fields):
            req_fields.add('SQ3-1k')

        # If you answered No to question 26 (established in the UK)
        if 'SQ5-2a' in self.answers and not self.answers['SQ5-2a']:
            req_fields.add('SQ1-1i-i')
            req_fields.add('SQ1-1j-i')

        return req_fields

class DOSValidator(DeclarationValidator):
    optional_fields = {
        "mitigatingFactors", "mitigatingFactors2", "mitigatingFactors3", "tradingStatusOther",
        "modernSlaveryStatement", "modernSlaveryStatementOptional", "modernSlaveryReportingRequirements",
        # Registered in UK = no
        "appropriateTradeRegisters", "appropriateTradeRegistersNumber",
        "licenceOrMemberRequired", "licenceOrMemberRequiredDetails",
    }

    dependent_fields = {
        # If you responded yes to any of questions 22 to 34
        "mitigatingFactors": [
            'misleadingInformation', 'confidentialInformation', 'influencedContractingAuthority',
            'witheldSupportingDocuments', 'seriousMisrepresentation', 'significantOrPersistentDeficiencies',
            'distortedCompetition', 'conflictOfInterest', 'distortingCompetition', 'graveProfessionalMisconduct',
            'bankrupt', 'environmentalSocialLabourLaw', 'taxEvasion'
        ],
        # If you responded yes to either 36 or 37
        "mitigatingFactors2": [
            "unspentTaxConvictions", "GAAR"
        ],
        # If you responded yes to 50 (Modern Slavery)
        "modernSlaveryStatement": [
            "modernSlaveryTurnover",
        ],
        "modernSlaveryReportingRequirements": [
            "modernSlaveryTurnover"
        ],
    }

    email_validation_fields = {"contactEmailContractNotice", "primaryContactEmail"}
    character_limit = 5000

    def get_required_fields(self) -> Set[str]:
        req_fields = super(DOSValidator, self).get_required_fields()

        for target_field, fields in self.dependent_fields.items():
            if any(self.answers.get(field) for field in fields):
                req_fields.add(target_field)

        # Describe your trading status
        if self.answers.get('tradingStatus') == "other (please specify)":
            req_fields.add('tradingStatusOther')

        # If your company was not established in the UK
        if self.answers.get('establishedInTheUK') is False:
            req_fields.add('appropriateTradeRegisters')
            # If yes to appropriate trade registers
            if self.answers.get('appropriateTradeRegisters') is True:
                req_fields.add('appropriateTradeRegistersNumber')

            req_fields.add('licenceOrMemberRequired')
            # If not 'none of the above' to licenceOrMemberRequired
            if self.answers.get('licenceOrMemberRequired') in ['licensed', 'a member of a relevant organisation']:
                req_fields.add('licenceOrMemberRequiredDetails')

        # If supplier doesn't meet the Modern Slavery reporting requirements, they don't need to upload a statement
        # but must have mitigatingFactors3 explanation
        if self.answers.get('modernSlaveryReportingRequirements') is False:
            req_fields.add('mitigatingFactors3')
            req_fields.remove('modernSlaveryStatement')

        return req_fields

class SharedValidator(DOSValidator):
    # From DOS2 and G8 onwards, validate DUNS number length
    number_string_fields = [('dunsNumber', 9)]
    word_limit = 500

class G12Validator(SharedValidator):

    def errors(self) -> Dict[str, str]:

        errors_map = super().errors()

        q1_answer = self.answers.get('servicesHaveOrSupportCloudHostingCloudSoftware')
        q2_answer = self.answers.get('servicesHaveOrSupportCloudSupport')

        q1_negative = "My organisation isn't submitting cloud hosting (lot 1) or cloud software (lot 2) services"
        q2_negative = "My organisation isn't submitting cloud support (lot 3) services"

        if q1_answer == q1_negative and q2_answer == q2_negative:
            errors_map.update({
                'servicesHaveOrSupportCloudHostingCloudSoftware': 'dependent_question_error',
                'servicesHaveOrSupportCloudSupport': 'dependent_question_error',
            })

        return errors_map

def is_valid_percentage(value: Any) -> bool:
    # Design System guidance is that we should allow users to provide answers with or without units.
    if isinstance(value, str):
        value = value.rstrip('%')

    try:
        number = float(value)
    except ValueError:
        return False
    else:
        return 0 <= number <= 100

class DOS5Validator(SharedValidator):
    """Following an accessibility review, a number of questions and answers were changed for DOS 5"""
    percentage_fields = ["subcontractingInvoicesPaid"]

    optional_fields = SharedValidator.optional_fields.union({
        "subcontracting30DayPayments",
        "subcontractingInvoicesPaid"}
    )

    def get_required_fields(self) -> Set[str]:
        req_fields = super(DOS5Validator, self).get_required_fields()

        # as per subcontracting configuration on digitalmarketplace-frameworks
        if self.answers.get("subcontracting") in [
            "as a prime contractor, using third parties (subcontractors) to provide some services",
            "as part of a consortium or special purpose vehicle, using third parties (subcontractors) to provide some "
            "services"
        ]:
            req_fields.add("subcontracting30DayPayments")
            req_fields.add("subcontractingInvoicesPaid")

        return req_fields

    def formatting_errors(self, answers) -> Dict[str, str]:
        error_map = super(DOS5Validator, self).formatting_errors(answers)

        for field in self.percentage_fields:
            value = self.answers.get(field)
            if value is not None and not is_valid_percentage(value):
                error_map[field] = 'not_a_number'

        return error_map

VALIDATORS = {
    "g-cloud-7": G7Validator,
    "g-cloud-8": SharedValidator,
    "digital-outcomes-and-specialists": DOSValidator,
    "digital-outcomes-and-specialists-2": SharedValidator,
    "g-cloud-9": SharedValidator,
    "g-cloud-10": SharedValidator,
    "digital-outcomes-and-specialists-3": SharedValidator,
    "g-cloud-11": SharedValidator,
    "digital-outcomes-and-specialists-4": SharedValidator,
    "g-cloud-12": G12Validator,
    "digital-outcomes-and-specialists-5": DOS5Validator,
}
<|endoftext|>


<commit_before># Python script to migrate all ECAS data to LabCAS

import os
from xml.etree import ElementTree as ET
import pprint
import glob
import urllib.parse
from shutil import copyfile
import xml.sax.saxutils as saxutils

ecas_metadata_dir = "/home/cinquini/ECAS_MIGRATION/datasets/"
ecas_data_dir = "/data/archive"
labcas_metadata_dir = "/home/cinquini/ECAS_MIGRATION/ecas-metadata/"
labcas_data_dir = "/home/cinquini/ECAS_MIGRATION/labcas_archive"

pp = pprint.PrettyPrinter(indent=4)

collection_dict = {
                   "CollectionName":"",
                   "CollectionDescription":"",
                   "Discipline":"",
                   "Institution":"",
                   "InstitutionId":"",
                   "LeadPI":"",
                   "LeadPIId":"",
                   "DataCustodian":"",
                   "DataCustodianEmail":"",
                   "Organ":"",
                   "OrganId":"",
                   "OwnerPrincipal":"cn=All Users,dc=edrn,dc=jpl,dc=nasa,dc=gov",
                   "CollaborativeGroup":"",
                   "QAState":"",
                   "Consortium":"EDRN",
                   "ProtocolName":"",
                   "ProtocolId":"",
                   "Species":"",
                   "MethodDetails":"",
                   "ResultsAndConclusionSummary":"",
                   "PubMedID":"",
                   "DateDatasetFrozen":"",
                   "QAState":"",
                   "DataDisclaimer":""
                   }

dataset_dict = {
                "DatasetId":"",
                "DatasetName":"",
                "DatasetDescription":""
                }

def read_product_type_metadata(input_xml_file):
    '''
    Reads product type metadata from XML and converts it to Python dictionaries.
    '''

    print("Reading %s" % input_xml_file)
    collection_metadata = collection_dict.copy()
    dataset_metadata = dataset_dict.copy()

    xml = ET.parse(input_xml_file)
    root_element = xml.getroot()

    # Description --> CollectionDescription
    description_element = root_element.find('.//description')
    collection_metadata['CollectionDescription'] = description_element.text

    metadata_element = root_element.find('.//metadata')
    for keyval_element in metadata_element.findall('./keyval'):
        key = keyval_element.find("key").text
        val = keyval_element.find("val").text

        # replace newline characters from metadata values
        print('val=%s' % val)
        if val:
            val = val.replace('\n',' ').replace('\r\n',' ')
            # un-escape XML characters
            val = saxutils.unescape('val')

        # DataSetName --> CollectionName, DatasetName
        if key == 'DataSetName':
            collection_metadata['CollectionName'] = val
            collection_metadata['CollectionId'] = val.replace(' ', '_')
            dataset_metadata['DatasetName'] = val
            dataset_metadata['DatasetDescription'] = val
            dataset_metadata['DatasetId'] = val.replace(" ","_")

        # ProtocolID --> ProtocolId, ProtocolName
        elif key == 'ProtocolID':
            collection_metadata['ProtocolId'] = val
            collection_metadata['ProtocolName'] = "FIXME"

        # LeadPI --> LeadPI, LeadPIId
        elif key == 'LeadPI':
            collection_metadata['LeadPI'] = val
            collection_metadata['LeadPIId'] = "FIXME"

        # SiteName --> Institution, InstitutionId
        elif key == 'SiteName':
            collection_metadata['InstitutionId'] = val
            collection_metadata['Institution'] = "FIXME"

        # DataCustodian --> DataCustodian
        elif key == 'DataCustodian':
            collection_metadata['DataCustodian'] = val

        # DataCustodianEmail --> DataCustodianEmail
        elif key == 'DataCustodianEmail':
            collection_metadata['DataCustodianEmail'] = val

        # OrganSite --> Organ, OrganId
        elif key == 'OrganSite':
            collection_metadata['Organ'] = val
            collection_metadata['OrganId'] = "FIXME"

        # CollaborativeGroup --> CollaborativeGroup
        elif key == 'CollaborativeGroup':
            collection_metadata['CollaborativeGroup'] = val

        # MethodDetails --> MethodDetails
        elif key == 'MethodDetails':
            collection_metadata['MethodDetails'] = val

        # ResultsAndConclusionSummary --> ResultsAndConclusionSummary
        elif key == 'ResultsAndConclusionSummary':
            collection_metadata['ResultsAndConclusionSummary'] = val

        # PubMedID --> PubMedID
        elif key == 'PubMedID':
            collection_metadata['PubMedID'] = "http://www.ncbi.nlm.nih.gov/pubmed/%s" % val

        # DateDatasetFrozen --> DateDatasetFrozen
        elif key == 'DateDatasetFrozen':
            collection_metadata['DateDatasetFrozen'] = val

        # Date --> Date
        elif key == 'Date':
            collection_metadata['Date'] = val

        # QAState --> QAState
        elif key == 'QAState':
            collection_metadata['QAState'] = val

        # DataDisclaimer --> DataDisclaimer
        elif key == 'DataDisclaimer':
            collection_metadata['DataDisclaimer'] = val

    #pp.pprint(collection_metadata)
    #pp.pprint(dataset_metadata)

    return {
             'Collection':collection_metadata,
             'Dataset': dataset_metadata
             }

def write_product_type_metadata(metadata):
    '''
    Serializes metadata from Python dictionaries into a Python configuration file.
    '''

    collection_id = metadata['Collection']['CollectionId']
    config_file_dir = "%s/%s" % (labcas_metadata_dir, collection_id)
    if not os.path.exists(config_file_dir):
        os.makedirs(config_file_dir)
    config_file_name = collection_id + ".cfg"
    config_file_full_name = os.path.join(config_file_dir, config_file_name)

    with open(config_file_full_name, 'w') as f:

        # collection metadata
        f.write('[Collection]\n')
        for key, value in metadata['Collection'].items():
            f.write('%s=%s\n' % ( key, value))

        # dataset metadata
        f.write('[Dataset]\n')
        for key, value in metadata['Dataset'].items():
            f.write('%s=%s\n' % ( key, value))

def read_product_metadata(dataset_dir):

    # list metadata files in all sub-direcories
    met_files = glob.glob('%s/**/*.met' % dataset_dir, recursive=True)

    file_metadata_array = []

    for met_file in met_files:

        file_metadata = {}

        xml = ET.parse(met_file)
        root_element = xml.getroot()

        for keyval_element in root_element.findall('./keyval'):
            key = keyval_element.find("key").text
            val = keyval_element.find("val").text
            if val:
                # reverse URL encoding
                val = urllib.parse.unquote(val).replace('+',' ')
                # replace newline characters from metadata values
                val = val.replace('\n',' ').replace('\r\n',' ')
                # un-escape XML characters
                val = saxutils.unescape('val')

                if key == 'CAS.ProductName' or key == 'CAS.ProductId' or key == 'CAS.ProductReceivedTime':
                    pass # ignore
                else:
                    file_metadata[key] = val

        #pp.pprint(file_metadata)

        # add dictionary to list, if not empty
        if file_metadata:
            file_metadata_array.append(file_metadata)

    return file_metadata_array

def copy_products(collection_id, file_metadata_array, output_dir):

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    for file_metadata in file_metadata_array:

        filename = file_metadata['Filename']
        output_file = os.path.join(output_dir, filename)

        input_file = find_most_recent_file(ecas_data_dir, filename)
        if input_file:

            # copy file
            print("Copying file: %s --> %s" % (input_file, output_file))
            copyfile(input_file, output_file)

            # write associated metadata
            write_product_metadata(file_metadata, output_file)

        else:
            print("Cannot find file: '%s' [collection: %s]" % (filename, collection_id) )

def write_product_metadata(file_metadata, output_file):

    met_file = output_file + ".xmlmet"
    with open(met_file, 'w') as f:
        f.write('<cas:metadata xmlns:cas="http://oodt.jpl.nasa.gov/1.0/cas">\n')

        # FIXME: select specific elements
        for key, value in file_metadata.items():
            f.write('\t<keyval type="vector">\n')
            f.write('\t\t<key>_File_%s</key>\n' % key)
            f.write('\t\t<val>%s</val>\n' % value)
            f.write('\t</keyval>\n')
        f.write('</cas:metadata>')

def find_most_recent_file(root_dir, file_name):
    result = None
    mtime = None
    for root, dirs, files in os.walk(root_dir):
        if file_name in files:
            file_full_path = os.path.join(root, file_name)
            file_mod_time = os.path.getmtime(file_full_path)
            if mtime:
                if file_mod_time > mtime:
                    result = file_full_path
                    mtime = file_mod_time
            else:
                result = file_full_path
                mtime = file_mod_time
    return result

if __name__== "__main__":

    # loop over directories
    filenames = os.listdir(ecas_metadata_dir)

    for filename in filenames:
        dataset_dir = os.path.join(ecas_metadata_dir, filename)
        if os.path.isdir(dataset_dir):

            # FIXME
            if filename == 'COPY_NUMBER_LEV1':
            #if filename == 'WHIColonWistarSpeicher':
            #if True:
                input_xml_file = os.path.join(ecas_metadata_dir, ("%s.met" % filename))

                # read product type metadata from XML, convert to dictionaries
                metadata = read_product_type_metadata(input_xml_file)
                collection_id = metadata['Collection']['CollectionId']
                dataset_id = metadata['Dataset']['DatasetId']

                # write dictionary metadata to collection+dataset configuration file
                write_product_type_metadata(metadata)

                # read product type metadata from XML, convert to dictionaries
                file_metadata_array = read_product_metadata(dataset_dir)

                # copy data files
                # FIXME: remove 1
                output_dir = os.path.join(labcas_data_dir, collection_id, dataset_id, '1')
                copy_products(collection_id, file_metadata_array, output_dir)

<commit_msg>Fix bug<commit_after># Python script to migrate all ECAS data to LabCAS

import os
from xml.etree import ElementTree as ET
import pprint
import glob
import urllib.parse
from shutil import copyfile
import xml.sax.saxutils as saxutils

ecas_metadata_dir = "/home/cinquini/ECAS_MIGRATION/datasets/"
ecas_data_dir = "/data/archive"
labcas_metadata_dir = "/home/cinquini/ECAS_MIGRATION/ecas-metadata/"
labcas_data_dir = "/home/cinquini/ECAS_MIGRATION/labcas_archive"

pp = pprint.PrettyPrinter(indent=4)

collection_dict = {
                   "CollectionName":"",
                   "CollectionDescription":"",
                   "Discipline":"",
                   "Institution":"",
                   "InstitutionId":"",
                   "LeadPI":"",
                   "LeadPIId":"",
                   "DataCustodian":"",
                   "DataCustodianEmail":"",
                   "Organ":"",
                   "OrganId":"",
                   "OwnerPrincipal":"cn=All Users,dc=edrn,dc=jpl,dc=nasa,dc=gov",
                   "CollaborativeGroup":"",
                   "QAState":"",
                   "Consortium":"EDRN",
                   "ProtocolName":"",
                   "ProtocolId":"",
                   "Species":"",
                   "MethodDetails":"",
                   "ResultsAndConclusionSummary":"",
                   "PubMedID":"",
                   "DateDatasetFrozen":"",
                   "QAState":"",
                   "DataDisclaimer":""
                   }

dataset_dict = {
                "DatasetId":"",
                "DatasetName":"",
                "DatasetDescription":""
                }

def read_product_type_metadata(input_xml_file):
    '''
    Reads product type metadata from XML and converts it to Python dictionaries.
    '''

    print("Reading %s" % input_xml_file)
    collection_metadata = collection_dict.copy()
    dataset_metadata = dataset_dict.copy()

    xml = ET.parse(input_xml_file)
    root_element = xml.getroot()

    # Description --> CollectionDescription
    description_element = root_element.find('.//description')
    collection_metadata['CollectionDescription'] = description_element.text

    metadata_element = root_element.find('.//metadata')
    for keyval_element in metadata_element.findall('./keyval'):
        key = keyval_element.find("key").text
        val = keyval_element.find("val").text

        # replace newline characters from metadata values
        print('val=%s' % val)
        if val:
            val = val.replace('\n',' ').replace('\r\n',' ')
            # un-escape XML characters
            val = saxutils.unescape(val)

        # DataSetName --> CollectionName, DatasetName
        if key == 'DataSetName':
            collection_metadata['CollectionName'] = val
            collection_metadata['CollectionId'] = val.replace(' ', '_')
            dataset_metadata['DatasetName'] = val
            dataset_metadata['DatasetDescription'] = val
            dataset_metadata['DatasetId'] = val.replace(" ","_")

        # ProtocolID --> ProtocolId, ProtocolName
        elif key == 'ProtocolID':
            collection_metadata['ProtocolId'] = val
            collection_metadata['ProtocolName'] = "FIXME"

        # LeadPI --> LeadPI, LeadPIId
        elif key == 'LeadPI':
            collection_metadata['LeadPI'] = val
            collection_metadata['LeadPIId'] = "FIXME"

        # SiteName --> Institution, InstitutionId
        elif key == 'SiteName':
            collection_metadata['InstitutionId'] = val
            collection_metadata['Institution'] = "FIXME"

        # DataCustodian --> DataCustodian
        elif key == 'DataCustodian':
            collection_metadata['DataCustodian'] = val

        # DataCustodianEmail --> DataCustodianEmail
        elif key == 'DataCustodianEmail':
            collection_metadata['DataCustodianEmail'] = val

        # OrganSite --> Organ, OrganId
        elif key == 'OrganSite':
            collection_metadata['Organ'] = val
            collection_metadata['OrganId'] = "FIXME"

        # CollaborativeGroup --> CollaborativeGroup
        elif key == 'CollaborativeGroup':
            collection_metadata['CollaborativeGroup'] = val

        # MethodDetails --> MethodDetails
        elif key == 'MethodDetails':
            collection_metadata['MethodDetails'] = val

        # ResultsAndConclusionSummary --> ResultsAndConclusionSummary
        elif key == 'ResultsAndConclusionSummary':
            collection_metadata['ResultsAndConclusionSummary'] = val

        # PubMedID --> PubMedID
        elif key == 'PubMedID':
            collection_metadata['PubMedID'] = "http://www.ncbi.nlm.nih.gov/pubmed/%s" % val

        # DateDatasetFrozen --> DateDatasetFrozen
        elif key == 'DateDatasetFrozen':
            collection_metadata['DateDatasetFrozen'] = val

        # Date --> Date
        elif key == 'Date':
            collection_metadata['Date'] = val

        # QAState --> QAState
        elif key == 'QAState':
            collection_metadata['QAState'] = val

        # DataDisclaimer --> DataDisclaimer
        elif key == 'DataDisclaimer':
            collection_metadata['DataDisclaimer'] = val

    #pp.pprint(collection_metadata)
    #pp.pprint(dataset_metadata)

    return {
             'Collection':collection_metadata,
             'Dataset': dataset_metadata
             }

def write_product_type_metadata(metadata):
    '''
    Serializes metadata from Python dictionaries into a Python configuration file.
    '''

    collection_id = metadata['Collection']['CollectionId']
    config_file_dir = "%s/%s" % (labcas_metadata_dir, collection_id)
    if not os.path.exists(config_file_dir):
        os.makedirs(config_file_dir)
    config_file_name = collection_id + ".cfg"
    config_file_full_name = os.path.join(config_file_dir, config_file_name)

    with open(config_file_full_name, 'w') as f:

        # collection metadata
        f.write('[Collection]\n')
        for key, value in metadata['Collection'].items():
            f.write('%s=%s\n' % ( key, value))

        # dataset metadata
        f.write('[Dataset]\n')
        for key, value in metadata['Dataset'].items():
            f.write('%s=%s\n' % ( key, value))

def read_product_metadata(dataset_dir):

    # list metadata files in all sub-direcories
    met_files = glob.glob('%s/**/*.met' % dataset_dir, recursive=True)

    file_metadata_array = []

    for met_file in met_files:

        file_metadata = {}

        xml = ET.parse(met_file)
        root_element = xml.getroot()

        for keyval_element in root_element.findall('./keyval'):
            key = keyval_element.find("key").text
            val = keyval_element.find("val").text
            if val:
                # reverse URL encoding
                val = urllib.parse.unquote(val).replace('+',' ')
                # replace newline characters from metadata values
                val = val.replace('\n',' ').replace('\r\n',' ')
                # un-escape XML characters
                val = saxutils.unescape('val')

                if key == 'CAS.ProductName' or key == 'CAS.ProductId' or key == 'CAS.ProductReceivedTime':
                    pass # ignore
                else:
                    file_metadata[key] = val

        #pp.pprint(file_metadata)

        # add dictionary to list, if not empty
        if file_metadata:
            file_metadata_array.append(file_metadata)

    return file_metadata_array

def copy_products(collection_id, file_metadata_array, output_dir):

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    for file_metadata in file_metadata_array:

        filename = file_metadata['Filename']
        output_file = os.path.join(output_dir, filename)

        input_file = find_most_recent_file(ecas_data_dir, filename)
        if input_file:

            # copy file
            print("Copying file: %s --> %s" % (input_file, output_file))
            copyfile(input_file, output_file)

            # write associated metadata
            write_product_metadata(file_metadata, output_file)

        else:
            print("Cannot find file: '%s' [collection: %s]" % (filename, collection_id) )

def write_product_metadata(file_metadata, output_file):

    met_file = output_file + ".xmlmet"
    with open(met_file, 'w') as f:
        f.write('<cas:metadata xmlns:cas="http://oodt.jpl.nasa.gov/1.0/cas">\n')

        # FIXME: select specific elements
        for key, value in file_metadata.items():
            f.write('\t<keyval type="vector">\n')
            f.write('\t\t<key>_File_%s</key>\n' % key)
            f.write('\t\t<val>%s</val>\n' % value)
            f.write('\t</keyval>\n')
        f.write('</cas:metadata>')

def find_most_recent_file(root_dir, file_name):
    result = None
    mtime = None
    for root, dirs, files in os.walk(root_dir):
        if file_name in files:
            file_full_path = os.path.join(root, file_name)
            file_mod_time = os.path.getmtime(file_full_path)
            if mtime:
                if file_mod_time > mtime:
                    result = file_full_path
                    mtime = file_mod_time
            else:
                result = file_full_path
                mtime = file_mod_time
    return result

if __name__== "__main__":

    # loop over directories
    filenames = os.listdir(ecas_metadata_dir)

    for filename in filenames:
        dataset_dir = os.path.join(ecas_metadata_dir, filename)
        if os.path.isdir(dataset_dir):

            # FIXME
            if filename == 'COPY_NUMBER_LEV1':
            #if filename == 'WHIColonWistarSpeicher':
            #if True:
                input_xml_file = os.path.join(ecas_metadata_dir, ("%s.met" % filename))

                # read product type metadata from XML, convert to dictionaries
                metadata = read_product_type_metadata(input_xml_file)
                collection_id = metadata['Collection']['CollectionId']
                dataset_id = metadata['Dataset']['DatasetId']

                # write dictionary metadata to collection+dataset configuration file
                write_product_type_metadata(metadata)

                # read product type metadata from XML, convert to dictionaries
                file_metadata_array = read_product_metadata(dataset_dir)

                # copy data files
                # FIXME: remove 1
                output_dir = os.path.join(labcas_data_dir, collection_id, dataset_id, '1')
                copy_products(collection_id, file_metadata_array, output_dir)

<|endoftext|>


<commit_before>#!/usr/bin/env python

import json
import sys
import urllib2

base_url='http://127.0.0.1:8000/court/'
oauth_token='foobar'
num_errors=0
put_tmpl='{"public_url":"https://www.gov.uk/prefix/%s"}'

#with open('../data/sample_court.json') as f:
#    sample_court_json = f.read()

sample_court_json_1 = '{"name":"blah","slug":"blah","updated_at": "2014-03-18T12:33:12.176Z","closed":false,"alert":"","lat":0.0,"lon":0.0,"number":"200","DX":"2039D"}'

def same_arrays(a,b):
    for i,item in enumerate(a):
        if b[i] and b[i] != item:
            return False
    else:
        return True

def fail(message):
    global num_errors
    num_errors = num_errors + 1
    print "FAIL: "+str(message)

def check(description, response, status_code, body=None, win='WIN'):
    sys.stdout.write(description+"...")
    if response['status_code'] != status_code:
        fail("Different status codes: expected %d, got %d" % (status_code, response['status_code']))
    elif body:
        b = json.loads(body)
        r = json.loads(response['text'])
        if type(b) == type([]) and not same_arrays(r,b):
            fail("FAIL: Different arrays: expected %s, got %s" % (body, response['text']))
        elif r != b:
            fail("FAIL: Different objects: expected %s, got %s" % (body, response['text']))
        else:
            print win
    else:
        print win

def http(method='GET', uuid='', auth=False, body=None):
    opener = urllib2.build_opener(urllib2.HTTPHandler)
    request = urllib2.Request(base_url+uuid, data=body)
    request.get_method = lambda: method
    if auth:
        request.add_header('Authorization', 'Bearer ' + oauth_token)
    try:
        response = opener.open(request)
    except urllib2.HTTPError as e:
        response = e
    return {'status_code': response.getcode(), 'text': response.read()};

def get(uuid, auth=True):
    return http('GET', uuid, auth)

def put(uuid, json, auth=True):
    return http('PUT', uuid, auth, json)

def list(auth=True):
    return http('GET', '', auth)

def delete(uuid, auth=True):
    return http('DELETE', uuid, auth)

if __name__ == "__main__":
    check('delete all the things', delete('all-the-things'), 200)
    check('bad auth on put',       put('foo-bar', '[]', auth=False), 403)
    check('bad auth on get',       put('foo-bar', '[]', auth=False), 403)
    check('bad auth on list',      put('foo-bar', '[]', auth=False), 403)
    check('check empty list',      list(), 200, '[]')
    check('missing court',         get('de305d54-75b4-431b-adb2-eb6b9e546013'), 404)
    check('create a court',        put('de305d54-75b4-431b-adb2-eb6b9e546013', sample_court_json_1), 201)
    check('bad json payload',      put('de305d54-75b4-431b-adb2-eb6b9e546013', 'bad json'), 400)
    check('update a court',        put('de305d54-75b4-431b-adb2-eb6b9e546013', sample_court_json_1), 200)
    check('check only one court',  list(), 200, '['+sample_court_json_1+']')
    check('check single court',    get('de305d54-75b4-431b-adb2-eb6b9e546013'), 200, sample_court_json_1)
    check('delete court',          delete('de305d54-75b4-431b-adb2-eb6b9e546013'), 200)
    check('check empty list 2',    list(), 200, '[]')
    check('create court 2',        put('de305d54-75b4-431b-adb2-eb6b9e546014', sample_court_json_1), 201)
    check('create a court',        put('de305d54-75b4-431b-adb2-eb6b9e546015', sample_court_json_1), 201)
    check('check two courts',      list(), 200, '['+sample_court_json_1+','+sample_court_json_1+']')
    check('bad uuid',              put('bad-uuid', sample_court_json_1), 400)
    check('check PUT response',    put('de305d54-75b4-431b-adb2-eb6b9e546013', sample_court_json_1),
                                   201, put_tmpl % 'blah')
    check('clean up',              delete('all-the-things'), 200)
    print "done: %d errors" % num_errors
<commit_msg>unwrap<commit_after>#!/usr/bin/env python

import json
import sys
import urllib2

base_url='http://127.0.0.1:8000/court/'
oauth_token='foobar'
num_errors=0
put_tmpl='{"public_url":"https://www.gov.uk/prefix/%s"}'

#with open('../data/sample_court.json') as f:
#    sample_court_json = f.read()

sample_court_json_1 = '{"name":"blah","slug":"blah","updated_at": "2014-03-18T12:33:12.176Z","closed":false,"alert":"","lat":0.0,"lon":0.0,"number":"200","DX":"2039D"}'

def same_arrays(a,b):
    for i,item in enumerate(a):
        if b[i] and b[i] != item:
            return False
    else:
        return True

def fail(message):
    global num_errors
    num_errors = num_errors + 1
    print "FAIL: "+str(message)

def check(description, response, status_code, body=None, win='WIN'):
    sys.stdout.write(description+"...")
    if response['status_code'] != status_code:
        fail("Different status codes: expected %d, got %d" % (status_code, response['status_code']))
    elif body:
        b = json.loads(body)
        r = json.loads(response['text'])
        if type(b) == type([]) and not same_arrays(r,b):
            fail("FAIL: Different arrays: expected %s, got %s" % (body, response['text']))
        elif r != b:
            fail("FAIL: Different objects: expected %s, got %s" % (body, response['text']))
        else:
            print win
    else:
        print win

def http(method='GET', uuid='', auth=False, body=None):
    opener = urllib2.build_opener(urllib2.HTTPHandler)
    request = urllib2.Request(base_url+uuid, data=body)
    request.get_method = lambda: method
    if auth:
        request.add_header('Authorization', 'Bearer ' + oauth_token)
    try:
        response = opener.open(request)
    except urllib2.HTTPError as e:
        response = e
    return {'status_code': response.getcode(), 'text': response.read()};

def get(uuid, auth=True):
    return http('GET', uuid, auth)

def put(uuid, json, auth=True):
    return http('PUT', uuid, auth, json)

def list(auth=True):
    return http('GET', '', auth)

def delete(uuid, auth=True):
    return http('DELETE', uuid, auth)

if __name__ == "__main__":
    check('delete all the things', delete('all-the-things'), 200)
    check('bad auth on put',       put('foo-bar', '[]', auth=False), 403)
    check('bad auth on get',       put('foo-bar', '[]', auth=False), 403)
    check('bad auth on list',      put('foo-bar', '[]', auth=False), 403)
    check('check empty list',      list(), 200, '[]')
    check('missing court',         get('de305d54-75b4-431b-adb2-eb6b9e546013'), 404)
    check('create a court',        put('de305d54-75b4-431b-adb2-eb6b9e546013', sample_court_json_1), 201)
    check('bad json payload',      put('de305d54-75b4-431b-adb2-eb6b9e546013', 'bad json'), 400)
    check('update a court',        put('de305d54-75b4-431b-adb2-eb6b9e546013', sample_court_json_1), 200)
    check('check only one court',  list(), 200, '['+sample_court_json_1+']')
    check('check single court',    get('de305d54-75b4-431b-adb2-eb6b9e546013'), 200, sample_court_json_1)
    check('delete court',          delete('de305d54-75b4-431b-adb2-eb6b9e546013'), 200)
    check('check empty list 2',    list(), 200, '[]')
    check('create court 2',        put('de305d54-75b4-431b-adb2-eb6b9e546014', sample_court_json_1), 201)
    check('create a court',        put('de305d54-75b4-431b-adb2-eb6b9e546015', sample_court_json_1), 201)
    check('check two courts',      list(), 200, '['+sample_court_json_1+','+sample_court_json_1+']')
    check('bad uuid',              put('bad-uuid', sample_court_json_1), 400)
    check('check PUT response',    put('de305d54-75b4-431b-adb2-eb6b9e546013', sample_court_json_1), 201, put_tmpl % 'blah')
    check('clean up',              delete('all-the-things'), 200)
    print "done: %d errors" % num_errors
<|endoftext|>


<commit_before>"""
All commands to create a functioning HR api dataset
"""

import logging

from django.core.management import BaseCommand

from datasets import build_hr_data
from datasets import build_cbs_sbi
from datasets import build_ds_data
from datasets.hr import improve_location_with_search
from datasets.hr import models
from datasets.hr import location_stats

LOG = logging.getLogger(__name__)

class Command(BaseCommand):
    """
    Import HR data from handel regster makelaarsuite (mks) dump

    clear data using:

    - manage.py migrate handelsregister zero

    apply new/updated migrations

    - manage.py migrate handelsregister

    """

    def add_arguments(self, parser):
        parser.add_argument(
            '--bagfix',
            action='store_true',
            dest='bag',
            default=False,
            help='Fill hr_locatie with hr_baggeo table')

        parser.add_argument(
            '--geovestigingen',
            action='store_true',
            dest='geo_vest',
            default=False,
            help='Fill hr_geovestigingen table')

        parser.add_argument(
            '--search',
            action='store_true',
            dest='searchapi',
            default=False,
            help='Fill hr_locatie with search api')

        parser.add_argument(
            '--clearsearch',
            action='store_true',
            dest='clearsearch',
            default=False,
            help='Clear hr_locate from search api results')

        parser.add_argument(
            '--status',
            action='store_true',
            dest='stats',
            default=False,
            help='print location stats')

        parser.add_argument(
            '--cbs_sbi',
            action='store_true',
            dest='cbs_sbi',
            default=False,
            help='Fill cbs sbi-codes')

        parser.add_argument(
            '--dataselectie',
            action='store_true',
            dest='dataselectie',
            default=False,
            help='Fill dataselectie view')

    def bag_check(self):
        if models.GeoVBO.objects.count() < 10000:
            raise ValueError(
                'Import bag data with "copy_bagvbo_to_hr(local).sh"')

    def handle(self, *args, **options):
        """
        validate and execute import task
        """
        LOG.info('Handelsregister import started')

        if options['bag']:
            # load bag data in GeoVBO with
            # copy_bag_to_hr script
            self.bag_check()
            build_hr_data.fill_location_with_bag()
            location_stats.log_rapport_counts(action='bag')
        elif options['geo_vest']:
            build_hr_data.fill_geo_table()
            location_stats.log_rapport_counts(action='map')
        elif options['cbs_sbi']:
            build_cbs_sbi.cbsbi_table()
        elif options['dataselectie']:
            build_ds_data._build_joined_ds_table(action='ds')
            # import cProfile
            # cProfile.runctx('build_ds_data._build_joined_ds_table()', globals(), locals(), '/tmp/statsds_data')
        elif options['searchapi']:
            improve_location_with_search.guess()
            location_stats.log_rapport_counts(action='fix')
        elif options['clearsearch']:
            build_hr_data.clear_autocorrect()
        elif options['stats']:
            location_stats.log_rapport_counts()
        else:
            # convert mks dump
            self.bag_check()
            build_hr_data.fill_stelselpedia()
            location_stats.log_rapport_counts()
            # now update mks locations with bag locations
            # check if bag data is correctly loaded
            # we need bag data to correct missing geometry data
            build_hr_data.fill_location_with_bag()
            LOG.info('hr_geovestigingen %s', models.Locatie.objects.count())
            assert models.GeoVestigingen.objects.count() == 0
            assert models.Locatie.objects.count() > 200000
            location_stats.log_rapport_counts(action='mks')
<commit_msg>log fix<commit_after>"""
All commands to create a functioning HR api dataset
"""

import logging

from django.core.management import BaseCommand

from datasets import build_hr_data
from datasets import build_cbs_sbi
from datasets import build_ds_data
from datasets.hr import improve_location_with_search
from datasets.hr import models
from datasets.hr import location_stats

LOG = logging.getLogger(__name__)

class Command(BaseCommand):
    """
    Import HR data from handel regster makelaarsuite (mks) dump

    clear data using:

    - manage.py migrate handelsregister zero

    apply new/updated migrations

    - manage.py migrate handelsregister

    """

    def add_arguments(self, parser):
        parser.add_argument(
            '--bagfix',
            action='store_true',
            dest='bag',
            default=False,
            help='Fill hr_locatie with hr_baggeo table')

        parser.add_argument(
            '--geovestigingen',
            action='store_true',
            dest='geo_vest',
            default=False,
            help='Fill hr_geovestigingen table')

        parser.add_argument(
            '--search',
            action='store_true',
            dest='searchapi',
            default=False,
            help='Fill hr_locatie with search api')

        parser.add_argument(
            '--clearsearch',
            action='store_true',
            dest='clearsearch',
            default=False,
            help='Clear hr_locate from search api results')

        parser.add_argument(
            '--status',
            action='store_true',
            dest='stats',
            default=False,
            help='print location stats')

        parser.add_argument(
            '--cbs_sbi',
            action='store_true',
            dest='cbs_sbi',
            default=False,
            help='Fill cbs sbi-codes')

        parser.add_argument(
            '--dataselectie',
            action='store_true',
            dest='dataselectie',
            default=False,
            help='Fill dataselectie view')

    def bag_check(self):
        if models.GeoVBO.objects.count() < 10000:
            raise ValueError(
                'Import bag data with "copy_bagvbo_to_hr(local).sh"')

    def handle(self, *args, **options):
        """
        validate and execute import task
        """
        LOG.info('Handelsregister import started')

        if options['bag']:
            # load bag data in GeoVBO with
            # copy_bag_to_hr script
            self.bag_check()
            build_hr_data.fill_location_with_bag()
            location_stats.log_rapport_counts(action='bag')
        elif options['geo_vest']:
            build_hr_data.fill_geo_table()
            location_stats.log_rapport_counts(action='map')
        elif options['cbs_sbi']:
            build_cbs_sbi.cbsbi_table()
        elif options['dataselectie']:
            build_ds_data._build_joined_ds_table()
            # import cProfile
            # cProfile.runctx('build_ds_data._build_joined_ds_table()', globals(), locals(), '/tmp/statsds_data')
            location_stats.log_rapport_counts(action='ds')
        elif options['searchapi']:
            improve_location_with_search.guess()
            location_stats.log_rapport_counts(action='fix')
        elif options['clearsearch']:
            build_hr_data.clear_autocorrect()
        elif options['stats']:
            location_stats.log_rapport_counts()
        else:
            # convert mks dump
            self.bag_check()
            build_hr_data.fill_stelselpedia()
            location_stats.log_rapport_counts()
            # now update mks locations with bag locations
            # check if bag data is correctly loaded
            # we need bag data to correct missing geometry data
            build_hr_data.fill_location_with_bag()
            LOG.info('hr_geovestigingen %s', models.Locatie.objects.count())
            assert models.GeoVestigingen.objects.count() == 0
            assert models.Locatie.objects.count() > 200000
            location_stats.log_rapport_counts(action='mks')
<|endoftext|>


<commit_before>#!/usr/bin/env python

from datetime import datetime
import glob
import json
import os
import time
import urllib

from cssmin import cssmin
from flask import Markup, g, render_template, request
from slimit import minify
from smartypants import smartypants

import app_config
import copytext

class BetterJSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime):
            encoded_object = obj.isoformat()
        else:
            encoded_object = json.JSONEncoder.default(self, obj)

        return encoded_object

class Includer(object):
    """
    Base class for Javascript and CSS psuedo-template-tags.

    See `make_context` for an explanation of `asset_depth`.
    """
    def __init__(self, asset_depth=0, static_path='', absolute=False):
        self.includes = []
        self.tag_string = None
        self.asset_depth = asset_depth
        self.static_path = static_path
        self.absolute = absolute

    def push(self, path):
        self.includes.append(path)

        return ''

    def _compress(self):
        raise NotImplementedError()

    def _relativize_path(self, path):
        relative_path = path
        depth = len(request.path.split('/')) - (2 + self.asset_depth)

        while depth > 0:
            relative_path = '../%s' % relative_path
            depth -= 1

        return relative_path

    def _absolutize_path(self, path):
        return '%s/%s/%s' % (app_config.S3_BASE_URL, self.static_path, path)

    def render(self, path):
        if self.absolute:
            pather = self._absolutize_path
        else:
            pather = self._relativize_path
        if getattr(g, 'compile_includes', False):
            if path in g.compiled_includes:
                timestamp_path = g.compiled_includes[path]
            else:
                # Add a timestamp to the rendered filename to prevent caching
                timestamp = int(time.time())
                front, back = path.rsplit('.', 1)
                timestamp_path = '%s.%i.%s' % (front, timestamp, back)

                # Delete old rendered versions, just to be tidy
                old_versions = glob.glob(('%s/www/%s.*.%s' % (self.static_path, front, back)).lstrip('/'))

                for f in old_versions:
                    os.remove(f)

            out_path = ('%s/www/%s' % (self.static_path, timestamp_path)).lstrip('/')

            if path not in g.compiled_includes:
                print 'Rendering %s' % out_path

                with open(out_path, 'w') as f:
                    f.write(self._compress().encode('utf-8'))

            # See "fab render"
            g.compiled_includes[path] = timestamp_path

            markup = Markup(self.tag_string % pather(timestamp_path))
        else:
            response = ','.join(self.includes)

            response = '\n'.join([
                self.tag_string % pather(src) for src in self.includes
            ])

            markup = Markup(response)

        del self.includes[:]

        return markup

class JavascriptIncluder(Includer):
    """
    Psuedo-template tag that handles collecting Javascript and serving appropriate clean or compressed versions.
    """
    def __init__(self, *args, **kwargs):
        Includer.__init__(self, *args, **kwargs)

        self.tag_string = '<script type="text/javascript" src="%s"></script>'

    def _compress(self):
        output = []
        src_paths = []

        for src in self.includes:
            src_paths.append('www/%s' % src)

            with open(('%s/www/%s' % (self.static_path, src)).lstrip('/')) as f:
                print '- compressing %s' % src
                output.append(minify(f.read().encode('utf-8')))

        context = make_context()
        context['paths'] = src_paths

        header = render_template('_js_header.js', **context)
        output.insert(0, header)

        return '\n'.join(output)

class CSSIncluder(Includer):
    """
    Psuedo-template tag that handles collecting CSS and serving appropriate clean or compressed versions.
    """
    def __init__(self, *args, **kwargs):
        Includer.__init__(self, *args, **kwargs)

        self.tag_string = '<link rel="stylesheet" type="text/css" href="%s" />'

    def _compress(self):
        output = []

        src_paths = []

        for src in self.includes:

            if src.endswith('less'):
                src_paths.append('%s' % src)
                src = src.replace('less', 'css') # less/example.less -> css/example.css
                src = '%s.less.css' % src[:-4]   # css/example.css -> css/example.less.css
            else:
                src_paths.append(('%s/www/%s' % (self.static_path, src)).lstrip('/'))

            with open(('%s/www/%s' % (self.static_path, src)).lstrip('/')) as f:
                print '- compressing %s' % src
                output.append(cssmin(f.read().encode('utf-8')))

        context = make_context()
        context['paths'] = src_paths

        header = render_template('_css_header.css', **context)
        output.insert(0, header)

        return '\n'.join(output)

def flatten_app_config():
    """
    Returns a copy of app_config containing only
    configuration variables.
    """
    config = {}

    # Only all-caps [constant] vars get included
    for k, v in app_config.__dict__.items():
        if k.upper() == k:
            config[k] = v

    return config

def make_context(asset_depth=0, static_path='', absolute=False):
    """
    Create a base-context for rendering views.
    Includes app_config and JS/CSS includers.

    `asset_depth` indicates how far into the url hierarchy
    the assets are hosted. If 0, then they are at the root.
    If 1 then at /foo/, etc.
    """
    context = flatten_app_config()

    context['COPY'] = copytext.Copy(app_config.COPY_PATH)
    context['JS'] = JavascriptIncluder(asset_depth=asset_depth, static_path=static_path, absolute=absolute)
    context['CSS'] = CSSIncluder(asset_depth=asset_depth, static_path=static_path, absolute=absolute)

    return context

def urlencode_filter(s):
    """
    Filter to urlencode strings.
    """
    if type(s) == 'Markup':
        s = s.unescape()

    # Evaulate COPY elements
    if type(s) is not unicode:
        s = unicode(s)

    s = s.encode('utf8')
    s = urllib.quote_plus(s)

    return Markup(s)

def smarty_filter(s):
    """
    Filter to smartypants strings.
    """
    if type(s) == 'Markup':
        s = s.unescape()

    # Evaulate COPY elements
    if type(s) is not unicode:
        s = unicode(s)

    s = smartypants(s)

    return Markup(s)

<commit_msg>Add docstring.<commit_after>#!/usr/bin/env python

from datetime import datetime
import glob
import json
import os
import time
import urllib

from cssmin import cssmin
from flask import Markup, g, render_template, request
from slimit import minify
from smartypants import smartypants

import app_config
import copytext

class BetterJSONEncoder(json.JSONEncoder):
    """
    A JSON encoder that handles complex types (dates).
    """
    def default(self, obj):
        if isinstance(obj, datetime):
            encoded_object = obj.isoformat()
        else:
            encoded_object = json.JSONEncoder.default(self, obj)

        return encoded_object

class Includer(object):
    """
    Base class for Javascript and CSS psuedo-template-tags.

    See `make_context` for an explanation of `asset_depth`.
    """
    def __init__(self, asset_depth=0, static_path='', absolute=False):
        self.includes = []
        self.tag_string = None
        self.asset_depth = asset_depth
        self.static_path = static_path
        self.absolute = absolute

    def push(self, path):
        self.includes.append(path)

        return ''

    def _compress(self):
        raise NotImplementedError()

    def _relativize_path(self, path):
        relative_path = path
        depth = len(request.path.split('/')) - (2 + self.asset_depth)

        while depth > 0:
            relative_path = '../%s' % relative_path
            depth -= 1

        return relative_path

    def _absolutize_path(self, path):
        return '%s/%s/%s' % (app_config.S3_BASE_URL, self.static_path, path)

    def render(self, path):
        if self.absolute:
            pather = self._absolutize_path
        else:
            pather = self._relativize_path
        if getattr(g, 'compile_includes', False):
            if path in g.compiled_includes:
                timestamp_path = g.compiled_includes[path]
            else:
                # Add a timestamp to the rendered filename to prevent caching
                timestamp = int(time.time())
                front, back = path.rsplit('.', 1)
                timestamp_path = '%s.%i.%s' % (front, timestamp, back)

                # Delete old rendered versions, just to be tidy
                old_versions = glob.glob(('%s/www/%s.*.%s' % (self.static_path, front, back)).lstrip('/'))

                for f in old_versions:
                    os.remove(f)

            out_path = ('%s/www/%s' % (self.static_path, timestamp_path)).lstrip('/')

            if path not in g.compiled_includes:
                print 'Rendering %s' % out_path

                with open(out_path, 'w') as f:
                    f.write(self._compress().encode('utf-8'))

            # See "fab render"
            g.compiled_includes[path] = timestamp_path

            markup = Markup(self.tag_string % pather(timestamp_path))
        else:
            response = ','.join(self.includes)

            response = '\n'.join([
                self.tag_string % pather(src) for src in self.includes
            ])

            markup = Markup(response)

        del self.includes[:]

        return markup

class JavascriptIncluder(Includer):
    """
    Psuedo-template tag that handles collecting Javascript and serving appropriate clean or compressed versions.
    """
    def __init__(self, *args, **kwargs):
        Includer.__init__(self, *args, **kwargs)

        self.tag_string = '<script type="text/javascript" src="%s"></script>'

    def _compress(self):
        output = []
        src_paths = []

        for src in self.includes:
            src_paths.append('www/%s' % src)

            with open(('%s/www/%s' % (self.static_path, src)).lstrip('/')) as f:
                print '- compressing %s' % src
                output.append(minify(f.read().encode('utf-8')))

        context = make_context()
        context['paths'] = src_paths

        header = render_template('_js_header.js', **context)
        output.insert(0, header)

        return '\n'.join(output)

class CSSIncluder(Includer):
    """
    Psuedo-template tag that handles collecting CSS and serving appropriate clean or compressed versions.
    """
    def __init__(self, *args, **kwargs):
        Includer.__init__(self, *args, **kwargs)

        self.tag_string = '<link rel="stylesheet" type="text/css" href="%s" />'

    def _compress(self):
        output = []

        src_paths = []

        for src in self.includes:

            if src.endswith('less'):
                src_paths.append('%s' % src)
                src = src.replace('less', 'css') # less/example.less -> css/example.css
                src = '%s.less.css' % src[:-4]   # css/example.css -> css/example.less.css
            else:
                src_paths.append(('%s/www/%s' % (self.static_path, src)).lstrip('/'))

            with open(('%s/www/%s' % (self.static_path, src)).lstrip('/')) as f:
                print '- compressing %s' % src
                output.append(cssmin(f.read().encode('utf-8')))

        context = make_context()
        context['paths'] = src_paths

        header = render_template('_css_header.css', **context)
        output.insert(0, header)

        return '\n'.join(output)

def flatten_app_config():
    """
    Returns a copy of app_config containing only
    configuration variables.
    """
    config = {}

    # Only all-caps [constant] vars get included
    for k, v in app_config.__dict__.items():
        if k.upper() == k:
            config[k] = v

    return config

def make_context(asset_depth=0, static_path='', absolute=False):
    """
    Create a base-context for rendering views.
    Includes app_config and JS/CSS includers.

    `asset_depth` indicates how far into the url hierarchy
    the assets are hosted. If 0, then they are at the root.
    If 1 then at /foo/, etc.
    """
    context = flatten_app_config()

    context['COPY'] = copytext.Copy(app_config.COPY_PATH)
    context['JS'] = JavascriptIncluder(asset_depth=asset_depth, static_path=static_path, absolute=absolute)
    context['CSS'] = CSSIncluder(asset_depth=asset_depth, static_path=static_path, absolute=absolute)

    return context

def urlencode_filter(s):
    """
    Filter to urlencode strings.
    """
    if type(s) == 'Markup':
        s = s.unescape()

    # Evaulate COPY elements
    if type(s) is not unicode:
        s = unicode(s)

    s = s.encode('utf8')
    s = urllib.quote_plus(s)

    return Markup(s)

def smarty_filter(s):
    """
    Filter to smartypants strings.
    """
    if type(s) == 'Markup':
        s = s.unescape()

    # Evaulate COPY elements
    if type(s) is not unicode:
        s = unicode(s)

    s = smartypants(s)

    return Markup(s)

<|endoftext|>