<commit_before>import sys
import operator
from jnius import autoclass, cast
from TripsModule import trips_module

# Declare KQML java classes
KQMLPerformative = autoclass('TRIPS.KQML.KQMLPerformative')
KQMLList = autoclass('TRIPS.KQML.KQMLList')
KQMLObject = autoclass('TRIPS.KQML.KQMLObject')

from bioagents.dtda import DTDA

class DTDA_Module(trips_module.TripsModule):
    '''
    The DTDA module is a TRIPS module built around the DTDA agent. Its role is to
    receive and decode messages and send responses from and to other agents in
    the system.
    '''
    def __init__(self, argv):
        # Call the constructor of TripsModule
        super(DTDA_Module, self).__init__(argv)
        # Instantiate a singleton DTDA agent
        self.tasks = ['is-drug-target', 'find-target-drug',
                    'find-disease-targets', 'find-treatment']

    def init(self):
        '''
        Initialize TRIPS module
        '''
        super(DTDA_Module, self).init()
        # Send subscribe messages
        for task in self.tasks:
            task_str = 'ONT::%s' % task.upper()
            msg_txt = '(subscribe :content (request &key :content (%s . *)))' % task_str
            self.send(KQMLPerformative.fromString(msg_txt))
        self.dtda = DTDA()
        self.ready()

    def receive_request(self, msg, content):
        '''
        If a "request" message is received, decode the task and the content
        and call the appropriate function to prepare the response. A reply
        "tell" message is then sent back.
        '''
        content_list = cast(KQMLList, content)
        task = content_list.get(0).toString().upper()
        sender = msg.getParameter(":sender")
        if task == 'ONT::IS-DRUG-TARGET':
            reply_content = self.respond_is_drug_target(content_list)
        elif task == 'ONT::FIND-TARGET-DRUG':
            reply_content = self.respond_find_target_drug(content_list)
        elif task == 'ONT::FIND-DISEASE-TARGETS':
            reply_content = self.respond_find_disease_targets(content_list)
        elif task == 'ONT::FIND-TREATMENT':
            reply_content = self.respond_find_treatment(content_list)
        else:
            self.error_reply(msg, 'unknown request task ' + task)
            return

        reply_msg = KQMLPerformative('reply')
        reply_msg.setParameter(':content', cast(KQMLObject, reply_content))
        self.reply(msg, reply_msg)

    def respond_is_drug_target(self, content_list):
        '''
        Response content to is-drug-target request
        '''
        # TODO: get parameters from content
        is_target = self.dtda.is_nominal_drug_target('Vemurafenib', 'BRAF')
        reply_content = KQMLList()
        if is_target:
            msg_str = 'TRUE'
        else:
            msg_str = 'FALSE'
        reply_content.add(msg_str)
        return reply_content

    def respond_find_target_drug(self, content_list):
        '''
        Response content to find-target-drug request
        '''
        # TODO: implement
        reply_content = KQMLList()
        reply_content.add('')
        return reply_content

    def respond_find_disease_targets(self, content_list):
        '''
        Response content to find-disease-targets request
        '''
        # TODO: implement
        self.dtda.get_mutation_statistics('pancreatic', 'missense')
        reply_content = KQMLList()
        reply_content.add('')
        return reply_content

    def respond_find_treatment(self, content_list):
        '''
        Response content to find-treatment request
        '''
        reply_content = KQMLList()

        # Parse content
        disease_str = content_list.getKeywordArg(':disease')
        if disease_str is None:
            print 'no disease set'
            reply_content.add('')
            return reply_content
        disease_str = disease_str.toString()
        disease_terms = disease_str[1:-1].split(' ')
        disease = disease_terms[0]
        disease_type = disease_terms[1]
        if disease.upper() != 'ONT::CANCER':
            print 'problem with disease name'
            reply_content.add('')
            return reply_content
        disease_type_filter = disease_type[5:].lower()

        # First, look for possible disease targets
        mutation_stats = self.dtda.get_mutation_statistics(disease_type_filter, 'missense')
        if mutation_stats is None:
            print 'no mutation stats'
            reply_content.add('')
            return reply_content
        # Return the top mutation as a possible target
        mutations_sorted = sorted(mutation_stats.items(),
            key=operator.itemgetter(1), reverse=True)
        top_mutation = mutations_sorted[0]
        mut_protein = top_mutation[0]
        mut_percent = int(top_mutation[1][0]*100.0)

        reply_content = KQMLList.fromString('(ONT::PROPORTION :REFSET (ONT::CANCER-PATIENT) :QUAN (ONT::PERCENT %d) :CONTENT (ONT::MUTATION :AFFECTED ONT::%s))' % (mut_percent, mut_protein))
        return reply_content

if __name__ == "__main__":
    dm = DTDA_Module(['-name', 'DTDA'] + sys.argv[1:])
    dm.run()

<commit_msg>Update message format for find-treatment task in DTDA<commit_after>import sys
import operator
from jnius import autoclass, cast
from TripsModule import trips_module

# Declare KQML java classes
KQMLPerformative = autoclass('TRIPS.KQML.KQMLPerformative')
KQMLList = autoclass('TRIPS.KQML.KQMLList')
KQMLObject = autoclass('TRIPS.KQML.KQMLObject')

from bioagents.dtda import DTDA

class DTDA_Module(trips_module.TripsModule):
    '''
    The DTDA module is a TRIPS module built around the DTDA agent. Its role is to
    receive and decode messages and send responses from and to other agents in
    the system.
    '''
    def __init__(self, argv):
        # Call the constructor of TripsModule
        super(DTDA_Module, self).__init__(argv)
        self.tasks = {'ONT::PERFORM': ['ONT::IS-DRUG-TARGET',
                    'ONT::FIND-TARGET-DRUG', 'ONT::FIND-DISEASE-TARGETS',
                    'ONT::FIND-TREATMENT']}

    def init(self):
        '''
        Initialize TRIPS module
        '''
        super(DTDA_Module, self).init()
        # Send subscribe messages
        for task, subtasks in self.tasks.iteritems():
            for subtask in subtasks:
                msg_txt = '(subscribe :content (request &key :content ' +\
                    '(%s &key :content (%s . *))))' % (task, subtask)
                self.send(KQMLPerformative.fromString(msg_txt))
        # Instantiate a singleton DTDA agent
        self.dtda = DTDA()
        # Send ready message
        self.ready()

    def receive_request(self, msg, content):
        '''
        If a "request" message is received, decode the task and the content
        and call the appropriate function to prepare the response. A reply
        "tell" message is then sent back.
        '''
        content_list = cast(KQMLList, content)
        task_str = content_list.get(0).toString().upper()
        if task_str == 'ONT::PERFORM':
            subtask = cast(KQMLList,content_list.getKeywordArg(':content'))
            subtask_str = subtask.get(0).toString().upper()
            if subtask_str == 'ONT::IS-DRUG-TARGET':
                reply_content = self.respond_is_drug_target(content_list)
            elif subtask_str == 'ONT::FIND-TARGET-DRUG':
                reply_content = self.respond_find_target_drug(content_list)
            elif subtask_str == 'ONT::FIND-DISEASE-TARGETS':
                reply_content = self.respond_find_disease_targets(content_list)
            elif subtask_str == 'ONT::FIND-TREATMENT':
                reply_content = self.respond_find_treatment(subtask)
            else:
                self.error_reply(msg, 'unknown request task ' + task)
                return

        reply_msg = KQMLPerformative('reply')
        reply_msg.setParameter(':content', cast(KQMLObject, reply_content))
        self.reply(msg, reply_msg)

    def respond_is_drug_target(self, content_list):
        '''
        Response content to is-drug-target request
        '''
        # TODO: get parameters from content
        is_target = self.dtda.is_nominal_drug_target('Vemurafenib', 'BRAF')
        reply_content = KQMLList()
        if is_target:
            msg_str = 'TRUE'
        else:
            msg_str = 'FALSE'
        reply_content.add(msg_str)
        return reply_content

    def respond_find_target_drug(self, content_list):
        '''
        Response content to find-target-drug request
        '''
        # TODO: implement
        reply_content = KQMLList()
        reply_content.add('')
        return reply_content

    def respond_find_disease_targets(self, content_list):
        '''
        Response content to find-disease-targets request
        '''
        # TODO: implement
        self.dtda.get_mutation_statistics('pancreatic', 'missense')
        reply_content = KQMLList()
        reply_content.add('')
        return reply_content

    def respond_find_treatment(self, content_list):
        '''
        Response content to find-treatment request
        '''
        reply_content = KQMLList()

        # Parse content
        disease_str = content_list.getKeywordArg(':disease')
        if disease_str is None:
            print 'no disease set'
            reply_content.add('')
            return reply_content
        disease_str = disease_str.toString()
        disease_terms = disease_str[1:-1].split(' ')
        disease = disease_terms[0]
        disease_type = disease_terms[1]
        if disease.upper() != 'W::CANCER':
            print 'problem with disease name'
            reply_content.add('')
            return reply_content
        disease_type_filter = disease_type[3:].lower()

        # First, look for possible disease targets
        mutation_stats = self.dtda.get_mutation_statistics(disease_type_filter, 'missense')
        if mutation_stats is None:
            print 'no mutation stats'
            reply_content.add('')
            return reply_content
        # Return the top mutation as a possible target
        mutations_sorted = sorted(mutation_stats.items(),
            key=operator.itemgetter(1), reverse=True)
        top_mutation = mutations_sorted[0]
        mut_protein = top_mutation[0]
        mut_percent = int(top_mutation[1][0]*100.0)

        mut_response = KQMLList.fromString('(ONT::TELL :content ' +\
            '(ONT::PROPORTION :refset (ONT::CANCER-PATIENT) ' +\
            ':quan (ONT::PERCENT %d) ' % mut_percent +\
            ':content (ONT::MUTATION :affected ONT::%s)))' % mut_protein)

        # Try to find a drug targeting KRAS
        drugs = self.dtda.find_target_drug(mut_protein)
        #import ipdb; ipdb.set_trace()
        if not drugs:
            drug_response = KQMLList.fromString('(ONT::TELL :content (ONT::DONT-KNOW ' +\
                ':content (ONT::A X1 :instance-of ONT::DRUG)))')
        else:
            drug_response = KQMLList.fromString('(ONT::TELL :content ())')
        reply_content.add(KQMLList(mut_response))
        reply_content.add(KQMLList(drug_response))

        return reply_content

if __name__ == "__main__":
    dm = DTDA_Module(['-name', 'DTDA'] + sys.argv[1:])
    dm.run()

<|endoftext|>


<commit_before>"""
To fix:
    'Corequisite:'
    'Recommended:'
    'x, y, and z' -> and_list([x, y, z])
"""

class or_list(list):
    def __init__(self, lst):
        list.__init__(self, lst)
        self.kind = "or"

    def english(self):
        lst = []
        for x in self:
            if isinstance(x, str):
                lst.append(x)
            else:
                lst.append(x.english())

        return '(' + ' or '.join(lst) + ')'

class and_list(list):
    def __init__(self, lst):
        list.__init__(self, lst)
        self.kind = "and"

    def english(self):
        lst = []
        for x in self:
            if isinstance(x, str):
                lst.append(x)
            else:
                lst.append(x.english())

        return '(' + ' and '.join(lst) + ')'

def clean_string(string):
    string = string.lstrip()
    string = string.replace('.', '')
    # Makes parsing a bit easier
    string = string.replace(' and either ', '; or ')
    string = string.replace(' with a grade of C- or better', '')
    string = string.replace(' or equivalent', '')
    string = string.replace('or consent of instructor', '')
    string = string.replace('or instructor consent', '')
    string = string.rstrip()
    string = string.rstrip(',')
    if debug:
        print(string)
    return string

def split_by_semicolon(string):
    outer_list = None
    for i,x in enumerate(string):
        if x == ";":
            search_section = string[i+1:i+5]
            remove_section = None

            split_list = string.split(";")

            if "and" in search_section:
                outer_list = and_list(split_list)
                remove_section = " and "
            elif "or" in search_section:
                outer_list = or_list(split_list)
                remove_section = " or "
            else:
                # Semicolon seems to imply and if not stated
                split_list[-1] = split_list[-1][1:]
                outer_list = and_list(split_list)

            if remove_section is not None:
                # Replace the search section since we don't need it anymore
                outer_list[-1] = outer_list[-1].replace(remove_section, "", 1)

    if outer_list is None:
        if debug:
            print(string)
        return split_by_comma(string)
    # It's a list, so need to dissect further
    else:
        if debug:
            print(outer_list)
        for i,x in enumerate(outer_list):
            outer_list[i] = split_by_comma(x)
        return outer_list

def split_by_comma(string):
    comma_list = None
    for i,x in enumerate(string):
        if x == ",":
            search_section = string[i+1:i+5]
            remove_section = None

            split_list = string.split(",")

            if "and" in search_section:
                comma_list = and_list(split_list)
                remove_section = " and "
            elif "or" in search_section:
                comma_list = or_list(split_list)
                remove_section = " or "

            if remove_section is not None:
                # Replace the search section since we don't need it anymore
                comma_list[-1] = comma_list[-1].replace(remove_section, "", 1)

    if comma_list is None:
        if debug:
            print(string)
        return split_by_coordinating_conjunction(string)
    else:
        if debug:
            print(comma_list)
        for i,x in enumerate(comma_list):
            comma_list[i] = split_by_coordinating_conjunction(x)
        return comma_list

def split_by_coordinating_conjunction(string):
    if "one of the following: " in string:
        string = string.replace("one of the following: ", '')
        if debug:
            print(string)
        return split_by_coordinating_conjunction(string)
    elif "and" in string:
        if ',' in string:
            string = string.replace(" and", ',')
            split_list = string.split(', ')
        else:
            split_list = string.split(" and ")

        return and_list(split_list)
    elif "or" in string:
        if ',' in string:
            string = string.replace(" or", ',')
            split_list = string.split(', ')
        else:
            split_list = string.split(" or ")
        return or_list(split_list)

    return string

def parse_string(string):
    print(string)
    string = clean_string(string)

    outer_list = split_by_semicolon(string)
    if not isinstance(outer_list, str):
        print(outer_list.english() + "\n")
    else:
        print(outer_list)
        print("\n")

debug=False
if __name__=="__main__":
    import sys

    if len(sys.argv) == 1:
        strings = [
        "Completion of ELM requirement, and passing score on MAPE or MATH 117 with a grade of C- or better or MATH 118 with a grade of C- or better, or consent of instructor.",

        "MATH 118 or equivalent.",

        "MATH 141 or MATH 161 with a grade of C- or better, or consent of instructor.",

        "CSC/CPE 103 with a grade of C- or better, or instructor consent.",

        "CPE/CSC 357; and CSC 141 or CSC 348.",

        "CSC 141 or CSC 348, and MATH 142; or CPE/CSC 103 and MATH 248."
        ]
    else:
        args = sys.argv
        if '-d' in args or '--debug' in args:
            debug=True
            args.remove('-d' if '-d' in args else '--debug')
        strings = args[1:]

    for x in strings:
        parse_string(x)
<commit_msg>Need to figure out how to pass func parameter<commit_after>"""
To fix:
    'Corequisite:'
    'Recommended:'
    'x, y, and z' -> and_list([x, y, z])
"""

class none_list(list):
    def __init__(self, lst):
        list.__init__(self, lst)
        self.kind = "none"

    def expand(self, func):
        lst = []
        for x in self:
            if isinstance(x, str):
                lst.append(x)
            else:
                lst.append(x.func())
        return lst

    def get_dict(self):
        lst = self.expand(self.get_dict)
        return {self.kind: self}

    def english(self):
        lst = self.expand(self.english)
        return '(' + f' {self.kind} '.join(lst) + ')'

class or_list(none_list):
    def __init__(self, lst):
        none_list.__init__(self, lst)
        self.kind = "or"

class and_list(none_list):
    def __init__(self, lst):
        none_list.__init__(self, lst)
        self.kind = "and"

def clean_string(string):
    string = string.lstrip()
    string = string.replace('.', '')
    # Makes parsing a bit easier
    string = string.replace(' and either ', '; or ')
    string = string.replace(' with a grade of C- or better', '')
    string = string.replace(' or equivalent', '')
    string = string.replace('or consent of instructor', '')
    string = string.replace('or instructor consent', '')
    string = string.rstrip()
    string = string.rstrip(',')
    if debug:
        print(string)
    return string

def split_by_semicolon(string):
    outer_list = None
    for i,x in enumerate(string):
        if x == ";":
            search_section = string[i+1:i+5]
            remove_section = None

            split_list = string.split(";")

            if "and" in search_section:
                outer_list = and_list(split_list)
                remove_section = " and "
            elif "or" in search_section:
                outer_list = or_list(split_list)
                remove_section = " or "
            else:
                # Semicolon seems to imply and if not stated
                split_list[-1] = split_list[-1][1:]
                outer_list = and_list(split_list)

            if remove_section is not None:
                # Replace the search section since we don't need it anymore
                outer_list[-1] = outer_list[-1].replace(remove_section, "", 1)

    if outer_list is None:
        if debug:
            print(string)
        return split_by_comma(string)
    # It's a list, so need to dissect further
    else:
        if debug:
            print(outer_list)
        for i,x in enumerate(outer_list):
            outer_list[i] = split_by_comma(x)
        return outer_list

def split_by_comma(string):
    comma_list = None
    for i,x in enumerate(string):
        if x == ",":
            search_section = string[i+1:i+5]
            remove_section = None

            split_list = string.split(",")

            if "and" in search_section:
                comma_list = and_list(split_list)
                remove_section = " and "
            elif "or" in search_section:
                comma_list = or_list(split_list)
                remove_section = " or "

            if remove_section is not None:
                # Replace the search section since we don't need it anymore
                comma_list[-1] = comma_list[-1].replace(remove_section, "", 1)

    if comma_list is None:
        if debug:
            print(string)
        return split_by_coordinating_conjunction(string)
    else:
        if debug:
            print(comma_list)
        for i,x in enumerate(comma_list):
            comma_list[i] = split_by_coordinating_conjunction(x)
        return comma_list

def split_by_coordinating_conjunction(string):
    if "one of the following: " in string:
        string = string.replace("one of the following: ", '')
        if debug:
            print(string)
        return split_by_coordinating_conjunction(string)
    elif "and" in string:
        if ',' in string:
            string = string.replace(" and", ',')
            split_list = string.split(', ')
        else:
            split_list = string.split(" and ")

        return and_list(split_list)
    elif "or" in string:
        if ',' in string:
            string = string.replace(" or", ',')
            split_list = string.split(', ')
        else:
            split_list = string.split(" or ")
        return or_list(split_list)

    return string

def parse_string(string):
    print(string)
    string = clean_string(string)

    outer_list = split_by_semicolon(string)
    if not isinstance(outer_list, str):
        print(outer_list.english() + "\n")
    else:
        print(outer_list)
        print("\n")

debug=False
if __name__=="__main__":
    import sys

    if len(sys.argv) == 1:
        strings = [
        "Completion of ELM requirement, and passing score on MAPE or MATH 117 with a grade of C- or better or MATH 118 with a grade of C- or better, or consent of instructor.",

        "MATH 118 or equivalent.",

        "MATH 141 or MATH 161 with a grade of C- or better, or consent of instructor.",

        "CSC/CPE 103 with a grade of C- or better, or instructor consent.",

        "CPE/CSC 357; and CSC 141 or CSC 348.",

        "CSC 141 or CSC 348, and MATH 142; or CPE/CSC 103 and MATH 248."
        ]
    else:
        args = sys.argv
        if '-d' in args or '--debug' in args:
            debug=True
            args.remove('-d' if '-d' in args else '--debug')
        strings = args[1:]

    for x in strings:
        parse_string(x)
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

import os
import logging

from datetime import date

from crispy_forms.templatetags.crispy_forms_filters import as_crispy_form
from crispy_forms.utils import render_crispy_form
from deposit.declaration import get_declaration_pdf
from deposit.forms import PaperDepositForm
from deposit.forms import UserPreferencesForm
from deposit.models import DepositRecord
from deposit.models import Repository
from deposit.models import UserPreferences
from django.conf import settings
from django.contrib.auth.decorators import user_passes_test
from django.contrib.auth.mixins import LoginRequiredMixin
from django.http import FileResponse
from django.http import HttpResponseForbidden
from django.http import Http404
from django.shortcuts import get_object_or_404
from django.shortcuts import render
from django.template import RequestContext
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_POST
from django.views.generic import View
from jsonview.decorators import json_view
from papers.models import Paper
from papers.user import is_authenticated
from ratelimit.decorators import ratelimit

logger = logging.getLogger('dissemin.' + __name__)

def get_all_repositories_and_protocols(paper, user):
    """
    Gets all enabled repositories together with their protocols.
    :param paper: The paper to be deposited
    :param user: The user who wnats to deposit
    :return: List of tupels containing each a pair of a repository and its protocol.
    """
    repositories = Repository.objects.filter(enabled=True)
    protocols = []
    for r in repositories:
        implem = r.protocol_for_deposit(paper, user)
        # if implem is not None:
        protocols.append((r, implem))
    return protocols

@json_view
@user_passes_test(is_authenticated)
def get_metadata_form(request):
    repo = get_object_or_404(Repository, pk=request.GET.get('repository'))
    if not repo.enabled:
        return HttpResponseForbidden(_('This repository is currently not enabled.'))
    paper = get_object_or_404(Paper, pk=request.GET.get('paper'))
    protocol = repo.protocol_for_deposit(paper, request.user)
    if protocol is None:
        logger.warning("No protocol")
        return {'status': 'repoNotAvailable',
                'message': _('This repository is not available for this paper.')}

    form = protocol.get_form()
    return {'status': 'success',
            'form': as_crispy_form(form)}

@user_passes_test(is_authenticated)
def start_view(request, pk):
    paper = get_object_or_404(
        Paper.objects.prefetch_related(
            'oairecord_set'
        ),
        pk=pk
    )
    repositories = get_all_repositories_and_protocols(paper, request.user)
    repositories_protocol = {repo.id :proto for repo, proto in repositories}

    # select the most appropriate repository
    userprefs = UserPreferences.get_by_user(request.user)
    preselected_repository = userprefs.get_preferred_or_last_repository()
    preselected_protocol = None
    if preselected_repository:
        preselected_protocol = repositories_protocol.get(preselected_repository.id)
    # If the preferred repository is not available for this paper, pick any
    if not preselected_protocol:
        for repo, protocol in repositories:
            if protocol is not None:
                preselected_repository = repo
                preselected_protocol = protocol
                break

    breadcrumbs = paper.breadcrumbs()
    breadcrumbs.append((_('Deposit'), ''))
    context = {
            'paper': paper,
            'max_file_size': settings.DEPOSIT_MAX_FILE_SIZE,
            'available_repositories': repositories,
            'selected_repository': preselected_repository,
            'selected_protocol': preselected_protocol,
            'is_owner': paper.is_owned_by(request.user, flexible=True),
            'breadcrumbs': breadcrumbs,
            'repositoryForm': None,
            }
    if request.GET.get('type') not in [None, 'preprint', 'postprint', 'pdfversion']:
        return HttpResponseForbidden()
    return render(request, 'deposit/start.html', context)

@user_passes_test(is_authenticated)
def list_deposits(request):
    deposits = DepositRecord.objects.filter(
        user=request.user,
        identifier__isnull=False
    ).order_by(
        '-date'
    ).select_related(
        'license',
        'oairecord',
        'paper',
        'repository',
    )
    context = {
        'deposits': deposits
    }
    return render(request, 'deposit/deposits.html', context)

@user_passes_test(is_authenticated)
def edit_repo_preferences(request, pk):
    repo = get_object_or_404(Repository, pk=pk)
    if not repo.enabled:
        return HttpResponseForbidden(_('This repository is currently not enabled.'))
    protocol = repo.get_implementation()
    context = {
        'repositories': Repository.objects.all(),
        'repository': repo,
        'protocol': protocol,
    }
    if not protocol:
        raise Http404(_('This repository could not be found.'))

    if request.method == 'POST':
        pref_form = protocol.get_preferences_form(request.user, request.POST)
        if not pref_form:
            raise Http404(_('This repository does not have any settings.'))
        if pref_form.is_valid():
            pref_form.save()
    else:
        pref_form = protocol.get_preferences_form(request.user)
        if not pref_form:
            raise Http404(_('This repository does not have any settings.'))

    context['preferences_form'] = pref_form
    return render(request, 'deposit/repo_preferences.html', context)

@user_passes_test(is_authenticated)
def edit_global_preferences(request):
    context = {
        'repositories': Repository.objects.filter(enabled=True),
    }
    prefs = UserPreferences.get_by_user(request.user)
    if request.method == 'POST':
        pref_form = UserPreferencesForm(request.POST, instance=prefs)
        pref_form.save()

    pref_form = UserPreferencesForm(instance=prefs)
    if not pref_form:
        raise Http404(_('This repository does not have any settings.'))

    context['preferences_form'] = pref_form
    return render(request, 'deposit/global_preferences.html', context)

@require_POST
@json_view
@user_passes_test(is_authenticated)
@ratelimit(key='ip', rate='200/d')
def submitDeposit(request, pk):
    paper = get_object_or_404(Paper, pk=pk)
    context = {'status': 'error'}
    form = PaperDepositForm(data=request.POST)
    if not form.is_valid():
        context['form'] = form.errors
        return context, 400

    # This validation could take place in the form (but we need access to the
    # paper and user?)
    repository = form.cleaned_data['radioRepository']
    protocol = repository.protocol_for_deposit(paper, request.user)
    if protocol is None:
        context['radioRepository'] = _(
            "This repository cannot be used for this paper.")
        return context, 400

    repositoryForm = protocol.get_bound_form(request.POST)

    if not repositoryForm.is_valid():
        request_context = RequestContext(request)
        form_html = render_crispy_form(repositoryForm, context=request_context)
        context['form_html'] = form_html
        return context, 400

    # Store the repository as in the user preferences
    userprefs = UserPreferences.get_by_user(request.user)
    userprefs.last_repository = repository
    userprefs.save(update_fields=['last_repository'])

    # Check that the paper has been uploaded by the same user
    pdf = form.cleaned_data['file_id']
    if pdf.user != request.user:
        context['message'] = _('Access to the PDF was denied.')
        return context, 400

    # Submit the paper to the repository
    path = os.path.join(settings.MEDIA_ROOT, pdf.file.name)

    # Create initial record
    d = DepositRecord(
            paper=paper,
            user=pdf.user,
            repository=repository,
            upload_type=form.cleaned_data['radioUploadType'],
            file=pdf)
    d.save()

    submitResult = protocol.submit_deposit_wrapper(path, repositoryForm)

    d.request = submitResult.logs
    if submitResult.status == 'failed':
        context['message'] = submitResult.message
        d.save()
        # Send the failed deposit as error to sentry
        msg = "Deposit failed for id %s for paper %s \n\n" % (d.pk, paper.pk)
        logger.error(msg + submitResult.logs)
        return context, 400

    d.identifier = submitResult.identifier
    d.additional_info = submitResult.additional_info
    d.status = submitResult.status
    d.oairecord = submitResult.oairecord
    d.license = submitResult.license
    d.pub_date = submitResult.embargo_date
    if d.pub_date is None and d.status == 'published':
        d.pub_date = date.today()
    d.save()
    paper.update_availability()
    paper.save()
    paper.update_index()

    context['status'] = 'success'
    # TODO change this (we don't need it)
    context['upload_id'] = d.id
    return context

class LetterDeclarationView(LoginRequiredMixin, View):
    """
    View to return a test file. The user must be logged in and must have access to the deposit.
    """

    def get(self, request, pk):
        """
        We test if the user is the user that own the deposit and return a PDF file if the repository specifies this
        """
        dr = get_object_or_404(DepositRecord.objects.select_related('paper', 'repository', 'user', 'license'), pk=pk)

        if dr.user != request.user:
            return HttpResponseForbidden(_("Access to this ressource not allowed."))
        # If the repository requires a letter of declaration, we try to create the pdf, otherwise we return 404.
        if dr.repository.letter_declaration != '' and dr.status == "pending":
            pdf = get_declaration_pdf(dr, request.user)
            pdf.seek(0)
            filename = _("Declaration {}.pdf").format(dr.paper.title)
            return FileResponse(pdf, as_attachment=True, filename=filename)
        else:
            raise Http404(_("No pdf found for dr {}".format(dr.pk)))
<commit_msg>imports unordered<commit_after># -*- encoding: utf-8 -*-
# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

import logging
import os

from crispy_forms.templatetags.crispy_forms_filters import as_crispy_form
from crispy_forms.utils import render_crispy_form
from datetime import date
from jsonview.decorators import json_view
from ratelimit.decorators import ratelimit

from django.conf import settings
from django.contrib.auth.decorators import user_passes_test
from django.contrib.auth.mixins import LoginRequiredMixin
from django.http import FileResponse
from django.http import HttpResponseForbidden
from django.http import Http404
from django.shortcuts import get_object_or_404
from django.shortcuts import render
from django.template import RequestContext
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_POST
from django.views.generic import View

from deposit.declaration import get_declaration_pdf
from deposit.forms import PaperDepositForm
from deposit.forms import UserPreferencesForm
from deposit.models import DepositRecord
from deposit.models import Repository
from deposit.models import UserPreferences
from papers.models import Paper
from papers.user import is_authenticated

logger = logging.getLogger('dissemin.' + __name__)

def get_all_repositories_and_protocols(paper, user):
    """
    Gets all enabled repositories together with their protocols.
    :param paper: The paper to be deposited
    :param user: The user who wnats to deposit
    :return: List of tupels containing each a pair of a repository and its protocol.
    """
    repositories = Repository.objects.filter(enabled=True)
    protocols = []
    for r in repositories:
        implem = r.protocol_for_deposit(paper, user)
        # if implem is not None:
        protocols.append((r, implem))
    return protocols

@json_view
@user_passes_test(is_authenticated)
def get_metadata_form(request):
    repo = get_object_or_404(Repository, pk=request.GET.get('repository'))
    if not repo.enabled:
        return HttpResponseForbidden(_('This repository is currently not enabled.'))
    paper = get_object_or_404(Paper, pk=request.GET.get('paper'))
    protocol = repo.protocol_for_deposit(paper, request.user)
    if protocol is None:
        logger.warning("No protocol")
        return {'status': 'repoNotAvailable',
                'message': _('This repository is not available for this paper.')}

    form = protocol.get_form()
    return {'status': 'success',
            'form': as_crispy_form(form)}

@user_passes_test(is_authenticated)
def start_view(request, pk):
    paper = get_object_or_404(
        Paper.objects.prefetch_related(
            'oairecord_set'
        ),
        pk=pk
    )
    repositories = get_all_repositories_and_protocols(paper, request.user)
    repositories_protocol = {repo.id :proto for repo, proto in repositories}

    # select the most appropriate repository
    userprefs = UserPreferences.get_by_user(request.user)
    preselected_repository = userprefs.get_preferred_or_last_repository()
    preselected_protocol = None
    if preselected_repository:
        preselected_protocol = repositories_protocol.get(preselected_repository.id)
    # If the preferred repository is not available for this paper, pick any
    if not preselected_protocol:
        for repo, protocol in repositories:
            if protocol is not None:
                preselected_repository = repo
                preselected_protocol = protocol
                break

    breadcrumbs = paper.breadcrumbs()
    breadcrumbs.append((_('Deposit'), ''))
    context = {
            'paper': paper,
            'max_file_size': settings.DEPOSIT_MAX_FILE_SIZE,
            'available_repositories': repositories,
            'selected_repository': preselected_repository,
            'selected_protocol': preselected_protocol,
            'is_owner': paper.is_owned_by(request.user, flexible=True),
            'breadcrumbs': breadcrumbs,
            'repositoryForm': None,
            }
    if request.GET.get('type') not in [None, 'preprint', 'postprint', 'pdfversion']:
        return HttpResponseForbidden()
    return render(request, 'deposit/start.html', context)

@user_passes_test(is_authenticated)
def list_deposits(request):
    deposits = DepositRecord.objects.filter(
        user=request.user,
        identifier__isnull=False
    ).order_by(
        '-date'
    ).select_related(
        'license',
        'oairecord',
        'paper',
        'repository',
    )
    context = {
        'deposits': deposits
    }
    return render(request, 'deposit/deposits.html', context)

@user_passes_test(is_authenticated)
def edit_repo_preferences(request, pk):
    repo = get_object_or_404(Repository, pk=pk)
    if not repo.enabled:
        return HttpResponseForbidden(_('This repository is currently not enabled.'))
    protocol = repo.get_implementation()
    context = {
        'repositories': Repository.objects.all(),
        'repository': repo,
        'protocol': protocol,
    }
    if not protocol:
        raise Http404(_('This repository could not be found.'))

    if request.method == 'POST':
        pref_form = protocol.get_preferences_form(request.user, request.POST)
        if not pref_form:
            raise Http404(_('This repository does not have any settings.'))
        if pref_form.is_valid():
            pref_form.save()
    else:
        pref_form = protocol.get_preferences_form(request.user)
        if not pref_form:
            raise Http404(_('This repository does not have any settings.'))

    context['preferences_form'] = pref_form
    return render(request, 'deposit/repo_preferences.html', context)

@user_passes_test(is_authenticated)
def edit_global_preferences(request):
    context = {
        'repositories': Repository.objects.filter(enabled=True),
    }
    prefs = UserPreferences.get_by_user(request.user)
    if request.method == 'POST':
        pref_form = UserPreferencesForm(request.POST, instance=prefs)
        pref_form.save()

    pref_form = UserPreferencesForm(instance=prefs)
    if not pref_form:
        raise Http404(_('This repository does not have any settings.'))

    context['preferences_form'] = pref_form
    return render(request, 'deposit/global_preferences.html', context)

@require_POST
@json_view
@user_passes_test(is_authenticated)
@ratelimit(key='ip', rate='200/d')
def submitDeposit(request, pk):
    paper = get_object_or_404(Paper, pk=pk)
    context = {'status': 'error'}
    form = PaperDepositForm(data=request.POST)
    if not form.is_valid():
        context['form'] = form.errors
        return context, 400

    # This validation could take place in the form (but we need access to the
    # paper and user?)
    repository = form.cleaned_data['radioRepository']
    protocol = repository.protocol_for_deposit(paper, request.user)
    if protocol is None:
        context['radioRepository'] = _(
            "This repository cannot be used for this paper.")
        return context, 400

    repositoryForm = protocol.get_bound_form(request.POST)

    if not repositoryForm.is_valid():
        request_context = RequestContext(request)
        form_html = render_crispy_form(repositoryForm, context=request_context)
        context['form_html'] = form_html
        return context, 400

    # Store the repository as in the user preferences
    userprefs = UserPreferences.get_by_user(request.user)
    userprefs.last_repository = repository
    userprefs.save(update_fields=['last_repository'])

    # Check that the paper has been uploaded by the same user
    pdf = form.cleaned_data['file_id']
    if pdf.user != request.user:
        context['message'] = _('Access to the PDF was denied.')
        return context, 400

    # Submit the paper to the repository
    path = os.path.join(settings.MEDIA_ROOT, pdf.file.name)

    # Create initial record
    d = DepositRecord(
            paper=paper,
            user=pdf.user,
            repository=repository,
            upload_type=form.cleaned_data['radioUploadType'],
            file=pdf)
    d.save()

    submitResult = protocol.submit_deposit_wrapper(path, repositoryForm)

    d.request = submitResult.logs
    if submitResult.status == 'failed':
        context['message'] = submitResult.message
        d.save()
        # Send the failed deposit as error to sentry
        msg = "Deposit failed for id %s for paper %s \n\n" % (d.pk, paper.pk)
        logger.error(msg + submitResult.logs)
        return context, 400

    d.identifier = submitResult.identifier
    d.additional_info = submitResult.additional_info
    d.status = submitResult.status
    d.oairecord = submitResult.oairecord
    d.license = submitResult.license
    d.pub_date = submitResult.embargo_date
    if d.pub_date is None and d.status == 'published':
        d.pub_date = date.today()
    d.save()
    paper.update_availability()
    paper.save()
    paper.update_index()

    context['status'] = 'success'
    # TODO change this (we don't need it)
    context['upload_id'] = d.id
    return context

class LetterDeclarationView(LoginRequiredMixin, View):
    """
    View to return a test file. The user must be logged in and must have access to the deposit.
    """

    def get(self, request, pk):
        """
        We test if the user is the user that own the deposit and return a PDF file if the repository specifies this
        """
        dr = get_object_or_404(DepositRecord.objects.select_related('paper', 'repository', 'user', 'license'), pk=pk)

        if dr.user != request.user:
            return HttpResponseForbidden(_("Access to this ressource not allowed."))
        # If the repository requires a letter of declaration, we try to create the pdf, otherwise we return 404.
        if dr.repository.letter_declaration != '' and dr.status == "pending":
            pdf = get_declaration_pdf(dr, request.user)
            pdf.seek(0)
            filename = _("Declaration {}.pdf").format(dr.paper.title)
            return FileResponse(pdf, as_attachment=True, filename=filename)
        else:
            raise Http404(_("No pdf found for dr {}".format(dr.pk)))
<|endoftext|>


<commit_before>
VERSION = '0.18.2.6'
<commit_msg>bump version in __init__.py<commit_after>
VERSION = '0.18.2.7'
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

from django.apps import apps
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.mail import EmailMultiAlternatives
from django.template import Template, Context
from django.utils.module_loading import import_string

def process_context(context):
    processor_path = getattr(settings, 'TEMPLATED_EMAIL_CONTEXT_PROCESSOR', None)

    if processor_path is None:
        return context

    processor = import_string(processor_path)
    process_data = {}
    if callable(processor):
        process_data = processor()

    return context.update(process_data)

def render(email_template, context):
    html_template_string = email_template.html_prerendered()
    html_template = Template(html_template_string)
    html_body = html_template.render(Context(context))

    text_template_string = email_template.get_plain_text()
    text_template = Template(text_template_string)
    plain_text = text_template.render(Context(context))

    return html_body, plain_text

def get_default_from_email():
        default_from_email = getattr(settings, 'DEFAULT_FROM_EMAIL', None)

        if default_from_email is None:
            raise ImproperlyConfigured(
                'DEFAULT_FROM_EMAIL setting required to send mail without explicit from_email'
            )

        return default_from_email

def send_templated_mail(template_name, to, context=None, subject=None, from_email=None, cc=None, bcc=None, connection=None, fail_silently=False):  # noqa
    if context is None:
        context = {}

    context = process_context(context)

    EmailTemplate = apps.get_model(app_label='templated_mail', model_name='EmailTemplate')
    email_template = EmailTemplate.objects.get(template_name=template_name)
    html_body, plain_text = render(email_template, context)

    if from_email is None:
        from_email = email_template.default_from

    if from_email == '':
        from_email = get_default_from_email()

    if not isinstance(to, list):
        to = [to]

    if subject is None:
        subject = email_template.subject

    if bcc is None:
        bcc = []

    if hasattr(settings, 'TEMPLATED_EMAIL_BCC'):
        bcc.append(settings.TEMPLATED_EMAIL_BCC)

    msg = EmailMultiAlternatives(subject, plain_text, from_email, to, bcc=bcc, connection=connection, cc=cc)
    msg.attach_alternative(html_body, "text/html")
    return msg.send(fail_silently=fail_silently)
<commit_msg>Fix return bug.<commit_after>from __future__ import absolute_import, unicode_literals

from django.apps import apps
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.mail import EmailMultiAlternatives
from django.template import Template, Context
from django.utils.module_loading import import_string

def process_context(context):
    processor_path = getattr(settings, 'TEMPLATED_EMAIL_CONTEXT_PROCESSOR', None)

    if processor_path is None:
        return context

    processor = import_string(processor_path)
    process_data = {}
    if callable(processor):
        process_data = processor()

    context.update(process_data)

    return context

def render(email_template, context):
    html_template_string = email_template.html_prerendered()
    html_template = Template(html_template_string)
    html_body = html_template.render(Context(context))

    text_template_string = email_template.get_plain_text()
    text_template = Template(text_template_string)
    plain_text = text_template.render(Context(context))

    return html_body, plain_text

def get_default_from_email():
        default_from_email = getattr(settings, 'DEFAULT_FROM_EMAIL', None)

        if default_from_email is None:
            raise ImproperlyConfigured(
                'DEFAULT_FROM_EMAIL setting required to send mail without explicit from_email'
            )

        return default_from_email

def send_templated_mail(template_name, to, context=None, subject=None, from_email=None, cc=None, bcc=None, connection=None, fail_silently=False):  # noqa
    if context is None:
        context = {}

    context = process_context(context)

    EmailTemplate = apps.get_model(app_label='templated_mail', model_name='EmailTemplate')
    email_template = EmailTemplate.objects.get(template_name=template_name)
    html_body, plain_text = render(email_template, context)

    if from_email is None:
        from_email = email_template.default_from

    if from_email == '':
        from_email = get_default_from_email()

    if not isinstance(to, list):
        to = [to]

    if subject is None:
        subject = email_template.subject

    if bcc is None:
        bcc = []

    if hasattr(settings, 'TEMPLATED_EMAIL_BCC'):
        bcc.append(settings.TEMPLATED_EMAIL_BCC)

    msg = EmailMultiAlternatives(subject, plain_text, from_email, to, bcc=bcc, connection=connection, cc=cc)
    msg.attach_alternative(html_body, "text/html")
    return msg.send(fail_silently=fail_silently)
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from builtins import object, str
from past.builtins import basestring

import re

trivial_value_re = re.compile(r'\$\{manifest\.[a-zA-Z0-9\$\.]+\}\Z')

def value_is_trivial(value):
	if isinstance(value, bool):
		return True

	if value is None or not isinstance(value, (str, basestring)):
		return False

	value = str(value)

	if value[0] == '(' and value[-1] == ')':
		value = value[1:-1]

	if value == 'true' or value == 'false' or value == 'null':
		return True

	if trivial_value_re.match(value):
		return True

	try:
		float(value)
		return True
	except:
		pass
	if value[0] == '"' and value[-1] == '"':
		if value.count('"') == value.count('\\"') + 2:
			return True
	#print "?trivial", value
	return False

def to_string(value):
	if isinstance(value, (str, basestring)):
		return value
	elif isinstance(value, bool):
		return 'true' if value else 'false'
	elif isinstance(value, object):
		return value
	else:
		return str(value)

def handle_property_path(t):
	return '${%s}' %t

class DocumentationString(object):
	def __init__(self, text):
		self.text = text

class Entity(object):
	def __init__(self):
		self.doc = None

class Component(Entity):
	def __init__(self, name, children):
		super(Component, self).__init__()
		self.name = name
		self.children = children

class Property(Entity):
	def __init__(self, type, properties):
		super(Property, self).__init__()
		self.lazy = type == 'lazy'
		self.const = type == 'const'
		self.type = type
		self.properties = properties

class Const(Entity):
	def __init__(self, type, name, value):
		super(Const, self).__init__()
		self.type, self.name, self.value = type, name, value

class AliasProperty(Entity):
	def __init__(self, name, target):
		super(AliasProperty, self).__init__()
		self.name = name
		self.target = target

class EnumProperty(Entity):
	def __init__(self, name, values, default):
		super(EnumProperty, self).__init__()
		self.name = name
		self.values = values
		self.default = default

class Method(Entity):
	def __init__(self, name, args, code, event, async_):
		super(Method, self).__init__()
		self.name = name
		self.args = args
		self.code = code
		self.event = event
		self.async_ = async_

class IdAssignment(Entity):
	def __init__(self, name):
		super(IdAssignment, self).__init__()
		self.name = name

class Assignment(Entity):
	re_name = re.compile('<property-name>')
	re_scale_name = re.compile('<scale-property-name>')

	def __init__(self, target, value):
		super(Assignment, self).__init__()
		self.target = target

		dot = target.rfind('.')
		property_name = target[dot + 1:] if dot >= 0 else target
		if property_name == 'x':
			property_name = 'width'
		elif property_name == 'y':
			property_name = 'height'

		value = self.replace(Assignment.re_name, property_name, value)
		property_name = 'virtualScale'
		self.value = self.replace(Assignment.re_scale_name, property_name, value)

	def replace(self, rex, property_name, value):
		if isinstance(value, (str, basestring)):
			return rex.sub(property_name, value)
		else:
			return to_string(value)

	def is_trivial(self):
		return value_is_trivial(self.value)

class AssignmentScope(Entity):
	def __init__(self, target, values):
		super(AssignmentScope, self).__init__()
		self.target = target
		self.values = values

class Behavior(Entity):
	def __init__(self, target, animation):
		super(Behavior, self).__init__()
		self.target = target
		self.animation = animation

class Signal(Entity):
	def __init__(self, name):
		super(Signal, self).__init__()
		self.name = name

class ListElement(Entity):
	def __init__(self, data):
		super(ListElement, self).__init__()
		self.data = data
<commit_msg>better fix for to_string (python3.x int is object too)<commit_after>from __future__ import unicode_literals
from builtins import object, str
from past.builtins import basestring

import re

trivial_value_re = re.compile(r'\$\{manifest\.[a-zA-Z0-9\$\.]+\}\Z')

def value_is_trivial(value):
	if isinstance(value, bool):
		return True

	if value is None or not isinstance(value, (str, basestring)):
		return False

	value = str(value)

	if value[0] == '(' and value[-1] == ')':
		value = value[1:-1]

	if value == 'true' or value == 'false' or value == 'null':
		return True

	if trivial_value_re.match(value):
		return True

	try:
		float(value)
		return True
	except:
		pass
	if value[0] == '"' and value[-1] == '"':
		if value.count('"') == value.count('\\"') + 2:
			return True
	#print "?trivial", value
	return False

class Entity(object):
	def __init__(self):
		self.doc = None

def to_string(value):
	if isinstance(value, (str, basestring)):
		return value
	elif isinstance(value, bool):
		return 'true' if value else 'false'
	elif isinstance(value, Entity):
		return value
	else:
		return str(value)

def handle_property_path(t):
	return '${%s}' %t

class DocumentationString(object):
	def __init__(self, text):
		self.text = text

class Component(Entity):
	def __init__(self, name, children):
		super(Component, self).__init__()
		self.name = name
		self.children = children

class Property(Entity):
	def __init__(self, type, properties):
		super(Property, self).__init__()
		self.lazy = type == 'lazy'
		self.const = type == 'const'
		self.type = type
		self.properties = properties

class Const(Entity):
	def __init__(self, type, name, value):
		super(Const, self).__init__()
		self.type, self.name, self.value = type, name, value

class AliasProperty(Entity):
	def __init__(self, name, target):
		super(AliasProperty, self).__init__()
		self.name = name
		self.target = target

class EnumProperty(Entity):
	def __init__(self, name, values, default):
		super(EnumProperty, self).__init__()
		self.name = name
		self.values = values
		self.default = default

class Method(Entity):
	def __init__(self, name, args, code, event, async_):
		super(Method, self).__init__()
		self.name = name
		self.args = args
		self.code = code
		self.event = event
		self.async_ = async_

class IdAssignment(Entity):
	def __init__(self, name):
		super(IdAssignment, self).__init__()
		self.name = name

class Assignment(Entity):
	re_name = re.compile('<property-name>')
	re_scale_name = re.compile('<scale-property-name>')

	def __init__(self, target, value):
		super(Assignment, self).__init__()
		self.target = target

		dot = target.rfind('.')
		property_name = target[dot + 1:] if dot >= 0 else target
		if property_name == 'x':
			property_name = 'width'
		elif property_name == 'y':
			property_name = 'height'

		value = self.replace(Assignment.re_name, property_name, value)
		property_name = 'virtualScale'
		self.value = self.replace(Assignment.re_scale_name, property_name, value)

	def replace(self, rex, property_name, value):
		if isinstance(value, (str, basestring)):
			return rex.sub(property_name, value)
		else:
			return to_string(value)

	def is_trivial(self):
		return value_is_trivial(self.value)

class AssignmentScope(Entity):
	def __init__(self, target, values):
		super(AssignmentScope, self).__init__()
		self.target = target
		self.values = values

class Behavior(Entity):
	def __init__(self, target, animation):
		super(Behavior, self).__init__()
		self.target = target
		self.animation = animation

class Signal(Entity):
	def __init__(self, name):
		super(Signal, self).__init__()
		self.name = name

class ListElement(Entity):
	def __init__(self, data):
		super(ListElement, self).__init__()
		self.data = data
<|endoftext|>


<commit_before>from django.test import TestCase
from django.utils import timezone
from freezegun import freeze_time
import datetime

from core.tests.mommy_utils import make_recipe
from cla_eventlog.models import Log
from complaints.models import Complaint
from legalaid.models import Case, EODDetails, EligibilityCheck, PersonalDetails
from diagnosis.models import DiagnosisTraversal
from cla_auditlog.models import AuditLog
from cla_eventlog.management.commands.find_and_delete_old_cases import Command

def _make_datetime(year=None, month=None, day=None, hour=0, minute=0, second=0):
    today = datetime.date.today()
    year = year if year else today.year
    month = month if month else today.month
    day = day if day else today.day
    dt = datetime.datetime(year, month, day, hour, minute, second)
    return timezone.make_aware(dt, timezone.get_current_timezone())

class FindAndDeleteOldCases(TestCase):
    def setUp(self):
        super(FindAndDeleteOldCases, self).setUp()
        self.command = Command()

    def create_case(self, current_time, case_type="legalaid.case"):
        freezer = freeze_time(current_time)
        freezer.start()
        case = make_recipe(case_type)
        freezer.stop()
        return case

    def create_event_log_for_case(self, case, code, created):
        make_recipe("cla_eventlog.log", case=case, code=code, created=created)

    def delete_old_cases(self, current_time):
        freezer = freeze_time(current_time)
        freezer.start()
        self.command.execute("delete")
        freezer.stop()

    def find_old_cases(self, current_time):
        freezer = freeze_time(current_time)
        freezer.start()
        # Using handle method because command execute method can only return a string i.e it can't return a queryset
        cases = self.command.handle()
        freezer.stop()
        return cases

    def test_old_case_with_recent_event_logs(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        case = self.create_case(date)

        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2014, month=5, day=30, hour=9))
        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2018, month=4, day=27, hour=9))
        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2020, month=4, day=27, hour=9))

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 0)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 1)
        self.assertEqual(Log.objects.count(), 3)

    def test_old_case_with_recent_complaint(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        case = self.create_case(date)

        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2014, month=5, day=30, hour=9))
        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2018, month=4, day=27, hour=9))

        complaint_date = _make_datetime(year=2020, month=4, day=27, hour=9)
        eod = make_recipe("legalaid.eod_details", case=case)
        log = make_recipe("cla_auditlog.audit_log")
        make_recipe("complaints.complaint", eod=eod, audit_log=[log], created=complaint_date)
        self.create_event_log_for_case(case, "COMPLAINT_CREATED", complaint_date)

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 0)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 1)
        self.assertEqual(Complaint.objects.count(), 1)
        self.assertEqual(Log.objects.count(), 3)
        self.assertEqual(EODDetails.objects.count(), 1)

    def test_old_case_with_recent_means_test_change(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        case = self.create_case(date, "legalaid.eligible_case")

        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2014, month=5, day=30, hour=9))
        self.create_event_log_for_case(case, "MT_CHANGED", _make_datetime(year=2020, month=4, day=27, hour=9))

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 0)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 1)
        self.assertEqual(Log.objects.count(), 2)
        self.assertEqual(EligibilityCheck.objects.count(), 1)

    def test_old_case_with_two_logs_with_same_date_of_creation(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        case = self.create_case(date, "legalaid.eligible_case")

        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2020, month=4, day=27, hour=9))
        self.create_event_log_for_case(case, "MT_CHANGED", _make_datetime(year=2020, month=4, day=27, hour=9))

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 0)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 1)
        self.assertEqual(Log.objects.count(), 2)
        self.assertEqual(EligibilityCheck.objects.count(), 1)

    def test_relatively_new_case_with_no_event_logs(self):
        date = _make_datetime(year=2020, month=4, day=27, hour=9)
        self.create_case(date)

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 0)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 1)
        self.assertEqual(Log.objects.count(), 0)

    def test_old_case_with_no_event_logs(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        self.create_case(date)
        self.create_case(date)
        self.create_case(date)

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)

        # Only need to check for this once as every test case uses this method via calling self.find_old_cases
        with self.assertNumQueries(1):
            oldCasesFound = self.find_old_cases(dt)
            oldCasesFound.count()

        self.assertEqual(oldCasesFound.count(), 3)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 0)
        self.assertEqual(Log.objects.count(), 0)
        self.assertEqual(AuditLog.objects.count(), 0)
        self.assertEqual(Complaint.objects.count(), 0)
        self.assertEqual(EODDetails.objects.count(), 0)
        self.assertEqual(EligibilityCheck.objects.count(), 0)
        self.assertEqual(DiagnosisTraversal.objects.count(), 0)
        self.assertEqual(PersonalDetails.objects.count(), 0)

    def test_old_case_with_old_event_logs(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        case = self.create_case(date)
        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2014, month=5, day=30, hour=9))
        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2018, month=4, day=27, hour=9))

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 1)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 0)
        self.assertEqual(Log.objects.count(), 0)
        self.assertEqual(AuditLog.objects.count(), 0)
        self.assertEqual(Complaint.objects.count(), 0)
        self.assertEqual(EODDetails.objects.count(), 0)
        self.assertEqual(EligibilityCheck.objects.count(), 0)
        self.assertEqual(DiagnosisTraversal.objects.count(), 0)
        self.assertEqual(PersonalDetails.objects.count(), 0)

    def test_old_case_with_old_means_test_changed_is_deleted(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        case = self.create_case(date, "legalaid.eligible_case")
        self.create_event_log_for_case(case, "MT_CHANGED", _make_datetime(year=2014, month=5, day=30, hour=9))

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 1)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 0)
        self.assertEqual(Log.objects.count(), 0)
        self.assertEqual(AuditLog.objects.count(), 0)
        self.assertEqual(Complaint.objects.count(), 0)
        self.assertEqual(EODDetails.objects.count(), 0)
        self.assertEqual(EligibilityCheck.objects.count(), 0)
        self.assertEqual(DiagnosisTraversal.objects.count(), 0)
        self.assertEqual(PersonalDetails.objects.count(), 0)
<commit_msg>Refactor test names and add test for new and old log<commit_after>from django.test import TestCase
from django.utils import timezone
from freezegun import freeze_time
import datetime

from core.tests.mommy_utils import make_recipe
from cla_eventlog.models import Log
from complaints.models import Complaint
from legalaid.models import Case, EODDetails, EligibilityCheck, PersonalDetails
from diagnosis.models import DiagnosisTraversal
from cla_auditlog.models import AuditLog
from cla_eventlog.management.commands.find_and_delete_old_cases import Command

def _make_datetime(year=None, month=None, day=None, hour=0, minute=0, second=0):
    today = datetime.date.today()
    year = year if year else today.year
    month = month if month else today.month
    day = day if day else today.day
    dt = datetime.datetime(year, month, day, hour, minute, second)
    return timezone.make_aware(dt, timezone.get_current_timezone())

class FindAndDeleteOldCases(TestCase):
    def setUp(self):
        super(FindAndDeleteOldCases, self).setUp()
        self.command = Command()

    def create_case(self, current_time, case_type="legalaid.case"):
        freezer = freeze_time(current_time)
        freezer.start()
        case = make_recipe(case_type)
        freezer.stop()
        return case

    def create_event_log_for_case(self, case, code, created):
        make_recipe("cla_eventlog.log", case=case, code=code, created=created)

    def delete_old_cases(self, current_time):
        freezer = freeze_time(current_time)
        freezer.start()
        self.command.execute("delete")
        freezer.stop()

    def find_old_cases(self, current_time):
        freezer = freeze_time(current_time)
        freezer.start()
        # Using handle method because command execute method can only return a string i.e it can't return a queryset
        cases = self.command.handle()
        freezer.stop()
        return cases

    def test_old_case_with_recent_event_logs_is_not_deleted(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        case = self.create_case(date)

        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2014, month=5, day=30, hour=9))
        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2018, month=4, day=27, hour=9))
        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2020, month=4, day=27, hour=9))

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 0)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 1)
        self.assertEqual(Log.objects.count(), 3)

    def test_old_case_with_recent_complaint_is_not_deleted(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        case = self.create_case(date)

        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2014, month=5, day=30, hour=9))
        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2018, month=4, day=27, hour=9))

        complaint_date = _make_datetime(year=2020, month=4, day=27, hour=9)
        eod = make_recipe("legalaid.eod_details", case=case)
        log = make_recipe("cla_auditlog.audit_log")
        make_recipe("complaints.complaint", eod=eod, audit_log=[log], created=complaint_date)
        self.create_event_log_for_case(case, "COMPLAINT_CREATED", complaint_date)

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 0)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 1)
        self.assertEqual(Complaint.objects.count(), 1)
        self.assertEqual(Log.objects.count(), 3)
        self.assertEqual(EODDetails.objects.count(), 1)

    def test_old_case_with_recent_means_test_change_is_not_deleted(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        case = self.create_case(date, "legalaid.eligible_case")

        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2014, month=5, day=30, hour=9))
        self.create_event_log_for_case(case, "MT_CHANGED", _make_datetime(year=2020, month=4, day=27, hour=9))

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 0)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 1)
        self.assertEqual(Log.objects.count(), 2)
        self.assertEqual(EligibilityCheck.objects.count(), 1)

    def test_old_case_with_two_logs_with_same_date_of_creation_is_not_deleted(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        case = self.create_case(date, "legalaid.eligible_case")

        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2020, month=4, day=27, hour=9))
        self.create_event_log_for_case(case, "MT_CHANGED", _make_datetime(year=2020, month=4, day=27, hour=9))

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 0)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 1)
        self.assertEqual(Log.objects.count(), 2)
        self.assertEqual(EligibilityCheck.objects.count(), 1)

    def test_relatively_new_case_with_no_event_logs_is_not_deleted(self):
        date = _make_datetime(year=2020, month=4, day=27, hour=9)
        self.create_case(date)

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 0)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 1)
        self.assertEqual(Log.objects.count(), 0)

    def test_old_case_with_no_event_logs_is_deleted(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        self.create_case(date)
        self.create_case(date)
        self.create_case(date)

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)

        # Only need to check for this once as every test case uses this method via calling self.find_old_cases
        with self.assertNumQueries(1):
            oldCasesFound = self.find_old_cases(dt)
            oldCasesFound.count()

        self.assertEqual(oldCasesFound.count(), 3)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 0)
        self.assertEqual(Log.objects.count(), 0)
        self.assertEqual(AuditLog.objects.count(), 0)
        self.assertEqual(Complaint.objects.count(), 0)
        self.assertEqual(EODDetails.objects.count(), 0)
        self.assertEqual(EligibilityCheck.objects.count(), 0)
        self.assertEqual(DiagnosisTraversal.objects.count(), 0)
        self.assertEqual(PersonalDetails.objects.count(), 0)

    def test_old_case_with_old_event_logs_is_deleted(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        case = self.create_case(date)
        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2014, month=5, day=30, hour=9))
        self.create_event_log_for_case(case, "CASE_VIEWED", _make_datetime(year=2018, month=4, day=27, hour=9))

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 1)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 0)
        self.assertEqual(Log.objects.count(), 0)
        self.assertEqual(AuditLog.objects.count(), 0)
        self.assertEqual(Complaint.objects.count(), 0)
        self.assertEqual(EODDetails.objects.count(), 0)
        self.assertEqual(EligibilityCheck.objects.count(), 0)
        self.assertEqual(DiagnosisTraversal.objects.count(), 0)
        self.assertEqual(PersonalDetails.objects.count(), 0)

    def test_old_case_with_old_means_test_changed_is_deleted(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        case = self.create_case(date, "legalaid.eligible_case")
        self.create_event_log_for_case(case, "MT_CHANGED", _make_datetime(year=2014, month=5, day=30, hour=9))

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 1)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 0)
        self.assertEqual(Log.objects.count(), 0)
        self.assertEqual(AuditLog.objects.count(), 0)
        self.assertEqual(Complaint.objects.count(), 0)
        self.assertEqual(EODDetails.objects.count(), 0)
        self.assertEqual(EligibilityCheck.objects.count(), 0)
        self.assertEqual(DiagnosisTraversal.objects.count(), 0)
        self.assertEqual(PersonalDetails.objects.count(), 0)

    def test_old_and_new_cases_are_deleted_correctly(self):
        date = _make_datetime(year=2014, month=4, day=27, hour=9)
        case_0 = self.create_case(date, "legalaid.eligible_case")
        case_1 = self.create_case(date, "legalaid.eligible_case")
        case_2 = self.create_case(date, "legalaid.eligible_case")
        self.create_event_log_for_case(case_0, "MT_CHANGED", _make_datetime(year=2014, month=5, day=30, hour=9))
        self.create_event_log_for_case(case_1, "MT_CHANGED", _make_datetime(year=2014, month=5, day=30, hour=9))
        self.create_event_log_for_case(case_2, "MT_CHANGED", _make_datetime(year=2020, month=4, day=27, hour=9))

        audit_log_1 = make_recipe("cla_auditlog.audit_log")
        audit_log_2 = make_recipe("cla_auditlog.audit_log")
        audit_log_3 = make_recipe("cla_auditlog.audit_log")
        eod_0 = make_recipe("legalaid.eod_details", case=case_0)
        eod_1 = make_recipe("legalaid.eod_details", case=case_1)
        eod_2 = make_recipe("legalaid.eod_details", case=case_2)
        make_recipe("complaints.complaint", eod=eod_0, audit_log=[audit_log_1])
        make_recipe("complaints.complaint", eod=eod_1, audit_log=[audit_log_2])
        make_recipe("complaints.complaint", eod=eod_2, audit_log=[audit_log_3])

        dt = _make_datetime(year=2021, month=1, day=1, hour=9)
        oldCasesFound = self.find_old_cases(dt)
        self.assertEqual(oldCasesFound.count(), 2)

        self.delete_old_cases(dt)
        self.assertEqual(Case.objects.count(), 1)
        self.assertEqual(Log.objects.count(), 1)
        self.assertEqual(AuditLog.objects.count(), 1)
        self.assertEqual(Complaint.objects.count(), 1)
        self.assertEqual(EODDetails.objects.count(), 1)
        self.assertEqual(EligibilityCheck.objects.count(), 1)
        self.assertEqual(DiagnosisTraversal.objects.count(), 1)
        self.assertEqual(PersonalDetails.objects.count(), 1)
<|endoftext|>


<commit_before>import os
import re
import logging

from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from django.contrib.auth import get_user_model

from rules.models import YaraRule
from core.services import parse_rule_submission

User = get_user_model()

# Configure Logging
logging.basicConfig(level=logging.INFO)

class Command(BaseCommand):
    help = 'Ingest yara rules based on specified Yara import file'

    def add_arguments(self, parser):

        # Positional argument
        parser.add_argument('rule_masters', nargs='+')

        # Additional required arguments
        parser.add_argument('--user', required=True)
        parser.add_argument('--group', required=True)

        # Optional arguments
        parser.add_argument('--source',
                            default='')

        parser.add_argument('--category',
                            default='')

    def handle(self, *args, **options):

        username = options['user']
        groupname = options['group']
        rules_source = options['source']
        rules_category = options['category']

        try:
            user = User.objects.get(username=username)
        except ObjectDoesNotExist:
            user = None
            logging.error('[!] Specified user does not exist')

        try:
            group = Group.objects.get(name=groupname)
        except ObjectDoesNotExist:
            group = None
            logging.error('[!] Specified group does not exist')

        if user and group:
            for yara_master in options['rule_masters']:

                if os.path.isfile(yara_master):

                    # Change working directory to account for relative imports
                    os.chdir(os.path.dirname(os.path.abspath(yara_master)))

                    with open(yara_master, 'r') as file_object:
                        master_file_contents = file_object.read()

                    include_pattern = r'\ninclude \"([^\"]+)\"'
                    import_files = re.findall(include_pattern, master_file_contents)

                    for file_path in import_files:
                        with open(file_path, 'rb') as raw_content:
                            content_results = parse_rule_submission(raw_content)

                        # Inspect the results from content submission
                        parsed_rules = content_results['parsed_rules']
                        parsing_error = content_results['parser_error']

                        # Identify any parsing errors that occur
                        if parsing_error:
                            message = '[!] Parsing Error: {}'.format(parsing_error)
                            logging.error(message)
                        else:
                            # Save successfully parsed rules
                            save_results = YaraRule.objects.process_parsed_rules(parsed_rules,
                                                                                rules_source,
                                                                                rules_category,
                                                                                user, group,
                                                                                status=YaraRule.ACTIVE_STATUS)
                            upload_count = save_results['rule_upload_count']
                            collision_count = save_results['rule_collision_count']
                            message = '[✓] Successfully uploaded {} rules and prevented {} rule collisions from {}'.format(upload_count, collision_count, file_path)
                            logging.info(message)

                            for error in save_results['errors']:
                                logging.error(error)

                            for warning in save_results['warnings']:
                                logging.warn(warning)
<commit_msg>Missing import<commit_after>import os
import re
import logging

from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from django.contrib.auth import get_user_model
from django.contrib.auth.models import Group

from rules.models import YaraRule
from core.services import parse_rule_submission

User = get_user_model()

# Configure Logging
logging.basicConfig(level=logging.INFO)

class Command(BaseCommand):
    help = 'Ingest yara rules based on specified Yara import file'

    def add_arguments(self, parser):

        # Positional argument
        parser.add_argument('rule_masters', nargs='+')

        # Additional required arguments
        parser.add_argument('--user', required=True)
        parser.add_argument('--group', required=True)

        # Optional arguments
        parser.add_argument('--source',
                            default='')

        parser.add_argument('--category',
                            default='')

    def handle(self, *args, **options):

        username = options['user']
        groupname = options['group']
        rules_source = options['source']
        rules_category = options['category']

        try:
            user = User.objects.get(username=username)
        except ObjectDoesNotExist:
            user = None
            logging.error('[!] Specified user does not exist')

        try:
            group = Group.objects.get(name=groupname)
        except ObjectDoesNotExist:
            group = None
            logging.error('[!] Specified group does not exist')

        if user and group:
            for yara_master in options['rule_masters']:

                if os.path.isfile(yara_master):

                    # Change working directory to account for relative imports
                    os.chdir(os.path.dirname(os.path.abspath(yara_master)))

                    with open(yara_master, 'r') as file_object:
                        master_file_contents = file_object.read()

                    include_pattern = r'\ninclude \"([^\"]+)\"'
                    import_files = re.findall(include_pattern, master_file_contents)

                    for file_path in import_files:
                        with open(file_path, 'rb') as raw_content:
                            content_results = parse_rule_submission(raw_content)

                        # Inspect the results from content submission
                        parsed_rules = content_results['parsed_rules']
                        parsing_error = content_results['parser_error']

                        # Identify any parsing errors that occur
                        if parsing_error:
                            message = '[!] Parsing Error: {}'.format(parsing_error)
                            logging.error(message)
                        else:
                            # Save successfully parsed rules
                            save_results = YaraRule.objects.process_parsed_rules(parsed_rules,
                                                                                rules_source,
                                                                                rules_category,
                                                                                user, group,
                                                                                status=YaraRule.ACTIVE_STATUS)
                            upload_count = save_results['rule_upload_count']
                            collision_count = save_results['rule_collision_count']
                            message = '[✓] Successfully uploaded {} rules and prevented {} rule collisions from {}'.format(upload_count, collision_count, file_path)
                            logging.info(message)

                            for error in save_results['errors']:
                                logging.error(error)

                            for warning in save_results['warnings']:
                                logging.warn(warning)
<|endoftext|>


<commit_before>import run_harness
import argparse
import os
import shlex

def main(tasks_dir, output_dir, excluding=[], including_only=None, run_sequential_set=False, just_setup=False, stop_on_error=False, extra_args=[], dataset_sizes=None, direct_ref_enabled=None):
    base_params = " ".join([
        "20",
        "--mutable-nodes",
        "--dynamic-nodes",
        "--num-updates 3000",
        "--batch-size 100",
        "--final-params-only",
        "--learning-rate 0.002",
        "--save-params-interval 100",
        "--validation-interval 100",
        "--batch-adjust 16000000"]
        + [shlex.quote(s) for s in extra_args])

    intermediate_propagate_tasks = {3,5}
    alt_sequential_set = {3,5}

    output_types = [
        "category", # [1]='WhereIsActor',
        "category", # [2]='WhereIsObject',
        "category", # [3]='WhereWasObject',
        "category", # [4]='IsDir',
        "category", # [5]='WhoWhatGave',
        "category", # [6]='IsActorThere',
        "category", # [7]='Counting',
        "subset",   # [8]='Listing',
        "category", # [9]='Negation',
        "category", # [10]='Indefinite',
        "category", # [11]='BasicCoreference',
        "category", # [12]='Conjunction',
        "category", # [13]='CompoundCoreference',
        "category", # [14]='Time',
        "category", # [15]='Deduction',
        "category", # [16]='Induction',
        "category", # [17]='PositionalReasoning',
        "category", # [18]='Size',
        "sequence", # [19]='PathFinding',
        "category", # [20]='Motivations'
    ]

    restrict_sizes=[50,100,250,500,1000] if dataset_sizes is None else dataset_sizes

    tasks_and_outputs = list(zip(range(1,21),output_types))
    if run_sequential_set:
        base_params = base_params + " --sequence-aggregate-repr"
        tasks_and_outputs = [tasks_and_outputs[x-1] for x in alt_sequential_set]
        intermediate_propagate_tasks = set()

    if just_setup:
        base_params = base_params + " --just-compile"
        restrict_sizes = [1000]

    direct_ref_options = (True,False) if direct_ref_enabled is None else (direct_ref_enabled,)

    specs = [run_harness.TaskSpec(  "task_{}".format(task_i),
                                    str(rsize) + ("-direct" if direct_ref else ""),
                                    "{} --restrict-dataset {} --stop-at-accuracy {} {} {} {}".format(
                                        output_type,
                                        rsize,
                                        "1.0" if rsize==1000 else "0.95",
                                        "--propagate-intermediate" if task_i in intermediate_propagate_tasks else "",
                                        "" if rsize==1000 else "--stop-at-overfitting 5",
                                        ("--direct-reference" if direct_ref else "")))
                for rsize in reversed(restrict_sizes)
                for direct_ref in direct_ref_options
                for task_i, output_type in tasks_and_outputs]

    specs = [x for x in specs if x.task_name[5:] not in excluding]
    if including_only is not None:
        specs = [x for x in specs if x.task_name[5:] in including_only]
    from pprint import pprint; pprint(specs); return
    run_harness.run(tasks_dir, output_dir, base_params, specs, stop_on_error=stop_on_error, skip_complete=just_setup)

parser = argparse.ArgumentParser(description="Train all bAbI tasks.")
parser.add_argument('tasks_dir', help="Directory with tasks")
parser.add_argument('output_dir', help="Directory to save output to")
parser.add_argument('--excluding', nargs='+', default=[], help="Tasks to exclude")
parser.add_argument('--including-only', nargs='+', default=None, help="Tasks to include, if given, else all tasks")
parser.add_argument('--run-sequential-set', action="store_true", help="Run tasks with sequential output instead, and only run tasks that need it")
parser.add_argument('--just-setup', action="store_true", help="Just setup the tasks, don't actually run them")
parser.add_argument('--stop-on-error', action="store_true", help="Stop if execution hits an error")
parser.add_argument('--dataset-sizes', nargs='+', default=None, type=int, help="Run the model on these sizes of input")
parser.add_argument('--direct-reference', action="store_true", dest="direct_ref_enabled", default=None, help="Only train with direct reference")
parser.add_argument('--no-direct-reference', action="store_false", dest="direct_ref_enabled", default=None, help="Only train without direct reference")

if __name__ == '__main__':
    namespace, extra = parser.parse_known_args()
    args = vars(namespace)
    main(extra_args=extra, **args)

<commit_msg>Remove debug dry-run in do_babi_run.py<commit_after>import run_harness
import argparse
import os
import shlex

def main(tasks_dir, output_dir, excluding=[], including_only=None, run_sequential_set=False, just_setup=False, stop_on_error=False, extra_args=[], dataset_sizes=None, direct_ref_enabled=None):
    base_params = " ".join([
        "20",
        "--mutable-nodes",
        "--dynamic-nodes",
        "--num-updates 3000",
        "--batch-size 100",
        "--final-params-only",
        "--learning-rate 0.002",
        "--save-params-interval 100",
        "--validation-interval 100",
        "--batch-adjust 16000000"]
        + [shlex.quote(s) for s in extra_args])

    intermediate_propagate_tasks = {3,5}
    alt_sequential_set = {3,5}

    output_types = [
        "category", # [1]='WhereIsActor',
        "category", # [2]='WhereIsObject',
        "category", # [3]='WhereWasObject',
        "category", # [4]='IsDir',
        "category", # [5]='WhoWhatGave',
        "category", # [6]='IsActorThere',
        "category", # [7]='Counting',
        "subset",   # [8]='Listing',
        "category", # [9]='Negation',
        "category", # [10]='Indefinite',
        "category", # [11]='BasicCoreference',
        "category", # [12]='Conjunction',
        "category", # [13]='CompoundCoreference',
        "category", # [14]='Time',
        "category", # [15]='Deduction',
        "category", # [16]='Induction',
        "category", # [17]='PositionalReasoning',
        "category", # [18]='Size',
        "sequence", # [19]='PathFinding',
        "category", # [20]='Motivations'
    ]

    restrict_sizes=[50,100,250,500,1000] if dataset_sizes is None else dataset_sizes

    tasks_and_outputs = list(zip(range(1,21),output_types))
    if run_sequential_set:
        base_params = base_params + " --sequence-aggregate-repr"
        tasks_and_outputs = [tasks_and_outputs[x-1] for x in alt_sequential_set]
        intermediate_propagate_tasks = set()

    if just_setup:
        base_params = base_params + " --just-compile"
        restrict_sizes = [1000]

    direct_ref_options = (True,False) if direct_ref_enabled is None else (direct_ref_enabled,)

    specs = [run_harness.TaskSpec(  "task_{}".format(task_i),
                                    str(rsize) + ("-direct" if direct_ref else ""),
                                    "{} --restrict-dataset {} --stop-at-accuracy {} {} {} {}".format(
                                        output_type,
                                        rsize,
                                        "1.0" if rsize==1000 else "0.95",
                                        "--propagate-intermediate" if task_i in intermediate_propagate_tasks else "",
                                        "" if rsize==1000 else "--stop-at-overfitting 5",
                                        ("--direct-reference" if direct_ref else "")))
                for rsize in reversed(restrict_sizes)
                for direct_ref in direct_ref_options
                for task_i, output_type in tasks_and_outputs]

    specs = [x for x in specs if x.task_name[5:] not in excluding]
    if including_only is not None:
        specs = [x for x in specs if x.task_name[5:] in including_only]
    # from pprint import pprint; pprint(specs); return
    run_harness.run(tasks_dir, output_dir, base_params, specs, stop_on_error=stop_on_error, skip_complete=just_setup)

parser = argparse.ArgumentParser(description="Train all bAbI tasks.")
parser.add_argument('tasks_dir', help="Directory with tasks")
parser.add_argument('output_dir', help="Directory to save output to")
parser.add_argument('--excluding', nargs='+', default=[], help="Tasks to exclude")
parser.add_argument('--including-only', nargs='+', default=None, help="Tasks to include, if given, else all tasks")
parser.add_argument('--run-sequential-set', action="store_true", help="Run tasks with sequential output instead, and only run tasks that need it")
parser.add_argument('--just-setup', action="store_true", help="Just setup the tasks, don't actually run them")
parser.add_argument('--stop-on-error', action="store_true", help="Stop if execution hits an error")
parser.add_argument('--dataset-sizes', nargs='+', default=None, type=int, help="Run the model on these sizes of input")
parser.add_argument('--direct-reference', action="store_true", dest="direct_ref_enabled", default=None, help="Only train with direct reference")
parser.add_argument('--no-direct-reference', action="store_false", dest="direct_ref_enabled", default=None, help="Only train without direct reference")

if __name__ == '__main__':
    namespace, extra = parser.parse_known_args()
    args = vars(namespace)
    main(extra_args=extra, **args)

<|endoftext|>