<commit_before>from django.views.generic.list import ListView
from django.views.generic.detail import DetailView
from django.db.models import Q
from django.http import HttpResponse
from django.core.exceptions import PermissionDenied
import csv
import re

from signbank.dictionary.models import *
from signbank.dictionary.forms import *
from signbank.feedback.models import *
from signbank.video.forms import VideoUploadForGlossForm
from tagging.models import Tag, TaggedItem

class GlossListView(ListView):

    model = Gloss
    template_name = 'dictionary/admin_gloss_list.html'
    paginate_by = 1000

    def get_context_data(self, **kwargs):
        # Call the base implementation first to get a context
        context = super(GlossListView, self).get_context_data(**kwargs)
        # Add in a QuerySet of all the books
        context['searchform'] = GlossSearchForm(self.request.GET)
        context['glosscount'] = Gloss.objects.all().count()
        context['add_gloss_form'] = GlossCreateForm()
        context['ADMIN_RESULT_FIELDS'] = settings.ADMIN_RESULT_FIELDS
        return context

    def get_paginate_by(self, queryset):
        """
        Paginate by specified value in querystring, or use default class property value.
        """
        return self.request.GET.get('paginate_by', self.paginate_by)

    def render_to_response(self, context):
        # Look for a 'format=json' GET argument
        if self.request.GET.get('format') == 'CSV':
            return self.render_to_csv_response(context)
        else:
            return super(GlossListView, self).render_to_response(context)

    # noinspection PyInterpreter,PyInterpreter
    def render_to_csv_response(self, context):

        if not self.request.user.has_perm('dictionary.export_csv'):
            raise PermissionDenied

        # Create the HttpResponse object with the appropriate CSV header.
        response = HttpResponse(content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename="dictionary-export.csv"'

#        fields = [f.name for f in Gloss._meta.fields]
        #We want to manually set which fields to export here

        fieldnames = ['idgloss', 'annotation_idgloss', 'annotation_idgloss_en', 'useInstr', 'sense', 'morph', 'StemSN', 'compound', 'rmrks', 'handedness', 'domhndsh', 'subhndsh', 'locprim', 'relatArtic', 'absOriPalm', 'absOriFing', 'relOriMov', 'relOriLoc', 'handCh', 'repeat', 'altern', 'movSh', 'movDir', 'movMan', 'contType', 'phonOth', 'mouthG', 'mouthing', 'phonetVar', 'iconImg', 'namEnt', 'tokNo', 'tokNoSgnr', 'tokNoA', 'tokNoV', 'tokNoR', 'tokNoGe', 'tokNoGr', 'tokNoO', 'tokNoSgnrA', 'tokNoSgnrV', 'tokNoSgnrR', 'tokNoSgnrGe', 'tokNoSgnrGr', 'tokNoSgnrO', 'inWeb', 'isNew'];
        fields = [Gloss._meta.get_field(fieldname) for fieldname in fieldnames]

        writer = csv.writer(response)
        header = ['Signbank ID'] + [f.verbose_name for f in fields]

        for extra_column in ['Languages','Dialects','Keywords','Relations to other signs','Relations to foreign signs',]:
            header.append(extra_column);

        writer.writerow(header)

        for gloss in self.get_queryset():
            row = [str(gloss.pk)]
            for f in fields:

                #Try the value of the choicelist
                try:
                    row.append(getattr(gloss, 'get_'+f.name+'_display')())

                #If it's not there, try the raw value
                except AttributeError:
                    value = getattr(gloss,f.name)
                    row.append(value)

            # get languages
            languages = [language.name for language in gloss.language.all()]
            row.append(", ".join(languages));

            # get dialects
            dialects = [dialect.name for dialect in gloss.dialect.all()]
            row.append(", ".join(dialects));

            # get translations
            trans = [t.translation.text for t in gloss.translation_set.all()]
            row.append(", ".join(trans))

            # get relations to other signs
            relations = [relation.target.idgloss for relation in Relation.objects.filter(source=gloss)]
            row.append(", ".join(relations))

            # get relations to foreign signs
            relations = [relation.other_lang_gloss for relation in RelationToForeignSign.objects.filter(gloss=gloss)]
            row.append(", ".join(relations))

            #Make it safe for weird chars
            safe_row = [];
            for column in row:
                try:
                    safe_row.append(column.encode('utf-8'))
                except AttributeError:
                    safe_row.append(None);

            writer.writerow(safe_row)

        return response

    def get_queryset(self):

        # get query terms from self.request
        qs = Gloss.objects.all()

        #print "QS:", len(qs)

        get = self.request.GET

        if get.has_key('search') and get['search'] != '':
            val = get['search']
            query = Q(idgloss__istartswith=val) | \
                    Q(annotation_idgloss__istartswith=val)

            if re.match('^\d+$', val):
                query = query | Q(sn__exact=val)

            qs = qs.filter(query)
            #print "A: ", len(qs)

        if get.has_key('keyword') and get['keyword'] != '':
            val = get['keyword']
            qs = qs.filter(translation__translation__text__istartswith=val)

        if get.has_key('inWeb') and get['inWeb'] != 'unspecified':
            val = get['inWeb'] == 'yes'
            qs = qs.filter(inWeb__exact=val)
            #print "B :", len(qs)

        if get.has_key('hasvideo') and get['hasvideo'] != 'unspecified':
            val = get['hasvideo'] == 'no'

            qs = qs.filter(glossvideo__isnull=val)

        if get.has_key('defspublished') and get['defspublished'] != 'unspecified':
            val = get['defspublished'] == 'yes'

            qs = qs.filter(definition__published=val)

        ## phonology field filters
        if get.has_key('domhndsh') and get['domhndsh'] != '':
            val = get['domhndsh']
            qs = qs.filter(domhndsh__exact=val)

            #print "C :", len(qs)

        if get.has_key('subhndsh') and get['subhndsh'] != '':
            val = get['subhndsh']
            qs = qs.filter(subhndsh__exact=val)
            #print "D :", len(qs)

        if get.has_key('final_domhndsh') and get['final_domhndsh'] != '':
            val = get['final_domhndsh']
            qs = qs.filter(final_domhndsh__exact=val)
            #print "E :", len(qs)

        if get.has_key('final_subhndsh') and get['final_subhndsh'] != '':
            val = get['final_subhndsh']
            qs = qs.filter(final_subhndsh__exact=val)
           # print "F :", len(qs)

        if get.has_key('locprim') and get['locprim'] != '':
            val = get['locprim']
            qs = qs.filter(locprim__exact=val)
            #print "G :", len(qs)

        if get.has_key('locsecond') and get['locsecond'] != '':
            val = get['locsecond']
            qs = qs.filter(locsecond__exact=val)

            #print "H :", len(qs)

        if get.has_key('final_loc') and get['final_loc'] != '':
            val = get['final_loc']
            qs = qs.filter(final_loc__exact=val)

        if get.has_key('initial_relative_orientation') and get['initial_relative_orientation'] != '':
            val = get['initial_relative_orientation']
            qs = qs.filter(initial_relative_orientation__exact=val)

        if get.has_key('final_relative_orientation') and get['final_relative_orientation'] != '':
            val = get['final_relative_orientation']
            qs = qs.filter(final_relative_orientation__exact=val)

        if get.has_key('initial_palm_orientation') and get['initial_palm_orientation'] != '':
            val = get['initial_palm_orientation']
            qs = qs.filter(initial_palm_orientation__exact=val)

        if get.has_key('final_palm_orientation') and get['final_palm_orientation'] != '':
            val = get['final_palm_orientation']
            qs = qs.filter(final_palm_orientation__exact=val)

        if get.has_key('initial_secondary_loc') and get['initial_secondary_loc'] != '':
            val = get['initial_secondary_loc']
            qs = qs.filter(initial_secondary_loc__exact=val)

        if get.has_key('final_secondary_loc') and get['final_secondary_loc'] != '':
            val = get['final_secondary_loc']
            qs = qs.filter(final_secondary_loc__exact=val)

           # print "G :", len(qs)
        # end of phonology filters

        if get.has_key('defsearch') and get['defsearch'] != '':

            val = get['defsearch']

            if get.has_key('defrole'):
                role = get['defrole']
            else:
                role = 'all'

            if role == 'all':
                qs = qs.filter(definition__text__icontains=val)
            else:
                qs = qs.filter(definition__text__icontains=val, definition__role__exact=role)

        vals = get.getlist('dialect', [])
        if vals != []:
            qs = qs.filter(dialect__in=vals)

           # print "H :", len(qs)

        vals = get.getlist('language', [])
        if vals != []:
            qs = qs.filter(language__in=vals)

            #print "I :", len(qs)

        if get.has_key('tags') and get['tags'] != '':
            vals = get.getlist('tags')

            tags = []
            for t in vals:
                tags.extend(Tag.objects.filter(name=t))

            # search is an implicit AND so intersection
            tqs = TaggedItem.objects.get_intersection_by_model(Gloss, tags)

            # intersection
            qs = qs & tqs

            #print "J :", len(qs)

        qs = qs.distinct()

        if get.has_key('nottags') and get['nottags'] != '':
            vals = get.getlist('nottags')

           # print "NOT TAGS: ", vals

            tags = []
            for t in vals:
                tags.extend(Tag.objects.filter(name=t))

            # search is an implicit AND so intersection
            tqs = TaggedItem.objects.get_intersection_by_model(Gloss, tags)

           # print "NOT", tags, len(tqs)
            # exclude all of tqs from qs
            qs = [q for q in qs if q not in tqs]

           # print "K :", len(qs)

       # print "Final :", len(qs)
        return qs

class GlossDetailView(DetailView):

    model = Gloss
    context_object_name = 'gloss'

    def get_context_data(self, **kwargs):
        # Call the base implementation first to get a context
        context = super(GlossDetailView, self).get_context_data(**kwargs)
        # Add in a QuerySet of all the books
        context['tagform'] = TagUpdateForm()
        context['videoform'] = VideoUploadForGlossForm()
        context['definitionform'] = DefinitionForm()
        context['relationform'] = RelationForm()
        context['navigation'] = context['gloss'].navigation(True)
        context['interpform'] = InterpreterFeedbackForm()
        context['SIGN_NAVIGATION']  = settings.SIGN_NAVIGATION
        if settings.SIGN_NAVIGATION:
            context['glosscount'] = Gloss.objects.count()
            context['glossposn'] =  Gloss.objects.filter(sn__lt=context['gloss'].sn).count()+1

        #Pass info about which fields we want to see
        gl = context['gloss'];
        labels = gl.field_labels();

	fields = {};

	fields['phonology'] = ['handedness','domhndsh','subhndsh','locprim','relatArtic','absOriPalm','absOriFing',
                  'relOriMov','relOriLoc','handCh','repeat', 'altern', 'movSh','movDir','movMan','contType','phonOth', 'mouthG',
                  'mouthing', 'phonetVar',];

	fields['semantics'] = ['iconImg','namEnt'];

	fields['frequency'] = ['tokNoA','tokNoSgnrA','tokNoV','tokNoSgnrV','tokNoR','tokNoSgnrR','tokNoGe','tokNoSgnrGe',
                               'tokNoGr','tokNoSgnrGr','tokNoO','tokNoSgnrO'];

        for topic in ['phonology','semantics','frequency']:
            context[topic+'_fields'] = [];

            for field in fields[topic]:

                try:
                    value = getattr(gl,'get_'+field+'_display');
                except AttributeError:
                    value = getattr(gl,field);

                if field in ['phonOth','mouthG','mouthing','phonetVar','iconImg']:
                    kind = 'text';
                else:
                    kind = 'list';

                context[topic+'_fields'].append([value,field,labels[field],kind]);

        return context

def gloss_ajax_complete(request, prefix):
    """Return a list of glosses matching the search term
    as a JSON structure suitable for typeahead."""

    query = Q(idgloss__istartswith=prefix) | \
            Q(annotation_idgloss__istartswith=prefix) | \
            Q(sn__startswith=prefix)
    qs = Gloss.objects.filter(query)

    result = []
    for g in qs:
        result.append({'idgloss': g.idgloss, 'annotation_idgloss': g.annotation_idgloss, 'sn': g.sn, 'pk': "%s (%s)" % (g.idgloss, g.pk)})

    return HttpResponse(json.dumps(result), {'content-type': 'application/json'})
<commit_msg>Discovered that booleans and integers were not exported yet<commit_after>from django.views.generic.list import ListView
from django.views.generic.detail import DetailView
from django.db.models import Q
from django.http import HttpResponse
from django.core.exceptions import PermissionDenied
import csv
import re

from signbank.dictionary.models import *
from signbank.dictionary.forms import *
from signbank.feedback.models import *
from signbank.video.forms import VideoUploadForGlossForm
from tagging.models import Tag, TaggedItem

class GlossListView(ListView):

    model = Gloss
    template_name = 'dictionary/admin_gloss_list.html'
    paginate_by = 1000

    def get_context_data(self, **kwargs):
        # Call the base implementation first to get a context
        context = super(GlossListView, self).get_context_data(**kwargs)
        # Add in a QuerySet of all the books
        context['searchform'] = GlossSearchForm(self.request.GET)
        context['glosscount'] = Gloss.objects.all().count()
        context['add_gloss_form'] = GlossCreateForm()
        context['ADMIN_RESULT_FIELDS'] = settings.ADMIN_RESULT_FIELDS
        return context

    def get_paginate_by(self, queryset):
        """
        Paginate by specified value in querystring, or use default class property value.
        """
        return self.request.GET.get('paginate_by', self.paginate_by)

    def render_to_response(self, context):
        # Look for a 'format=json' GET argument
        if self.request.GET.get('format') == 'CSV':
            return self.render_to_csv_response(context)
        else:
            return super(GlossListView, self).render_to_response(context)

    # noinspection PyInterpreter,PyInterpreter
    def render_to_csv_response(self, context):

        if not self.request.user.has_perm('dictionary.export_csv'):
            raise PermissionDenied

        # Create the HttpResponse object with the appropriate CSV header.
        response = HttpResponse(content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename="dictionary-export.csv"'

#        fields = [f.name for f in Gloss._meta.fields]
        #We want to manually set which fields to export here

        fieldnames = ['idgloss', 'annotation_idgloss', 'annotation_idgloss_en', 'useInstr', 'sense', 'morph', 'StemSN', 'compound', 'rmrks', 'handedness', 'domhndsh', 'subhndsh', 'locprim', 'relatArtic', 'absOriPalm', 'absOriFing', 'relOriMov', 'relOriLoc', 'handCh', 'repeat', 'altern', 'movSh', 'movDir', 'movMan', 'contType', 'phonOth', 'mouthG', 'mouthing', 'phonetVar', 'iconImg', 'namEnt', 'tokNo', 'tokNoSgnr', 'tokNoA', 'tokNoV', 'tokNoR', 'tokNoGe', 'tokNoGr', 'tokNoO', 'tokNoSgnrA', 'tokNoSgnrV', 'tokNoSgnrR', 'tokNoSgnrGe', 'tokNoSgnrGr', 'tokNoSgnrO', 'inWeb', 'isNew'];
        fields = [Gloss._meta.get_field(fieldname) for fieldname in fieldnames]

        writer = csv.writer(response)
        header = ['Signbank ID'] + [f.verbose_name for f in fields]

        for extra_column in ['Languages','Dialects','Keywords','Relations to other signs','Relations to foreign signs',]:
            header.append(extra_column);

        writer.writerow(header)

        for gloss in self.get_queryset():
            row = [str(gloss.pk)]
            for f in fields:

                #Try the value of the choicelist
                try:
                    row.append(getattr(gloss, 'get_'+f.name+'_display')())

                #If it's not there, try the raw value
                except AttributeError:
                    value = getattr(gloss,f.name)
                    row.append(str(value))

            # get languages
            languages = [language.name for language in gloss.language.all()]
            row.append(", ".join(languages));

            # get dialects
            dialects = [dialect.name for dialect in gloss.dialect.all()]
            row.append(", ".join(dialects));

            # get translations
            trans = [t.translation.text for t in gloss.translation_set.all()]
            row.append(", ".join(trans))

            # get relations to other signs
            relations = [relation.target.idgloss for relation in Relation.objects.filter(source=gloss)]
            row.append(", ".join(relations))

            # get relations to foreign signs
            relations = [relation.other_lang_gloss for relation in RelationToForeignSign.objects.filter(gloss=gloss)]
            row.append(", ".join(relations))

            #Make it safe for weird chars
            safe_row = [];
            for column in row:
                try:
                    safe_row.append(column.encode('utf-8'))
                except AttributeError:
                    safe_row.append(None);

            writer.writerow(safe_row)

        return response

    def get_queryset(self):

        # get query terms from self.request
        qs = Gloss.objects.all()

        #print "QS:", len(qs)

        get = self.request.GET

        if get.has_key('search') and get['search'] != '':
            val = get['search']
            query = Q(idgloss__istartswith=val) | \
                    Q(annotation_idgloss__istartswith=val)

            if re.match('^\d+$', val):
                query = query | Q(sn__exact=val)

            qs = qs.filter(query)
            #print "A: ", len(qs)

        if get.has_key('keyword') and get['keyword'] != '':
            val = get['keyword']
            qs = qs.filter(translation__translation__text__istartswith=val)

        if get.has_key('inWeb') and get['inWeb'] != 'unspecified':
            val = get['inWeb'] == 'yes'
            qs = qs.filter(inWeb__exact=val)
            #print "B :", len(qs)

        if get.has_key('hasvideo') and get['hasvideo'] != 'unspecified':
            val = get['hasvideo'] == 'no'

            qs = qs.filter(glossvideo__isnull=val)

        if get.has_key('defspublished') and get['defspublished'] != 'unspecified':
            val = get['defspublished'] == 'yes'

            qs = qs.filter(definition__published=val)

        ## phonology field filters
        if get.has_key('domhndsh') and get['domhndsh'] != '':
            val = get['domhndsh']
            qs = qs.filter(domhndsh__exact=val)

            #print "C :", len(qs)

        if get.has_key('subhndsh') and get['subhndsh'] != '':
            val = get['subhndsh']
            qs = qs.filter(subhndsh__exact=val)
            #print "D :", len(qs)

        if get.has_key('final_domhndsh') and get['final_domhndsh'] != '':
            val = get['final_domhndsh']
            qs = qs.filter(final_domhndsh__exact=val)
            #print "E :", len(qs)

        if get.has_key('final_subhndsh') and get['final_subhndsh'] != '':
            val = get['final_subhndsh']
            qs = qs.filter(final_subhndsh__exact=val)
           # print "F :", len(qs)

        if get.has_key('locprim') and get['locprim'] != '':
            val = get['locprim']
            qs = qs.filter(locprim__exact=val)
            #print "G :", len(qs)

        if get.has_key('locsecond') and get['locsecond'] != '':
            val = get['locsecond']
            qs = qs.filter(locsecond__exact=val)

            #print "H :", len(qs)

        if get.has_key('final_loc') and get['final_loc'] != '':
            val = get['final_loc']
            qs = qs.filter(final_loc__exact=val)

        if get.has_key('initial_relative_orientation') and get['initial_relative_orientation'] != '':
            val = get['initial_relative_orientation']
            qs = qs.filter(initial_relative_orientation__exact=val)

        if get.has_key('final_relative_orientation') and get['final_relative_orientation'] != '':
            val = get['final_relative_orientation']
            qs = qs.filter(final_relative_orientation__exact=val)

        if get.has_key('initial_palm_orientation') and get['initial_palm_orientation'] != '':
            val = get['initial_palm_orientation']
            qs = qs.filter(initial_palm_orientation__exact=val)

        if get.has_key('final_palm_orientation') and get['final_palm_orientation'] != '':
            val = get['final_palm_orientation']
            qs = qs.filter(final_palm_orientation__exact=val)

        if get.has_key('initial_secondary_loc') and get['initial_secondary_loc'] != '':
            val = get['initial_secondary_loc']
            qs = qs.filter(initial_secondary_loc__exact=val)

        if get.has_key('final_secondary_loc') and get['final_secondary_loc'] != '':
            val = get['final_secondary_loc']
            qs = qs.filter(final_secondary_loc__exact=val)

           # print "G :", len(qs)
        # end of phonology filters

        if get.has_key('defsearch') and get['defsearch'] != '':

            val = get['defsearch']

            if get.has_key('defrole'):
                role = get['defrole']
            else:
                role = 'all'

            if role == 'all':
                qs = qs.filter(definition__text__icontains=val)
            else:
                qs = qs.filter(definition__text__icontains=val, definition__role__exact=role)

        vals = get.getlist('dialect', [])
        if vals != []:
            qs = qs.filter(dialect__in=vals)

           # print "H :", len(qs)

        vals = get.getlist('language', [])
        if vals != []:
            qs = qs.filter(language__in=vals)

            #print "I :", len(qs)

        if get.has_key('tags') and get['tags'] != '':
            vals = get.getlist('tags')

            tags = []
            for t in vals:
                tags.extend(Tag.objects.filter(name=t))

            # search is an implicit AND so intersection
            tqs = TaggedItem.objects.get_intersection_by_model(Gloss, tags)

            # intersection
            qs = qs & tqs

            #print "J :", len(qs)

        qs = qs.distinct()

        if get.has_key('nottags') and get['nottags'] != '':
            vals = get.getlist('nottags')

           # print "NOT TAGS: ", vals

            tags = []
            for t in vals:
                tags.extend(Tag.objects.filter(name=t))

            # search is an implicit AND so intersection
            tqs = TaggedItem.objects.get_intersection_by_model(Gloss, tags)

           # print "NOT", tags, len(tqs)
            # exclude all of tqs from qs
            qs = [q for q in qs if q not in tqs]

           # print "K :", len(qs)

       # print "Final :", len(qs)
        return qs

class GlossDetailView(DetailView):

    model = Gloss
    context_object_name = 'gloss'

    def get_context_data(self, **kwargs):
        # Call the base implementation first to get a context
        context = super(GlossDetailView, self).get_context_data(**kwargs)
        # Add in a QuerySet of all the books
        context['tagform'] = TagUpdateForm()
        context['videoform'] = VideoUploadForGlossForm()
        context['definitionform'] = DefinitionForm()
        context['relationform'] = RelationForm()
        context['navigation'] = context['gloss'].navigation(True)
        context['interpform'] = InterpreterFeedbackForm()
        context['SIGN_NAVIGATION']  = settings.SIGN_NAVIGATION
        if settings.SIGN_NAVIGATION:
            context['glosscount'] = Gloss.objects.count()
            context['glossposn'] =  Gloss.objects.filter(sn__lt=context['gloss'].sn).count()+1

        #Pass info about which fields we want to see
        gl = context['gloss'];
        labels = gl.field_labels();

	fields = {};

	fields['phonology'] = ['handedness','domhndsh','subhndsh','locprim','relatArtic','absOriPalm','absOriFing',
                  'relOriMov','relOriLoc','handCh','repeat', 'altern', 'movSh','movDir','movMan','contType','phonOth', 'mouthG',
                  'mouthing', 'phonetVar',];

	fields['semantics'] = ['iconImg','namEnt'];

	fields['frequency'] = ['tokNoA','tokNoSgnrA','tokNoV','tokNoSgnrV','tokNoR','tokNoSgnrR','tokNoGe','tokNoSgnrGe',
                               'tokNoGr','tokNoSgnrGr','tokNoO','tokNoSgnrO'];

        for topic in ['phonology','semantics','frequency']:
            context[topic+'_fields'] = [];

            for field in fields[topic]:

                try:
                    value = getattr(gl,'get_'+field+'_display');
                except AttributeError:
                    value = getattr(gl,field);

                if field in ['phonOth','mouthG','mouthing','phonetVar','iconImg']:
                    kind = 'text';
                else:
                    kind = 'list';

                context[topic+'_fields'].append([value,field,labels[field],kind]);

        return context

def gloss_ajax_complete(request, prefix):
    """Return a list of glosses matching the search term
    as a JSON structure suitable for typeahead."""

    query = Q(idgloss__istartswith=prefix) | \
            Q(annotation_idgloss__istartswith=prefix) | \
            Q(sn__startswith=prefix)
    qs = Gloss.objects.filter(query)

    result = []
    for g in qs:
        result.append({'idgloss': g.idgloss, 'annotation_idgloss': g.annotation_idgloss, 'sn': g.sn, 'pk': "%s (%s)" % (g.idgloss, g.pk)})

    return HttpResponse(json.dumps(result), {'content-type': 'application/json'})
<|endoftext|>


<commit_before>#!/usr/bin/env python
# sigtools - Collection of Python modules for manipulating function signatures
# Copyright (c) 2013-2015 Yann Kaiser
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import sys

from sigtools import modifiers, specifiers, support, _util, signatures
from sigtools.tests.util import Fixtures, SignatureTests

# bulk of the testing happens in test_merge and test_embed

not_py33 = sys.version_info < (3,3)

def _func(*args, **kwargs):
    raise NotImplementedError

def _free_func(x, y, z):
    raise NotImplementedError

class _cls(object):
    method = _func
_inst = _cls()
_im_type = type(_inst.method)

class ForwardsTest(Fixtures):
    def _test(self, outer, inner, args, kwargs, expected, expected_get):
        outer_f = support.f(outer)
        inner_f = support.f(inner)
        forw = specifiers.forwards_to_function(inner_f, *args, **kwargs)(outer_f)

        self.assertSigsEqual(
            specifiers.signature(forw),
            support.s(expected)
            )

        self.assertSigsEqual(
            specifiers.signature(_util.safe_get(forw, object(), object)),
            support.s(expected_get)
            )

    a = (
        'a, *p, b, **k', 'c, *, d', (), {},
        'a, c, *, b, d', 'c, *, b, d')
    b = (
        'a, *p, b, **k', 'a, c, *, b, d', (1, 'b'), {},
        'a, c, *, b, d', 'c, *, b, d')

    def test_call(self):
        outer = support.f('*args, **kwargs')
        inner = support.f('a, *, b')
        forw = specifiers.forwards_to_function(inner)(outer)
        instance = object()
        forw_get_prox = _util.safe_get(forw, instance, object)
        self.assertEqual(
            forw_get_prox(1, b=2),
            {'args': (instance, 1), 'kwargs': {'b': 2}}
            )

def sig_equal(self, obj, sig_str):
    self.assertSigsEqual(specifiers.signature(obj), support.s(sig_str),
                         conv_first_posarg=True)

class _Coop(object):
    @modifiers.kwoargs('cb')
    def method(self, ca, cb, *cr, **ck):
        raise NotImplementedError

class ForwardsAttributeTests(Fixtures):
    _test = sig_equal

    class _Base(object):
        def __init__(self, decorated=None):
            self.decorated = decorated
            self.coop = _Coop()

        @modifiers.kwoargs('b')
        def inner(self, a, b):
            raise NotImplementedError

        @specifiers.forwards_to(_free_func)
        def ft(self, *args, **kwargs):
            raise NotImplementedError

        @specifiers.forwards_to_method('inner')
        def ftm(self, *args, **kwargs):
            raise NotImplementedError

        @specifiers.forwards_to_ivar('decorated')
        def fti(self, *args, **kwargs):
            raise NotImplementedError

        @specifiers.forwards_to_method('ftm')
        @modifiers.kwoargs('d')
        def ftm2(self, c, d, *args2, **kwargs2):
            raise NotImplementedError

        @modifiers.kwoargs('m')
        def fts(self, l, m):
            raise NotImplementedError

        @modifiers.kwoargs('o')
        def afts(self, n, o):
            raise NotImplementedError

        @specifiers.forwards_to_method('ftm2')
        @modifiers.kwoargs('q')
        def chain_fts(self, p, q, *args, **kwargs):
            raise NotImplementedError

        @specifiers.forwards_to_method('ftm2')
        @modifiers.kwoargs('s')
        def chain_afts(self, r, s, *args, **kwargs):
            raise NotImplementedError

        @specifiers.forwards_to_method('coop.method')
        @modifiers.kwoargs('bc')
        def ccm(self, ba, bb, bc, *args, **kwargs):
            raise NotImplementedError

    @_Base
    @modifiers.kwoargs('b')
    def _base_inst(a, b):
        raise NotImplementedError

    @specifiers.apply_forwards_to_super('afts', 'chain_afts')
    class _Derivate(_Base):
        @specifiers.forwards_to_method('inner')
        def ftm(self, e, *args, **kwoargs):
            raise NotImplementedError

        @specifiers.forwards_to_super()
        def fts(self, s, *args, **kwargs):
            super() # pragma: no cover

        def afts(self, asup, *args, **kwargs):
            raise NotImplementedError

        @specifiers.forwards_to_super()
        def chain_fts(self, u, *args, **kwargs):
            super() # pragma: no cover

        def chain_afts(self, v, *args, **kwargs):
            raise NotImplementedError

    @_Derivate
    @modifiers.kwoargs('y')
    def _sub_inst(x, y):
        raise NotImplementedError

    base_func = _base_inst.ft, 'x, y, z'

    base_method = _base_inst.ftm, 'a, *, b'
    base_method2 = _base_inst.ftm2, 'c, a, *, d, b'

    base_method_cls = _Base.ftm, 'self, *args, **kwargs'

    base_ivar = _base_inst.fti, 'a, *, b'

    base_coop = _base_inst.ccm, 'ba, bb, ca, *cr, bc, cb, **ck'

    sub_method = _sub_inst.ftm, 'e, a, *, b'

    sub_method2 = _sub_inst.ftm2, 'c, e, a, *, d, b'

    sub_ivar = _sub_inst.fti, 'x, *, y'

    def test_fts(self):
        if sys.version_info >= (3,3):
            self._test(self._sub_inst.fts, 's, l, *, m')

    sub_afts_cls = _Derivate.afts, 'self, asup, *args, **kwargs'
    sub_afts = _sub_inst.afts, 'asup, n, *, o'

    def test_chain_fts(self):
        if sys.version_info < (3,3):
            return

        self._test(self._Derivate.chain_fts,
                   'self, u, *args, **kwargs')
        self._test(self._sub_inst.chain_fts,
                   'u, p, c, e, a, *, d, b, q')

    chain_afts_cls = _Derivate.chain_afts, 'self, v, *args, **kwargs'
    chain_afts = _sub_inst.chain_afts, 'v, r, c, e, a, *, d, b, s'

    def test_transform(self):
        class _callable(object):
            def __call__(self):
                raise NotImplementedError

        class Cls(object):
            @specifiers.forwards_to_method('__init__', emulate=True)
            def __new__(cls):
                raise NotImplementedError

            def __init__(self):
                raise NotImplementedError

            abc = None
            if sys.version_info >= (3,):
                abc = specifiers.forwards_to_method('__init__', emulate=True)(
                    _callable()
                    )
        Cls.abc
        Cls.__new__
        self.assertEqual(type(Cls.__dict__['__new__'].__wrapped__),
                         staticmethod)
        Cls.__new__
        self.assertEqual(type(Cls.__dict__['__new__'].__wrapped__),
                         staticmethod)

    def test_emulation(self):
        func = specifiers.forwards_to_method('abc', emulate=False)(_func)
        self.assertTrue(_func is func)

        func = specifiers.forwards_to_method('abc')(_func)
        self.assertTrue(_func is func)

        class Cls(object):
            func = specifiers.forwards_to_method('abc')(_func)
        func = getattr(Cls.func, '__func__', func)
        self.assertTrue(_func is func)
        self.assertTrue(_func is Cls().func.__func__)

        class Cls(object):
            def func(self, abc, *args, **kwargs):
                raise NotImplementedError
            def abc(self, x):
                raise NotImplementedError
        method = Cls().func
        func = specifiers.forwards_to_method('abc')(method)
        self.assertTrue(isinstance(func, specifiers._ForgerWrapper))
        self.assertEqual(func.__wrapped__, method)
        self.assertRaises(
            AttributeError,
            specifiers.forwards_to_method('abc', emulate=False), Cls().func)
        exp = support.s('abc, x')
        self.assertSigsEqual(signatures.signature(func), exp)
        self.assertSigsEqual(specifiers.signature(func), exp)

        class Emulator(object):
            def __init__(self, obj, forger):
                self.obj = obj
                self.forger = forger

        func = specifiers.forwards_to_function(func, emulate=Emulator)(_func)
        self.assertTrue(isinstance(func, Emulator))

        @specifiers.forwards_to_function(_func, emulate=True)
        def func(x, y, *args, **kwargs):
            return x + y
        self.assertEqual(5, func(2, 3))

    def test_super_fail(self):
        class Cls(object):
            def m(self):
                raise NotImplementedError
            def n(self):
                raise NotImplementedError
        class Sub(Cls):
            @specifiers.forwards_to_super()
            def m(self, *args, **kwargs):
                raise NotImplementedError
            @specifiers.forwards_to_super()
            def n(self, *args, **kwargs):
                super(Sub, self).n(*args, **kwargs) # pragma: no cover
        self.assertRaises(ValueError, specifiers.signature, Sub().m)
        if sys.version_info < (3,):
            self.assertRaises(ValueError, specifiers.signature, Sub().n)

class ForgerFunctionTests(SignatureTests):
    def test_deco(self):
        @specifiers.forger_function
        def forger(obj):
            return support.s('abc')
        @forger()
        def forged():
            raise NotImplementedError
        self.assertSigsEqual(support.s('abc'), specifiers.signature(forged))

    def test_directly_applied(self):
        def forger(obj):
            return support.s('abc')
        def forged():
            raise NotImplementedError
        specifiers.set_signature_forger(forged, forger)
        self.assertSigsEqual(support.s('abc'), specifiers.signature(forged))

    def test_forger_lazy(self):
        class Flag(Exception): pass
        @specifiers.forger_function
        def forger(obj):
            raise Flag
        @forger()
        def forged():
            pass
        self.assertSigsEqual(forged(), None)
        self.assertRaises(Flag, specifiers.signature, forged)

    def test_orig_sig(self):
        @specifiers.forger_function
        def forger(obj):
            return None
        @forger()
        def forged(alpha):
            raise NotImplementedError
        self.assertSigsEqual(support.s('alpha'), specifiers.signature(forged))

    def test_as_forged(self):
        sig = support.s('this, isa, test')
        @specifiers.forger_function
        def forger(obj):
            return sig
        class MyClass(object):
            __signature__ = specifiers.as_forged
            def __init__(self):
                forger()(self)
            def __call__(self):
                raise NotImplementedError
        self.assertSigsEqual(signatures.signature(MyClass()), sig)

    def test_as_forged_forwards(self):
        def function(a, b, c):
            raise NotImplementedError
        class MyClass(object):
            __signature__ = specifiers.as_forged
            def __init__(self):
                specifiers.forwards_to_function(function)(self)
            def __call__(self, x, *args, **kwargs):
                raise NotImplementedError
        self.assertSigsEqual(signatures.signature(MyClass()),
                             support.s('x, a, b, c'))

    def test_dunder_call(self):
        sig = support.s('dunder, call')
        @specifiers.forger_function
        def forger(obj):
            return sig
        class MyClass(object):
            def __init__(self):
                forger()(self)
            def __call__(self):
                raise NotImplementedError
        self.assertSigsEqual(specifiers.signature(MyClass()), sig)

    def test_as_forged_dunder_call_method(self):
        class MyClass(object):
            __signature__ = specifiers.as_forged
            @specifiers.forwards_to_method('method')
            def __call__(self, x, *args, **kwags):
                raise NotImplementedError
            def method(self, a, b, c):
                raise NotImplementedError
        self.assertSigsEqual(specifiers.signature(MyClass()),
                             support.s('x, a, b, c'))
<commit_msg>Test sourcing of specifiers.forwards<commit_after>#!/usr/bin/env python
# sigtools - Collection of Python modules for manipulating function signatures
# Copyright (c) 2013-2015 Yann Kaiser
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import sys

from sigtools import modifiers, specifiers, support, _util, signatures
from sigtools.tests.util import Fixtures, SignatureTests

# bulk of the testing happens in test_merge and test_embed

not_py33 = sys.version_info < (3,3)

def _func(*args, **kwargs):
    raise NotImplementedError

def _free_func(x, y, z):
    raise NotImplementedError

class _cls(object):
    method = _func
_inst = _cls()
_im_type = type(_inst.method)

class ForwardsTest(Fixtures):
    def _test(self, outer, inner, args, kwargs,
                    expected, expected_get, exp_src, exp_src_get):
        outer_f = support.f(outer, name='outer')
        inner_f = support.f(inner, name='inner')
        forw = specifiers.forwards_to_function(inner_f, *args, **kwargs)(outer_f)

        sig = specifiers.signature(forw)
        self.assertSigsEqual(sig, support.s(expected))

        self.assertSourcesEqual(sig.sources, {
                'outer': exp_src[0], 'inner': exp_src[1]})

        sig_get = specifiers.signature(_util.safe_get(forw, object(), object))
        self.assertSigsEqual(sig_get, support.s(expected_get))

        self.assertSourcesEqual(sig_get.sources, {
                'outer': exp_src_get[0], 'inner': exp_src_get[1]})

    a = (
        'a, *p, b, **k', 'c, *, d', (), {},
        'a, c, *, b, d', 'c, *, b, d', ['ab', 'cd'], ['b', 'cd'])
    b = (
        'a, *p, b, **k', 'a, c, *, b, d', (1, 'b'), {},
        'a, c, *, b, d', 'c, *, b, d', ['ab', 'cd'], ['b', 'cd'])

    def test_call(self):
        outer = support.f('*args, **kwargs')
        inner = support.f('a, *, b')
        forw = specifiers.forwards_to_function(inner)(outer)
        instance = object()
        forw_get_prox = _util.safe_get(forw, instance, object)
        self.assertEqual(
            forw_get_prox(1, b=2),
            {'args': (instance, 1), 'kwargs': {'b': 2}})

def sig_equal(self, obj, sig_str):
    self.assertSigsEqual(specifiers.signature(obj), support.s(sig_str),
                         conv_first_posarg=True)

class _Coop(object):
    @modifiers.kwoargs('cb')
    def method(self, ca, cb, *cr, **ck):
        raise NotImplementedError

class ForwardsAttributeTests(Fixtures):
    _test = sig_equal

    class _Base(object):
        def __init__(self, decorated=None):
            self.decorated = decorated
            self.coop = _Coop()

        @modifiers.kwoargs('b')
        def inner(self, a, b):
            raise NotImplementedError

        @specifiers.forwards_to(_free_func)
        def ft(self, *args, **kwargs):
            raise NotImplementedError

        @specifiers.forwards_to_method('inner')
        def ftm(self, *args, **kwargs):
            raise NotImplementedError

        @specifiers.forwards_to_ivar('decorated')
        def fti(self, *args, **kwargs):
            raise NotImplementedError

        @specifiers.forwards_to_method('ftm')
        @modifiers.kwoargs('d')
        def ftm2(self, c, d, *args2, **kwargs2):
            raise NotImplementedError

        @modifiers.kwoargs('m')
        def fts(self, l, m):
            raise NotImplementedError

        @modifiers.kwoargs('o')
        def afts(self, n, o):
            raise NotImplementedError

        @specifiers.forwards_to_method('ftm2')
        @modifiers.kwoargs('q')
        def chain_fts(self, p, q, *args, **kwargs):
            raise NotImplementedError

        @specifiers.forwards_to_method('ftm2')
        @modifiers.kwoargs('s')
        def chain_afts(self, r, s, *args, **kwargs):
            raise NotImplementedError

        @specifiers.forwards_to_method('coop.method')
        @modifiers.kwoargs('bc')
        def ccm(self, ba, bb, bc, *args, **kwargs):
            raise NotImplementedError

    @_Base
    @modifiers.kwoargs('b')
    def _base_inst(a, b):
        raise NotImplementedError

    @specifiers.apply_forwards_to_super('afts', 'chain_afts')
    class _Derivate(_Base):
        @specifiers.forwards_to_method('inner')
        def ftm(self, e, *args, **kwoargs):
            raise NotImplementedError

        @specifiers.forwards_to_super()
        def fts(self, s, *args, **kwargs):
            super() # pragma: no cover

        def afts(self, asup, *args, **kwargs):
            raise NotImplementedError

        @specifiers.forwards_to_super()
        def chain_fts(self, u, *args, **kwargs):
            super() # pragma: no cover

        def chain_afts(self, v, *args, **kwargs):
            raise NotImplementedError

    @_Derivate
    @modifiers.kwoargs('y')
    def _sub_inst(x, y):
        raise NotImplementedError

    base_func = _base_inst.ft, 'x, y, z'

    base_method = _base_inst.ftm, 'a, *, b'
    base_method2 = _base_inst.ftm2, 'c, a, *, d, b'

    base_method_cls = _Base.ftm, 'self, *args, **kwargs'

    base_ivar = _base_inst.fti, 'a, *, b'

    base_coop = _base_inst.ccm, 'ba, bb, ca, *cr, bc, cb, **ck'

    sub_method = _sub_inst.ftm, 'e, a, *, b'

    sub_method2 = _sub_inst.ftm2, 'c, e, a, *, d, b'

    sub_ivar = _sub_inst.fti, 'x, *, y'

    def test_fts(self):
        if sys.version_info >= (3,3):
            self._test(self._sub_inst.fts, 's, l, *, m')

    sub_afts_cls = _Derivate.afts, 'self, asup, *args, **kwargs'
    sub_afts = _sub_inst.afts, 'asup, n, *, o'

    def test_chain_fts(self):
        if sys.version_info < (3,3):
            return

        self._test(self._Derivate.chain_fts,
                   'self, u, *args, **kwargs')
        self._test(self._sub_inst.chain_fts,
                   'u, p, c, e, a, *, d, b, q')

    chain_afts_cls = _Derivate.chain_afts, 'self, v, *args, **kwargs'
    chain_afts = _sub_inst.chain_afts, 'v, r, c, e, a, *, d, b, s'

    def test_transform(self):
        class _callable(object):
            def __call__(self):
                raise NotImplementedError

        class Cls(object):
            @specifiers.forwards_to_method('__init__', emulate=True)
            def __new__(cls):
                raise NotImplementedError

            def __init__(self):
                raise NotImplementedError

            abc = None
            if sys.version_info >= (3,):
                abc = specifiers.forwards_to_method('__init__', emulate=True)(
                    _callable()
                    )
        Cls.abc
        Cls.__new__
        self.assertEqual(type(Cls.__dict__['__new__'].__wrapped__),
                         staticmethod)
        Cls.__new__
        self.assertEqual(type(Cls.__dict__['__new__'].__wrapped__),
                         staticmethod)

    def test_emulation(self):
        func = specifiers.forwards_to_method('abc', emulate=False)(_func)
        self.assertTrue(_func is func)

        func = specifiers.forwards_to_method('abc')(_func)
        self.assertTrue(_func is func)

        class Cls(object):
            func = specifiers.forwards_to_method('abc')(_func)
        func = getattr(Cls.func, '__func__', func)
        self.assertTrue(_func is func)
        self.assertTrue(_func is Cls().func.__func__)

        class Cls(object):
            def func(self, abc, *args, **kwargs):
                raise NotImplementedError
            def abc(self, x):
                raise NotImplementedError
        method = Cls().func
        func = specifiers.forwards_to_method('abc')(method)
        self.assertTrue(isinstance(func, specifiers._ForgerWrapper))
        self.assertEqual(func.__wrapped__, method)
        self.assertRaises(
            AttributeError,
            specifiers.forwards_to_method('abc', emulate=False), Cls().func)
        exp = support.s('abc, x')
        self.assertSigsEqual(signatures.signature(func), exp)
        self.assertSigsEqual(specifiers.signature(func), exp)

        class Emulator(object):
            def __init__(self, obj, forger):
                self.obj = obj
                self.forger = forger

        func = specifiers.forwards_to_function(func, emulate=Emulator)(_func)
        self.assertTrue(isinstance(func, Emulator))

        @specifiers.forwards_to_function(_func, emulate=True)
        def func(x, y, *args, **kwargs):
            return x + y
        self.assertEqual(5, func(2, 3))

    def test_super_fail(self):
        class Cls(object):
            def m(self):
                raise NotImplementedError
            def n(self):
                raise NotImplementedError
        class Sub(Cls):
            @specifiers.forwards_to_super()
            def m(self, *args, **kwargs):
                raise NotImplementedError
            @specifiers.forwards_to_super()
            def n(self, *args, **kwargs):
                super(Sub, self).n(*args, **kwargs) # pragma: no cover
        self.assertRaises(ValueError, specifiers.signature, Sub().m)
        if sys.version_info < (3,):
            self.assertRaises(ValueError, specifiers.signature, Sub().n)

class ForgerFunctionTests(SignatureTests):
    def test_deco(self):
        @specifiers.forger_function
        def forger(obj):
            return support.s('abc')
        @forger()
        def forged():
            raise NotImplementedError
        self.assertSigsEqual(support.s('abc'), specifiers.signature(forged))

    def test_directly_applied(self):
        def forger(obj):
            return support.s('abc')
        def forged():
            raise NotImplementedError
        specifiers.set_signature_forger(forged, forger)
        self.assertSigsEqual(support.s('abc'), specifiers.signature(forged))

    def test_forger_lazy(self):
        class Flag(Exception): pass
        @specifiers.forger_function
        def forger(obj):
            raise Flag
        @forger()
        def forged():
            pass
        self.assertSigsEqual(forged(), None)
        self.assertRaises(Flag, specifiers.signature, forged)

    def test_orig_sig(self):
        @specifiers.forger_function
        def forger(obj):
            return None
        @forger()
        def forged(alpha):
            raise NotImplementedError
        self.assertSigsEqual(support.s('alpha'), specifiers.signature(forged))

    def test_as_forged(self):
        sig = support.s('this, isa, test')
        @specifiers.forger_function
        def forger(obj):
            return sig
        class MyClass(object):
            __signature__ = specifiers.as_forged
            def __init__(self):
                forger()(self)
            def __call__(self):
                raise NotImplementedError
        self.assertSigsEqual(signatures.signature(MyClass()), sig)

    def test_as_forged_forwards(self):
        def function(a, b, c):
            raise NotImplementedError
        class MyClass(object):
            __signature__ = specifiers.as_forged
            def __init__(self):
                specifiers.forwards_to_function(function)(self)
            def __call__(self, x, *args, **kwargs):
                raise NotImplementedError
        self.assertSigsEqual(signatures.signature(MyClass()),
                             support.s('x, a, b, c'))

    def test_dunder_call(self):
        sig = support.s('dunder, call')
        @specifiers.forger_function
        def forger(obj):
            return sig
        class MyClass(object):
            def __init__(self):
                forger()(self)
            def __call__(self):
                raise NotImplementedError
        self.assertSigsEqual(specifiers.signature(MyClass()), sig)

    def test_as_forged_dunder_call_method(self):
        class MyClass(object):
            __signature__ = specifiers.as_forged
            @specifiers.forwards_to_method('method')
            def __call__(self, x, *args, **kwags):
                raise NotImplementedError
            def method(self, a, b, c):
                raise NotImplementedError
        self.assertSigsEqual(specifiers.signature(MyClass()),
                             support.s('x, a, b, c'))
<|endoftext|>


<commit_before>from memory_profiler import profile

@profile
def foo():
    a = [1] * (10 ** 6)
    b = [2] * (2 * 10 ** 7)
    # add dome lines here and push it to your branch
    del b
    return a

if __name__ == '__main__':
    foo()
<commit_msg>move back to normal<commit_after>from memory_profiler import profile

@profile
def foo():
    a = [1] * (10 ** 6)
    b = [2] * (2 * 10 ** 7)
    # add dome lines here and push it to your branch
    #
    del b
    return a

if __name__ == '__main__':
    foo()
<|endoftext|>


<commit_before>from simulator.merchant import Merchant
from simulator.customer_unimaus import GenuineCustomer, FraudulentCustomer
from mesa.time import RandomActivation
from simulator import parameters
from simulator.log_collector import LogCollector
from mesa import Model
from datetime import timedelta
import numpy as np

class UniMausTransactionModel(Model):
    def __init__(self, model_parameters):
        super().__init__()

        # make sure we didn't accidentally add new params anywhere
        assert len(model_parameters) == len(parameters.get_default_parameters())

        # load parameters
        self.parameters = model_parameters
        self.random_state = np.random.RandomState(self.parameters["seed"])
        self.curr_global_date = self.parameters['start_date']
        self.curr_month = -1
        self.today_int = int(self.curr_global_date.date().strftime("%s"))

        # set termination status
        self.terminated = False

        # create merchants, customers and fraudsters
        self.merchants = self.initialise_merchants()

        self.next_customer_id = 0
        self.next_fraudster_id = 0
        self.next_card_id = 0
        self.customers = self.initialise_customers()
        self.fraudsters = self.initialise_fraudsters()

        # set up a scheduler
        self.schedule = RandomActivation(self)

        # create data collector for the transaction logs
        self.log_collector = LogCollector(
            agent_reporters={"Global_Date": lambda c: c.model.curr_global_date,
                             "Local_Date": lambda c: c.curr_local_date,
                             "CardID": lambda c: c.card_id,
                             "MerchantID": lambda c: c.curr_merchant.unique_id,
                             "Amount": lambda c: c.curr_amount,
                             "Currency": lambda c: c.currency,
                             "Country": lambda c: c.country,
                             "Target": lambda c: c.fraudster})

        # get the true fractions of transactions per month/hour/weekday/monthday
        self.t_frac_month = self.parameters['frac_month']
        self.t_frac_hour = self.parameters['frac_hour']
        self.t_frac_monthday = self.parameters['frac_monthday']
        self.t_frac_weekday = self.parameters['frac_weekday']

    def step(self):

        # add some noise to the transaction probabilities
        self.t_frac_month = self.parameters['frac_month'] + self.parameters['noise_level']*self.random_state.normal(0, 1./120, 1)[0]
        self.t_frac_hour = self.parameters['frac_hour'] + self.parameters['noise_level']*self.random_state.normal(0, 1./240, 1)[0]
        self.t_frac_monthday = self.parameters['frac_monthday'] + self.parameters['noise_level']*self.random_state.normal(0, 1./310, 1)[0]
        self.t_frac_weekday = self.parameters['frac_weekday'] + self.parameters['noise_level']*self.random_state.normal(0, 1./70, 1)[0]

        # this calls the step function of each agent in the schedule (customer, fraudster)
        self.schedule.agents = []
        self.schedule.agents.extend(self.customers)
        self.schedule.agents.extend(self.fraudsters)
        self.schedule.step()

        # write new transactions to log
        self.log_collector.collect(self)

        # migration of customers/fraudsters
        self.customer_migration()

        # update time
        if self.curr_month != self.curr_global_date.month:
            self.curr_month = self.curr_global_date.month
            print(self.curr_global_date.date())
        self.curr_global_date = self.curr_global_date + timedelta(hours=1)

        # check if termination criterion met
        if self.curr_global_date.date() > self.parameters['end_date'].date():
            self.terminated = True

    def customer_migration(self):

        num_customers_before = len(self.customers)
        num_fraudsters_before = len(self.fraudsters)

        # emigration
        self.customers = [c for c in self.customers if c.stay]
        self.fraudsters = [f for f in self.fraudsters if f.stay]

        # immigration
        self.immigration_customers(num_customers_before)
        self.immigration_fraudsters(num_fraudsters_before)

    def immigration_customers(self, num_customers_before):
        # add as many customers as we removed
        num_new_cust = num_customers_before - len(self.customers)
        self.customers.extend([GenuineCustomer(self) for _ in range(num_new_cust)])

    def immigration_fraudsters(self, num_fraudsters_before):
        # estimate number of fraudsters we add
        num_new_frauds = num_fraudsters_before - len(self.fraudsters)
        self.fraudsters.extend([FraudulentCustomer(self) for _ in range(num_new_frauds)])

    def initialise_merchants(self):
        return [Merchant(i, self) for i in range(self.parameters["num_merchants"])]

    def initialise_customers(self):
        return [GenuineCustomer(self) for _ in range(self.parameters['num_customers'])]

    def initialise_fraudsters(self):
        return [FraudulentCustomer(self) for _ in range(self.parameters["num_fraudsters"])]

    def get_next_customer_id(self):
        next_id = self.next_customer_id
        self.next_customer_id += 1
        return next_id

    def get_next_fraudster_id(self):
        next_id = self.next_fraudster_id
        self.next_fraudster_id += 1
        return next_id

    def get_next_card_id(self):
        next_id = self.next_card_id
        self.next_card_id += 1
        return next_id
<commit_msg>self-sufficient customer/fraudster immigration<commit_after>from simulator.merchant import Merchant
from mesa.time import RandomActivation
from simulator import parameters
from simulator.log_collector import LogCollector
from mesa import Model
from datetime import timedelta
import numpy as np

class UniMausTransactionModel(Model):
    def __init__(self, model_parameters, CustomerClass, FraudsterClass):
        super().__init__()

        # make sure we didn't accidentally add new params anywhere
        assert len(model_parameters) == len(parameters.get_default_parameters())

        # load parameters
        self.parameters = model_parameters
        self.CustomerClass = CustomerClass
        self.FraudsterClass = FraudsterClass

        self.random_state = np.random.RandomState(self.parameters["seed"])
        self.curr_global_date = self.parameters['start_date']
        self.curr_month = -1
        self.today_int = int(self.curr_global_date.date().strftime("%s"))

        # set termination status
        self.terminated = False

        # create merchants, customers and fraudsters
        self.merchants = self.initialise_merchants()

        self.next_customer_id = 0
        self.next_fraudster_id = 0
        self.next_card_id = 0
        self.customers = self.initialise_customers()
        self.fraudsters = self.initialise_fraudsters()

        # set up a scheduler
        self.schedule = RandomActivation(self)

        # create data collector for the transaction logs
        self.log_collector = LogCollector(
            agent_reporters={"Global_Date": lambda c: c.model.curr_global_date,
                             "Local_Date": lambda c: c.curr_local_date,
                             "CardID": lambda c: c.card_id,
                             "MerchantID": lambda c: c.curr_merchant.unique_id,
                             "Amount": lambda c: c.curr_amount,
                             "Currency": lambda c: c.currency,
                             "Country": lambda c: c.country,
                             "Target": lambda c: c.fraudster})

        # get the true fractions of transactions per month/hour/weekday/monthday
        self.t_frac_month = self.parameters['frac_month']
        self.t_frac_hour = self.parameters['frac_hour']
        self.t_frac_monthday = self.parameters['frac_monthday']
        self.t_frac_weekday = self.parameters['frac_weekday']

    def step(self):

        # add some noise to the transaction probabilities
        self.t_frac_month = self.parameters['frac_month'] + self.parameters['noise_level']*self.random_state.normal(0, 1./120, 1)[0]
        self.t_frac_hour = self.parameters['frac_hour'] + self.parameters['noise_level']*self.random_state.normal(0, 1./240, 1)[0]
        self.t_frac_monthday = self.parameters['frac_monthday'] + self.parameters['noise_level']*self.random_state.normal(0, 1./310, 1)[0]
        self.t_frac_weekday = self.parameters['frac_weekday'] + self.parameters['noise_level']*self.random_state.normal(0, 1./70, 1)[0]

        # this calls the step function of each agent in the schedule (customer, fraudster)
        self.schedule.agents = []
        self.schedule.agents.extend(self.customers)
        self.schedule.agents.extend(self.fraudsters)
        self.schedule.step()

        # write new transactions to log
        self.log_collector.collect(self)

        # migration of customers/fraudsters
        self.customer_migration()

        # update time
        if self.curr_month != self.curr_global_date.month:
            self.curr_month = self.curr_global_date.month
            print(self.curr_global_date.date())
        self.curr_global_date = self.curr_global_date + timedelta(hours=1)

        # check if termination criterion met
        if self.curr_global_date.date() > self.parameters['end_date'].date():
            self.terminated = True

    def customer_migration(self):

        # emigration
        self.customers = [c for c in self.customers if c.stay]
        self.fraudsters = [f for f in self.fraudsters if f.stay]

        # immigration
        self.immigration_customers()
        self.immigration_fraudsters()

    def immigration_customers(self):
        fraudster = 0

        noise_level = self.parameters['noise_level']
        # estimate how many genuine transactions there were
        num_transactions = self.parameters['trans_per_year'][fraudster]
        num_transactions /= 356 * 24
        # scale by current month
        num_trans_month = num_transactions * 12 * self.parameters['frac_month'][self.curr_global_date.month - 1, fraudster]
        num_transactions = (1 - noise_level) * num_trans_month + noise_level * num_transactions

        # estimate how many customers on avg left
        num_customers_left = num_transactions * (1 - self.parameters['stay_prob'][fraudster])

        if num_customers_left > 1:
            num_customers_left += self.random_state.normal(0, 1, 1)[0]
            num_customers_left = int(np.round(num_customers_left, 0))
            num_customers_left = np.max([0, num_customers_left])
        else:
            if num_customers_left > np.random.uniform(0, 1, 1)[0]:
                num_customers_left = 1
            else:
                num_customers_left = 0

        # add as many customers as we think that left
        self.customers.extend([self.CustomerClass(self) for _ in range(num_customers_left)])

    def immigration_fraudsters(self):
        fraudster = 1
        noise_level = self.parameters['noise_level']
        # estimate how many fraudulent transactions there were
        num_transactions = self.parameters['trans_per_year'][fraudster]
        num_transactions /= 356 * 24
        # scale by current month
        num_trans_month = num_transactions * 12 * self.parameters['frac_month'][self.curr_global_date.month - 1, fraudster]
        num_transactions = (1 - noise_level) * num_trans_month + noise_level * num_transactions

        # estimate how many fraudsters on avg left
        num_fraudsters_left = num_transactions * (1 - self.parameters['stay_prob'][fraudster])

        if num_fraudsters_left > 1:
            num_fraudsters_left += self.random_state.normal(0, 1, 1)[0]
            num_fraudsters_left = int(np.round(num_fraudsters_left, 0))
            num_fraudsters_left = np.max([0, num_fraudsters_left])
        else:
            if num_fraudsters_left > np.random.uniform(0, 1, 1)[0]:
                num_fraudsters_left = 1
            else:
                num_fraudsters_left = 0

        # add as many fraudsters as we think that left
        self.fraudsters.extend([self.FraudsterClass(self) for _ in range(num_fraudsters_left)])

    def initialise_merchants(self):
        return [Merchant(i, self) for i in range(self.parameters["num_merchants"])]

    def initialise_customers(self):
        return [self.CustomerClass(self) for _ in range(self.parameters['num_customers'])]

    def initialise_fraudsters(self):
        return [self.FraudsterClass(self) for _ in range(self.parameters["num_fraudsters"])]

    def get_next_customer_id(self):
        next_id = self.next_customer_id
        self.next_customer_id += 1
        return next_id

    def get_next_fraudster_id(self):
        next_id = self.next_fraudster_id
        self.next_fraudster_id += 1
        return next_id

    def get_next_card_id(self):
        next_id = self.next_card_id
        self.next_card_id += 1
        return next_id
<|endoftext|>


<commit_before>from stdnet import test
from stdnet.utils import zip

from .fuzzy import makeItems, Item

class TestIndexItem(test.ProfileTest):

    def initialise(self):
        makeItems(100,300)

    def run(self):
        engine = self.engine.index_item
        for item in Item.objects.all():
            engine(item)
    <commit_msg>profile test in search engine. Maasive room for speed improvement<commit_after>from stdnet import test, orm
from stdnet.utils import zip

from .fuzzy import makeItems, Item, Word, WordItem, SearchEngine

class TestIndexItem(test.ProfileTest):

    def initialise(self):
        self.engine = SearchEngine(autocomplete = False)
        orm.register(Word)
        orm.register(Item)
        orm.register(WordItem)
        makeItems(100,300)

    def run(self):
        engine = self.engine.index_item
        for item in Item.objects.all():
            engine(item)
    <|endoftext|>


<commit_before># Here goes example wrappers that you are free to modify depending on
# your needs. It's not meant to be updated with new releases of
# clj-cljs-buck: Use it as an example, change it and store in your
# repo

# Import clj-cljs macroses
include_defs('//lib.py')

# Helpers for easier referencing different things
def ext(name):
    return '//tests:' + name
def executer(name):
    return '//:' + name
def resource(name):
    return '//clj-cljs-config:' + name
cljs_deps = [ext('org.clojure/clojure'),
             ext('org.clojure/clojurescript'),
             ext('figwheel-sidecar'),
             ext('com.cemerick/piggieback')]
builder = executer('builder-planck')

# First is CLJ wrapper - nothing fancy, we just predefine what project
# file to use, setup tester (which is an only options for CLJ for now)
# and add Clojure as dependency that is added always.

# Only interesting part is ensure_list that ensures that item is a
# list or wraps item with it. Most often you want to use one source
# file, one test, one module dependency so you can use
# clj_module(src='a.clj',tests='a_test.clj',modules=':b') without
# wrapping each with []
def clj_module(name,src=[],modules=[],main=None,tests=[],resources=[]):
    clj_cljs_module(ext = 'clj',
                    project_file = resource('project-clj'),
                    builder = builder,
                    resources = ensure_list(resources),
                    tester = executer('tester-lein-clj'),
                    name = name,
                    src = ensure_list(src),
                    modules = ensure_list(modules) + [ext('org.clojure/clojure')],
                    main = main,
                    tests = tests)

# Here goes CLJS wrapper with example of custom logic - if tests
# supplied than planck test executer is used (because it's 10 times
# faster!), otherwise classic doo.

# Another custom logic is release task - if main is specified then new
# target is added which creates release bundle
def cljs_module(name,src=[],modules=[],main=None,tests=[],resources=[],itests=[]):
    clj_cljs_module(ext = 'cljs',
                    project_file = resource('project-cljs'),
                    builder = builder,
                    resources = ensure_list(resources) + [resource('figwheel-index')],
                    tester = executer('tester-lein-cljs-doo') if itests else executer('tester-lein-cljs-planck'),
                    name = name,
                    src = ensure_list(src),
                    modules = ensure_list(modules) + cljs_deps,
                    main = main,
                    tests = ensure_list(tests) + ensure_list(itests))

    if (main):
        genrule(name + '-release',
                srcs = [],
                bash = 'mkdir $OUT && cd $(location :__{0}) && lein cljsbuild once release && cp release/{0}.js $OUT && cp -r resources $OUT'.format(name),
                out = 'build')

# Nothing interesting here - same as for CLJS, but wihtout release task
def cljc_module(name,src=[],modules=[],main=None,tests=[],resources=[],itests=[]):
    clj_cljs_module(ext = 'cljc',
                    project_file = resource('project-cljs'),
                    builder = builder,
                    resources = ensure_list(resources) + [resource('figwheel-index')],
                    tester = executer('tester-lein-cljs-doo') if itests else executer('tester-lein-cljs-planck'),
                    name = name,
                    src = ensure_list(src),
                    modules = ensure_list(modules) + cljs_deps,
                    main = main,
                    tests = ensure_list(tests) + ensure_list(itests))
<commit_msg>RULES/clojure-clojurescript-buck: Fixed an issue when empty src was always passed, althoug automatic src finding based on name works only when src is None<commit_after># Here goes example wrappers that you are free to modify depending on
# your needs. It's not meant to be updated with new releases of
# clj-cljs-buck: Use it as an example, change it and store in your
# repo

# Import clj-cljs macroses
include_defs('//lib.py')

# Helpers for easier referencing different things
def ext(name):
    return '//tests:' + name
def executer(name):
    return '//:' + name
def resource(name):
    return '//clj-cljs-config:' + name
cljs_deps = [ext('org.clojure/clojure'),
             ext('org.clojure/clojurescript'),
             ext('figwheel-sidecar'),
             ext('com.cemerick/piggieback')]
builder = executer('builder-planck')

# First is CLJ wrapper - nothing fancy, we just predefine what project
# file to use, setup tester (which is an only options for CLJ for now)
# and add Clojure as dependency that is added always.

# Only interesting part is ensure_list that ensures that item is a
# list or wraps item with it. Most often you want to use one source
# file, one test, one module dependency so you can use
# clj_module(src='a.clj',tests='a_test.clj',modules=':b') without
# wrapping each with []
def clj_module(name,src=None,modules=[],main=None,tests=[],resources=[]):
    clj_cljs_module(ext = 'clj',
                    project_file = resource('project-clj'),
                    builder = builder,
                    resources = ensure_list(resources),
                    tester = executer('tester-lein-clj'),
                    name = name,
                    src = src,
                    modules = ensure_list(modules) + [ext('org.clojure/clojure')],
                    main = main,
                    tests = tests)

# Here goes CLJS wrapper with example of custom logic - if tests
# supplied than planck test executer is used (because it's 10 times
# faster!), otherwise classic doo.

# Another custom logic is release task - if main is specified then new
# target is added which creates release bundle
def cljs_module(name,src=None,modules=[],main=None,tests=[],resources=[],itests=[]):
    clj_cljs_module(ext = 'cljs',
                    project_file = resource('project-cljs'),
                    builder = builder,
                    resources = ensure_list(resources) + [resource('figwheel-index')],
                    tester = executer('tester-lein-cljs-doo') if itests else executer('tester-lein-cljs-planck'),
                    name = name,
                    src = src,
                    modules = ensure_list(modules) + cljs_deps,
                    main = main,
                    tests = ensure_list(tests) + ensure_list(itests))

    if (main):
        genrule(name + '-release',
                srcs = [],
                bash = 'mkdir $OUT && cd $(location :__{0}) && lein cljsbuild once release && cp release/{0}.js $OUT && cp -r resources $OUT'.format(name),
                out = 'build')

# Nothing interesting here - same as for CLJS, but wihtout release task
def cljc_module(name,src=None,modules=[],main=None,tests=[],resources=[],itests=[]):
    clj_cljs_module(ext = 'cljc',
                    project_file = resource('project-cljs'),
                    builder = builder,
                    resources = ensure_list(resources) + [resource('figwheel-index')],
                    tester = executer('tester-lein-cljs-doo') if itests else executer('tester-lein-cljs-planck'),
                    name = name,
                    src = src,
                    modules = ensure_list(modules) + cljs_deps,
                    main = main,
                    tests = ensure_list(tests) + ensure_list(itests))
<|endoftext|>


<commit_before>"""CloudBioLinux specific scripts
"""
import os
from fabric.api import *
from fabric.contrib.files import *

def _freenx_scripts(env):
    """Provide graphical access to clients via FreeNX.
    """
    setup_script = "setupnx.sh"
    remote_setup = "%s/bin/%s" % (env.system_install, setup_script)
    if not exists(os.path.dirname(remote_setup)):
        sudo('mkdir -p {0}'.format(os.path.dirname(remote_setup)))
    install_file_dir = os.path.join(env.config_dir, os.pardir, "installed_files")
    if not exists(remote_setup):
        put(os.path.join(install_file_dir, setup_script), setup_script,
                mode=0777)
        env.safe_sudo("mv %s %s" % (setup_script, remote_setup))
    remote_login = "configure_freenx.sh"
    if not exists(remote_login):
        put(os.path.join(install_file_dir, 'bash_login'), remote_login,
                mode=0777)
    _configure_gnome(env)

def _cleanup_space(env):
    """Cleanup to recover space from builds and packages.
    """
    env.logger.info("Cleaning up space from package builds")
    env.safe_sudo("rm -rf .cpanm")
    env.safe_sudo("rm -f /var/crash/*")

def _configure_gnome(env):
    """Configure NX server to use classic GNOME.

    http://askubuntu.com/questions/50503/why-do-i-get-unity-instead-of-classic-when-using-nx
    http://notepad2.blogspot.com/2012/04/install-freenx-server-on-ubuntu-1110.html
    """
    add = 'COMMAND_START_GNOME="gnome-session --session gnome-fallback"'
    fname = "/etc/nxserver/node.conf"
    if exists("/etc/nxserver/"):
        append(fname, add, use_sudo=True)
<commit_msg>Clean up additional files in home directory from noisy programs<commit_after>"""CloudBioLinux specific scripts
"""
import os
from fabric.api import *
from fabric.contrib.files import *

def _freenx_scripts(env):
    """Provide graphical access to clients via FreeNX.
    """
    setup_script = "setupnx.sh"
    remote_setup = "%s/bin/%s" % (env.system_install, setup_script)
    if not exists(os.path.dirname(remote_setup)):
        sudo('mkdir -p {0}'.format(os.path.dirname(remote_setup)))
    install_file_dir = os.path.join(env.config_dir, os.pardir, "installed_files")
    if not exists(remote_setup):
        put(os.path.join(install_file_dir, setup_script), setup_script,
                mode=0777)
        env.safe_sudo("mv %s %s" % (setup_script, remote_setup))
    remote_login = "configure_freenx.sh"
    if not exists(remote_login):
        put(os.path.join(install_file_dir, 'bash_login'), remote_login,
                mode=0777)
    _configure_gnome(env)

def _cleanup_space(env):
    """Cleanup to recover space from builds and packages.
    """
    env.logger.info("Cleaning up space from package builds")
    env.safe_sudo("rm -rf .cpanm")
    env.safe_sudo("rm -f /var/crash/*")
    run("rm -f ~/*.dot")
    run("rm -f ~/*.log")

def _configure_gnome(env):
    """Configure NX server to use classic GNOME.

    http://askubuntu.com/questions/50503/why-do-i-get-unity-instead-of-classic-when-using-nx
    http://notepad2.blogspot.com/2012/04/install-freenx-server-on-ubuntu-1110.html
    """
    add = 'COMMAND_START_GNOME="gnome-session --session gnome-fallback"'
    fname = "/etc/nxserver/node.conf"
    if exists("/etc/nxserver/"):
        append(fname, add, use_sudo=True)
<|endoftext|>


<commit_before>from __future__ import print_function

import subprocess
from cloudmesh_base.ConfigDict import ConfigDict
from cloudmesh_base.tables import dict_printer
from cloudmesh_base.Shell import Shell
from cloudmesh_base.util import banner
from cloudmesh_base.util import path_expand
from cmd3.console import Console
from cloudmesh_base.locations import config_file
import pymongo
from pymongo import MongoClient
import datetime
import os
from pprint import pprint
import sys

class JobDB(object):

    database = None
    jobs = None

    def load(self, filename="/cloudmesh_pbs.yaml"):
        self.filename = config_file(filename)
        self.data = ConfigDict(filename=self.filename)

        self.port = self.data["cloudmesh"]["jobdatabase"]["port"]
        self.db_path = path_expand(self.data["cloudmesh"]["jobdatabase"]["db_path"])
        self.log_file = path_expand(self.db_path + "/dbjobs.log")
        self.dbname = self.data["cloudmesh"]["jobdatabase"]["dbname"]

    def __init__(self, yaml_filename="/cloudmesh_pbs.yaml"):
        """
        Creates an object instance of a job database as defined in the cloudmesh_pbs yaml file

        :param deploy: If True, creates the configuration files
        :param yaml_filename: The cloudmesh pbs yaml file. Defaults to
                              cloudmesh_pbs.yaml
        :return: an object for manageing jobs in the database
        """
        self.load(filename=yaml_filename)
        self.deploy()

    def deploy(self):
        """
        creates the directories if they do not exist
        :return:
        """
        banner("CREATE DIRS")
        try:
            r = Shell.mkdir(self.db_path)
        except Exception, e:
            Console.error("Problem creating the database directory {:}".format(self.db_path))
            print(e)
        try:
            file = open(self.log_file, 'a+')
            file.close()
        except Exception, e:
            Console.error("Problem creating the log file {:}".format(self.log_file))
            print(e)

    # TODO COMBINE WITH __INIT__ ???
    # BUG not a good location for  the data /data
    # BUG not a good location for logpath as we run this in user mode
    # port and location should probably read from cloudmesh_pbs.yaml and values here set to None.see load function

    def ps(self):
        r = Shell.fgrep(Shell.ps("aux", _tty_out=False), "mongod", _tty_in=False)
        print(r)

    def start(self):
        try:

            mongod = Shell.which("mongod")
            command = [
                mongod,
                "--dbpath", str(self.db_path),
                "--port", str(self.port),
                "--fork",
                "--logpath", str(self.log_file),
                "--bind_ip", "127.0.0.1"
            ]
            print(" ".join(command))
            #a = subprocess.call(command)
            os.system(" ".join(command))
            os.system ("ps aux | fgrep mongod")
            Console.ok("MongoDB has been deployed")
            self.info()

        except Exception, e:
            Console.error("we had a problem starting the  mongo daemon")
            print(e)
            Console.error("MongoDB has stopped")
            # TODO remove the exit in final code, for debugging only
            sys.exit()

    def stop(self):
        try:
            #command = ["mongod", "--shutdown"]
            command = ["killall", "mongod"]
            print(" ".join(command))
            os.system(" ".join(command))
            Console.ok("MongoDB has been shutdown")
            #os.system("mongod --shutdown")
        except Exception, e:
            Console.error("we had a problem shutting the mongo daemon down")
            print(e)

    def info(self):
        # TODO: implement self. dbath, self.port, self.logpath
        Console.ok("Mongo parameters")
        Console.ok("  dbpath:  {:}".format(self.db_path))
        Console.ok("  port:    {:}".format(self.port))
        Console.ok("  logfile: {:}".format(self.log_file))
        Console.ok("  dbname:  {:}".format(self.dbname))

    def connect(self):

        client = MongoClient('localhost', self.port)
        self.database = client["jobsdb"]
        self.jobs = self.database["jobs"]
        self.id = self.database["id"]  # manages the counter for the job

        Console.info("Connecting to the Mongo Database")

    def getid(self):
        pass

    def setid(self):
        pass

    def incrid(self):
        pass

    # job-name is unique

    def insert(self,
               job_name,
               input_filename="",
               output_filename="",
               parameters="",
               job_group=None,
               job_label=None,
               job_id=None,  # possibly same as job_name ?
               host=None,
               start_time=str(datetime.datetime.now()),
               end_time=str(datetime.datetime.now()),
               update_time=str(datetime.datetime.now()),
               job_status="C_DEFINED",  # see what we wrote in paper
):
        # TODO: end_time should be None as unlikely
        # it so short and we need to deal with jobs
        # that we do not know how long they are

        if self.database is not None:

            job = {
                   "_id": job_name,
                   "job_name": job_name,
                   "job_id": job_name,
                   "job_group": job_group,
                   "job_label": job_label,
                   "job_status": job_status,
                   "host": host,
                   "input_filename": input_filename,  # must be array
                   "output_filename": output_filename,  # must be array
                   "start_time": start_time,
                   "end_time": end_time,
                   "update_time": update_time,
                   }

            banner("job")
            pprint(job)
            banner("insert")

            db_job_object = self.jobs.save(job)
            # print (db_job_object)
            # db_job_id = db_job_object
            # print (db_job_id)

            return db_job_object

        else:

            Console.error("Please connect to the database first")
            return -1

    def insert_job_object(self, job):

        if self.database is not None:

            job_id = self.jobs.insert_one(job).inserted_id

            return job_id

        else:
            Console.error("Please connect to the database first")
            return -1

    def find_jobs(self, attribute="", value=""):

        if self.database is not None:

            # Return all jobs if not given search parameters
            if attribute == "" and value == "":
                return self.jobs.find()

            # Return all jobs given the search parameters
            else:
                return self.jobs.find({attribute: value})

        else:
            Console.error("Please connect to the database first")
            return -1

    def delete_jobs(self, attribute="", value=""):

        if self.database is not None:

            # Delete all jobs if no key name or value has been given
            if attribute == "" and value == "":
                self.jobs.remove()

            # Delete only records from a query result if key name and value have been given
            else:
                self.jobs.remove({attribute: value})

        else:
            Console.error("Please connect to the database first")
            return -1

    def count(self, attribute="", value=""):
        """
        return the count of all jobs with a given value at the attribute
        :param attribute:
        :param value:
        :return:
        """
        if self.database is not None:

            # Return the count of all jobs if not given search parameters
            if attribute == "" and value == "":
                return self.jobs.count()

            # Return number of results of the query given the search parameters
            else:
                return self.jobs.find(({attribute: value})).count()

        else:
            Console.error("Please connect to the database first")
            return -1

    def update_job_end_time(self, job_id, end_time=str(datetime.datetime.now())):

        if self.database is not None:

            self.jobs.update({"_id": job_id}, {"$set": {"end_time": end_time}}, upsert=False)

        else:
            Console.error("Please connect to the database first")
            return -1

    def updateJobAttribute(self, job_id, attribute, value):

        if self.database is not None:

            if attribute == "_id":

                print 'You cannot change the unique Object ID generated by MongoDB'

            else:

                self.jobs.update({"_id" : job_id}, {"$set": {attribute : value}}, upsert=False)

        else:

            print "Please connect to the database first"
            return -1

<commit_msg>Added jobStatusStats<commit_after>from __future__ import print_function

import subprocess
from cloudmesh_base.ConfigDict import ConfigDict
from cloudmesh_base.tables import dict_printer
from cloudmesh_base.Shell import Shell
from cloudmesh_base.util import banner
from cloudmesh_base.util import path_expand
from cmd3.console import Console
from cloudmesh_base.locations import config_file
import pymongo
from pymongo import MongoClient
import datetime
import os
from pprint import pprint
import sys

class JobDB(object):

    database = None
    jobs = None

    def load(self, filename="/cloudmesh_pbs.yaml"):
        self.filename = config_file(filename)
        self.data = ConfigDict(filename=self.filename)

        self.port = self.data["cloudmesh"]["jobdatabase"]["port"]
        self.db_path = path_expand(self.data["cloudmesh"]["jobdatabase"]["db_path"])
        self.log_file = path_expand(self.db_path + "/dbjobs.log")
        self.dbname = self.data["cloudmesh"]["jobdatabase"]["dbname"]

    def __init__(self, yaml_filename="/cloudmesh_pbs.yaml"):
        """
        Creates an object instance of a job database as defined in the cloudmesh_pbs yaml file

        :param deploy: If True, creates the configuration files
        :param yaml_filename: The cloudmesh pbs yaml file. Defaults to
                              cloudmesh_pbs.yaml
        :return: an object for manageing jobs in the database
        """
        self.load(filename=yaml_filename)
        self.deploy()

    def deploy(self):
        """
        creates the directories if they do not exist
        :return:
        """
        banner("CREATE DIRS")
        try:
            r = Shell.mkdir(self.db_path)
        except Exception, e:
            Console.error("Problem creating the database directory {:}".format(self.db_path))
            print(e)
        try:
            file = open(self.log_file, 'a+')
            file.close()
        except Exception, e:
            Console.error("Problem creating the log file {:}".format(self.log_file))
            print(e)

    # TODO COMBINE WITH __INIT__ ???
    # BUG not a good location for  the data /data
    # BUG not a good location for logpath as we run this in user mode
    # port and location should probably read from cloudmesh_pbs.yaml and values here set to None.see load function

    def ps(self):
        r = Shell.fgrep(Shell.ps("aux", _tty_out=False), "mongod", _tty_in=False)
        print(r)

    def start(self):
        try:

            mongod = Shell.which("mongod")
            command = [
                mongod,
                "--dbpath", str(self.db_path),
                "--port", str(self.port),
                "--fork",
                "--logpath", str(self.log_file),
                "--bind_ip", "127.0.0.1"
            ]
            print(" ".join(command))
            #a = subprocess.call(command)
            os.system(" ".join(command))
            os.system ("ps aux | fgrep mongod")
            Console.ok("MongoDB has been deployed")
            self.info()

        except Exception, e:
            Console.error("we had a problem starting the  mongo daemon")
            print(e)
            Console.error("MongoDB has stopped")
            # TODO remove the exit in final code, for debugging only
            sys.exit()

    def stop(self):
        try:
            #command = ["mongod", "--shutdown"]
            command = ["killall", "mongod"]
            print(" ".join(command))
            os.system(" ".join(command))
            Console.ok("MongoDB has been shutdown")
            #os.system("mongod --shutdown")
        except Exception, e:
            Console.error("we had a problem shutting the mongo daemon down")
            print(e)

    def info(self):
        # TODO: implement self. dbath, self.port, self.logpath
        Console.ok("Mongo parameters")
        Console.ok("  dbpath:  {:}".format(self.db_path))
        Console.ok("  port:    {:}".format(self.port))
        Console.ok("  logfile: {:}".format(self.log_file))
        Console.ok("  dbname:  {:}".format(self.dbname))

    def connect(self):

        client = MongoClient('localhost', self.port)
        self.database = client["jobsdb"]
        self.jobs = self.database["jobs"]
        self.id = self.database["id"]  # manages the counter for the job

        Console.info("Connecting to the Mongo Database")

    def getid(self):
        pass

    def setid(self):
        pass

    def incrid(self):
        pass

    # job-name is unique

    def insert(self,
               job_name,
               input_filename="",
               output_filename="",
               parameters="",
               job_group=None,
               job_label=None,
               job_id=None,  # possibly same as job_name ?
               host=None,
               start_time=str(datetime.datetime.now()),
               end_time=str(datetime.datetime.now()),
               update_time=str(datetime.datetime.now()),
               job_status="C_DEFINED",  # see what we wrote in paper
):
        # TODO: end_time should be None as unlikely
        # it so short and we need to deal with jobs
        # that we do not know how long they are

        if self.database is not None:

            job = {
                   "_id": job_name,
                   "job_name": job_name,
                   "job_id": job_name,
                   "job_group": job_group,
                   "job_label": job_label,
                   "job_status": job_status,
                   "host": host,
                   "input_filename": input_filename,  # must be array
                   "output_filename": output_filename,  # must be array
                   "start_time": start_time,
                   "end_time": end_time,
                   "update_time": update_time,
                   }

            banner("job")
            pprint(job)
            banner("insert")

            db_job_object = self.jobs.save(job)
            # print (db_job_object)
            # db_job_id = db_job_object
            # print (db_job_id)

            return db_job_object

        else:

            Console.error("Please connect to the database first")
            return -1

    def insert_job_object(self, job):

        if self.database is not None:

            job_id = self.jobs.insert_one(job).inserted_id

            return job_id

        else:
            Console.error("Please connect to the database first")
            return -1

    def find_jobs(self, attribute="", value=""):

        if self.database is not None:

            # Return all jobs if not given search parameters
            if attribute == "" and value == "":
                return self.jobs.find()

            # Return all jobs given the search parameters
            else:
                return self.jobs.find({attribute: value})

        else:
            Console.error("Please connect to the database first")
            return -1

    def delete_jobs(self, attribute="", value=""):

        if self.database is not None:

            # Delete all jobs if no key name or value has been given
            if attribute == "" and value == "":
                self.jobs.remove()

            # Delete only records from a query result if key name and value have been given
            else:
                self.jobs.remove({attribute: value})

        else:
            Console.error("Please connect to the database first")
            return -1

    def count(self, attribute="", value=""):
        """
        return the count of all jobs with a given value at the attribute
        :param attribute:
        :param value:
        :return:
        """
        if self.database is not None:

            # Return the count of all jobs if not given search parameters
            if attribute == "" and value == "":
                return self.jobs.count()

            # Return number of results of the query given the search parameters
            else:
                return self.jobs.find(({attribute: value})).count()

        else:
            Console.error("Please connect to the database first")
            return -1

    def update_job_end_time(self, job_id, end_time=str(datetime.datetime.now())):

        if self.database is not None:

            self.jobs.update({"_id": job_id}, {"$set": {"end_time": end_time}}, upsert=False)

        else:
            Console.error("Please connect to the database first")
            return -1

    def updateJobAttribute(self, job_id, attribute, value):

        if self.database is not None:

            if attribute == "_id":

                print 'You cannot change the unique Object ID generated by MongoDB'

            else:

                self.jobs.update({"_id" : job_id}, {"$set": {attribute : value}}, upsert=False)

        else:

            print "Please connect to the database first"
            return -1

    def jobStatusStats(self, printOutAllJobs=False):

        #Array of different job statuses
        jobStatuses = []

        #Parallel array of counts of each job status
        jobStatusCounts = []

        #Loop through all jobs
        for job in self.findJobs():

            jobStatusFound = False

            index = 0

            #Loop through all existing job statuses
            for jobStatus in jobStatuses:

                #Job status match
                if job["job_status"] == jobStatus:

                    #Increment the counter for this job status
                    jobStatusCounts[index] += 1
                    jobStatusFound = True
                    break

                index += 1

            #New job status
            if jobStatusFound == False:

                #Add new job status and new counter to arrays
                jobStatuses.append(job["job_status"])
                jobStatusCounts.append(1)

        index = 0

        #Print out all job statuses and counts
        for jobStatus in jobStatuses:

            print "JOB STATUS: " + jobStatus + " COUNT: " + str(jobStatusCounts[index])

            #Print out all jobs for this status if flagged to do so
            if printOutAllJobs:

                #Loop through all jobs
                for job in self.findJobs():

                    #Matching job status
                    if job["job_status"] == jobStatus:

                        print job

                #Print an blank line to make the output more pleasing
                print ""

            index += 1

<|endoftext|>