<commit_before>from BaseTasksProvider import BaseTasksProvider
from datetime import datetime, timedelta
import threading
from settingscron import MAX_NUMBER_OF_ACTIVE_DB_SESSIONS, TIMEOUT_WHEN_DB_LOADED
import logging

class SQLAggregator(BaseTasksProvider):

    lock = threading.RLock()
    logger = logging.getLogger(__name__ + ' SQLAggregator')

    def processPayload(self):

        self.logger.info("processPayload started")

        while (self.getNumberOfActiveDBSessions() > MAX_NUMBER_OF_ACTIVE_DB_SESSIONS):
            threading.sleep(TIMEOUT_WHEN_DB_LOADED)

        quotas = ('cpua1', 'cpua7', 'cpup1', 'cpup7')
        ## Pick up the users with nonzero usage and zero them out
        try:
            query = "SELECT name,dn FROM ATLAS_PANDAMETA.USERS where cpua1>0 or cpua7>0 or cpup1>0 or cpup7>0"
            db = self.pool.acquire()
            cursor = db.cursor()
            rows = cursor.execute(query)
        except Exception as e:
            self.logger.error(e)
            return -1

        users = {}
        for r in rows:
            users[r[0]] = {}
            users[r[0]]['name'] = r[0]
            for var_name in quotas:
                users[r[0]][var_name] = 0

        for days in (1, 7):
            current_time = datetime.utcnow()
            start_time = current_time - timedelta(days=days)

            tables = ('ATLAS_PANDA.JOBSACTIVE4', 'ATLAS_PANDA.JOBSARCHIVED4', 'ATLAS_PANDAARCH.JOBSARCHIVED')  # List of Tables

            for t in tables:  # For each table get the info from Database
                try:
                    # utils.initDB(t)
                    query = "SELECT prodUserName, SUM(cpuConsumptionTime) as cpuConsumptionTime, workingGroup \
                                FROM %s WHERE modificationTime > :start_time AND (prodSourceLabel = 'user' OR \
                                prodSourceLabel = 'panda') AND jobStatus != 'cancelled' GROUP BY workingGroup, prodUserName" % (t)
                    rows = cursor.execute(query, {'start_time':start_time})
                except Exception as e:
                    self.logger.error(e)
                    return -1

                for r in rows:
                    user = r[0].replace("'", "")
                    cpuconsumption = (r[1])
                    if not r[2] is None:
                        ## Include in group production stats
                        var_name = 'cpup%s' % days
                    else:
                        ## Include in personal analysis stats
                        var_name = 'cpua%s' % days
                    if user in users:
                        if not var_name in users[user]: users[user][var_name] = 0
                    else:
                        users[user] = {}
                        users[user]['name'] = user
                        users[user][var_name] = 0
                    users[user][var_name] += cpuconsumption

        userlist = []
        for u in users:
            userdict = {}
            userdict['name'] = u
            for days in (1, 7):
                for p in ('cpup%s' % days, 'cpua%s' % days):
                    if p in users[u]: userdict[p] = users[u][p]
            userlist.append(userdict)
        try:
            cursor.executemany("UPDATE ATLAS_PANDAMETA.USERS SET cpua1 = :cpua1, cpua7 = :cpua7, cpup1 = :cpup1, cpup7 = :cpup7 where name = :name", userlist)
            db.commit()
        except Exception as e:
            self.logger.error(e)

        cursor.close()
        return 0

<commit_msg>added log message<commit_after>from BaseTasksProvider import BaseTasksProvider
from datetime import datetime, timedelta
import threading
from settingscron import MAX_NUMBER_OF_ACTIVE_DB_SESSIONS, TIMEOUT_WHEN_DB_LOADED
import logging

class SQLAggregator(BaseTasksProvider):

    lock = threading.RLock()
    logger = logging.getLogger(__name__ + ' SQLAggregator')

    def processPayload(self):

        self.logger.info("SQLAggregator started")

        while (self.getNumberOfActiveDBSessions() > MAX_NUMBER_OF_ACTIVE_DB_SESSIONS):
            threading.sleep(TIMEOUT_WHEN_DB_LOADED)

        quotas = ('cpua1', 'cpua7', 'cpup1', 'cpup7')
        ## Pick up the users with nonzero usage and zero them out
        try:
            query = "SELECT name,dn FROM ATLAS_PANDAMETA.USERS where cpua1>0 or cpua7>0 or cpup1>0 or cpup7>0"
            db = self.pool.acquire()
            cursor = db.cursor()
            rows = cursor.execute(query)
        except Exception as e:
            self.logger.error(e)
            return -1

        users = {}
        for r in rows:
            users[r[0]] = {}
            users[r[0]]['name'] = r[0]
            for var_name in quotas:
                users[r[0]][var_name] = 0

        for days in (1, 7):
            current_time = datetime.utcnow()
            start_time = current_time - timedelta(days=days)

            tables = ('ATLAS_PANDA.JOBSACTIVE4', 'ATLAS_PANDA.JOBSARCHIVED4', 'ATLAS_PANDAARCH.JOBSARCHIVED')  # List of Tables

            for t in tables:  # For each table get the info from Database
                try:
                    # utils.initDB(t)
                    query = "SELECT prodUserName, SUM(cpuConsumptionTime) as cpuConsumptionTime, workingGroup \
                                FROM %s WHERE modificationTime > :start_time AND (prodSourceLabel = 'user' OR \
                                prodSourceLabel = 'panda') AND jobStatus != 'cancelled' GROUP BY workingGroup, prodUserName" % (t)
                    rows = cursor.execute(query, {'start_time':start_time})
                except Exception as e:
                    self.logger.error(e)
                    return -1

                for r in rows:
                    user = r[0].replace("'", "")
                    cpuconsumption = (r[1])
                    if not r[2] is None:
                        ## Include in group production stats
                        var_name = 'cpup%s' % days
                    else:
                        ## Include in personal analysis stats
                        var_name = 'cpua%s' % days
                    if user in users:
                        if not var_name in users[user]: users[user][var_name] = 0
                    else:
                        users[user] = {}
                        users[user]['name'] = user
                        users[user][var_name] = 0
                    users[user][var_name] += cpuconsumption

        userlist = []
        for u in users:
            userdict = {}
            userdict['name'] = u
            for days in (1, 7):
                for p in ('cpup%s' % days, 'cpua%s' % days):
                    if p in users[u]: userdict[p] = users[u][p]
            userlist.append(userdict)
        try:
            cursor.executemany("UPDATE ATLAS_PANDAMETA.USERS SET cpua1 = :cpua1, cpua7 = :cpua7, cpup1 = :cpup1, cpup7 = :cpup7 where name = :name", userlist)
            db.commit()
        except Exception as e:
            self.logger.error(e)

        self.logger.info("SQLAggregator finished")
        cursor.close()
        return 0

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
cat wekex_test_plain | python ../notebooks/kilogram/mapreduce/wikipedia/resolve_entities.py > wekex_test_linked
"""

import sys
from kilogram.dataset.dbpedia import NgramEntityResolver

ner = NgramEntityResolver("/home/roman/dbpedia/dbpedia_types.txt", "/home/roman/dbpedia/dbpedia_uri_excludes.txt", "/home/roman/dbpedia/dbpedia_lower_includes.txt", "/home/roman/dbpedia/dbpedia_redirects.txt", "/home/roman/dbpedia/dbpedia_2015-04.owl")

for line in sys.stdin:
    line = line.strip()
    # split the line into words
    new_words = ner.resolve_entities(line.split())
    new_line = ' '.join(new_words)
    print new_line
<commit_msg>fix resolve entities<commit_after>#!/usr/bin/env python
"""
cat wekex_test_plain | python ../notebooks/kilogram/mapreduce/wikipedia/resolve_entities.py > wekex_test_linked
"""

import sys
from kilogram.dataset.dbpedia import NgramEntityResolver

ner = NgramEntityResolver("/Users/dragoon/Downloads/dbpedia/dbpedia_data.txt",
                          "/Users/dragoon/Downloads/dbpedia/dbpedia_uri_excludes.txt",
                          "/Users/dragoon/Downloads/dbpedia/dbpedia_lower_includes.txt",
                          "/Users/dragoon/Downloads/dbpedia/dbpedia_2015-04.owl")

for line in sys.stdin:
    line = line.strip()
    # split the line into words
    new_words = ner.resolve_entities(line.split())
    new_line = ' '.join(new_words)
    print new_line
<|endoftext|>


<commit_before># Copyright 2018 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Run YCSB benchmark against AWS DynamoDB.

This benchmark does not provision VMs for the corresponding DynamboDB database.
The only VM group is client group that sends requests to specifiedDB.
TODO: add DAX option.
TODO: add global table option.
"""

import os

from perfkitbenchmarker import configs
from perfkitbenchmarker import flags
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.linux_packages import ycsb
from perfkitbenchmarker.providers.aws import aws_dynamodb

FLAGS = flags.FLAGS

BENCHMARK_NAME = 'aws_dynamodb_ycsb'
BENCHMARK_CONFIG = """
aws_dynamodb_ycsb:
  description: >
      Run YCSB against AWS DynamoDB.
      Configure the number of VMs via --ycsb_client_vms.
  vm_groups:
    default:
      vm_spec: *default_single_core
      vm_count: 1"""

flags.DEFINE_boolean('aws_dynamodb_ycsb_consistentReads',
                     False,
                     "Consistent reads cost 2x eventual reads. "
                     "'false' is default which is eventual")

def GetConfig(user_config):
  config = configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)
  if FLAGS['ycsb_client_vms'].present:
    config['vm_groups']['default']['vm_count'] = FLAGS.ycsb_client_vms
  return config

def CheckPrerequisites(benchmark_config):
  """Verifies that the required resources are present.

  Args:
    benchmark_config: Unused.
  Raises:
    perfkitbenchmarker.data.ResourceNotFound: On missing resource.
  """
  del benchmark_config
  ycsb.CheckPrerequisites()

def Prepare(benchmark_spec):
  """Install YCSB on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.
  """
  benchmark_spec.always_call_cleanup = True

  benchmark_spec.dynamodb_instance = aws_dynamodb.AwsDynamoDBInstance(
      table_name='pkb-{0}'.format(FLAGS.run_uri))
  benchmark_spec.dynamodb_instance.Create()

  vms = benchmark_spec.vms
  # Install required packages.
  vm_util.RunThreaded(_Install, vms)
  benchmark_spec.executor = ycsb.YCSBExecutor('dynamodb')

def Run(benchmark_spec):
  """Run YCSB on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.
  Returns:
    A list of sample.Sample objects.
  """
  vms = benchmark_spec.vms

  run_kwargs = {
      'dynamodb.awsCredentialsFile': GetRemoteVMCredentialsFullPath(vms[0]),
      'dynamodb.primaryKey': FLAGS.aws_dynamodb_primarykey,
      'dynamodb.endpoint': benchmark_spec.dynamodb_instance.GetEndPoint(),
      'table': 'pkb-{0}'.format(FLAGS.run_uri),
  }
  if FLAGS.aws_dynamodb_use_sort:
    run_kwargs.update({'dynamodb.primaryKeyType': 'HASH_AND_RANGE',
                       'dynamodb.hashKeyName': FLAGS.aws_dynamodb_primarykey,
                       'dynamodb.primaryKey': FLAGS.aws_dynamodb_sortkey})
  if FLAGS.aws_dynamodb_ycsb_consistentReads:
    run_kwargs.update({'dynamodb.consistentReads': 'true'})
  load_kwargs = run_kwargs.copy()
  if FLAGS['ycsb_preload_threads'].present:
    load_kwargs['threads'] = FLAGS['ycsb_preload_threads']
  samples = list(benchmark_spec.executor.LoadAndRun(
      vms, load_kwargs=load_kwargs, run_kwargs=run_kwargs))
  return samples

def Cleanup(benchmark_spec):
  """Cleanup YCSB on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
    required to run the benchmark.
  """
  benchmark_spec.dynamodb_instance.Delete()

def GetRemoteVMCredentialsFullPath(vm):
  """Returns the full path for AWS credentials file."""
  home_dir, _ = vm.RemoteCommand('echo ~')
  # aws credentials are always located in the 'credentials' directory of .aws
  # configurations folder
  # https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html
  return os.path.join(home_dir.rstrip('\n'),
                      FLAGS.aws_credentials_remote_path, 'credentials')

def _Install(vm):
  """Install YCSB on client 'vm'."""
  vm.Install('ycsb')
  # copy AWS creds
  vm.Install('aws_credentials')
  # aws credentials file format to ycsb recognized format
  vm.RemoteCommand('sed -i "s/aws_access_key_id/accessKey/g" {0}'.format(
      GetRemoteVMCredentialsFullPath(vm)))
  vm.RemoteCommand('sed -i "s/aws_secret_access_key/secretKey/g" {0}'.format(
      GetRemoteVMCredentialsFullPath(vm)))
<commit_msg>Update dynamodb run config.<commit_after># Copyright 2018 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Run YCSB benchmark against AWS DynamoDB.

This benchmark does not provision VMs for the corresponding DynamboDB database.
The only VM group is client group that sends requests to specifiedDB.
TODO: add DAX option.
TODO: add global table option.
"""

import os

from perfkitbenchmarker import configs
from perfkitbenchmarker import flags
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.linux_packages import ycsb
from perfkitbenchmarker.providers.aws import aws_dynamodb

FLAGS = flags.FLAGS

BENCHMARK_NAME = 'aws_dynamodb_ycsb'
BENCHMARK_CONFIG = """
aws_dynamodb_ycsb:
  description: >
      Run YCSB against AWS DynamoDB.
      Configure the number of VMs via --ycsb_client_vms.
  vm_groups:
    default:
      vm_spec: *default_single_core
      vm_count: 1"""

flags.DEFINE_boolean('aws_dynamodb_ycsb_consistentReads',
                     False,
                     "Consistent reads cost 2x eventual reads. "
                     "'false' is default which is eventual")

def GetConfig(user_config):
  config = configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)
  if FLAGS['ycsb_client_vms'].present:
    config['vm_groups']['default']['vm_count'] = FLAGS.ycsb_client_vms
  return config

def CheckPrerequisites(benchmark_config):
  """Verifies that the required resources are present.

  Args:
    benchmark_config: Unused.
  Raises:
    perfkitbenchmarker.data.ResourceNotFound: On missing resource.
  """
  del benchmark_config
  ycsb.CheckPrerequisites()

def Prepare(benchmark_spec):
  """Install YCSB on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.
  """
  benchmark_spec.always_call_cleanup = True

  benchmark_spec.dynamodb_instance = aws_dynamodb.AwsDynamoDBInstance(
      table_name='pkb-{0}'.format(FLAGS.run_uri))
  benchmark_spec.dynamodb_instance.Create()

  vms = benchmark_spec.vms
  # Install required packages.
  vm_util.RunThreaded(_Install, vms)
  benchmark_spec.executor = ycsb.YCSBExecutor('dynamodb')

def Run(benchmark_spec):
  """Run YCSB on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.
  Returns:
    A list of sample.Sample objects.
  """
  vms = benchmark_spec.vms

  run_kwargs = {
      'dynamodb.awsCredentialsFile': GetRemoteVMCredentialsFullPath(vms[0]),
      'dynamodb.primaryKey': FLAGS.aws_dynamodb_primarykey,
      'dynamodb.endpoint': benchmark_spec.dynamodb_instance.GetEndPoint(),
      'table': 'pkb-{0}'.format(FLAGS.run_uri),
  }
  if FLAGS.aws_dynamodb_use_sort:
    run_kwargs.update({'dynamodb.primaryKeyType': 'HASH_AND_RANGE',
                       'dynamodb.hashKeyName': FLAGS.aws_dynamodb_primarykey,
                       'dynamodb.primaryKey': FLAGS.aws_dynamodb_sortkey})
  if FLAGS.aws_dynamodb_ycsb_consistentReads:
    run_kwargs.update({'dynamodb.consistentReads': 'true'})
  load_kwargs = run_kwargs.copy()
  if FLAGS['ycsb_preload_threads'].present:
    load_kwargs['threads'] = FLAGS['ycsb_preload_threads']
  samples = list(benchmark_spec.executor.LoadAndRun(
      vms, load_kwargs=load_kwargs, run_kwargs=run_kwargs))
  return samples

def Cleanup(benchmark_spec):
  """Cleanup YCSB on the target vm.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
    required to run the benchmark.
  """
  benchmark_spec.dynamodb_instance.Delete()

def GetRemoteVMCredentialsFullPath(vm):
  """Returns the full path for AWS credentials file."""
  home_dir, _ = vm.RemoteCommand('echo ~')
  search_path = os.path.join(home_dir.rstrip('\n'),
                             FLAGS.aws_credentials_remote_path)
  result, _ = vm.RemoteCommand('grep -irl "key" {0}'.format(search_path))
  return result.rstrip('\n')

def _Install(vm):
  """Install YCSB on client 'vm'."""
  vm.Install('ycsb')
  # copy AWS creds
  vm.Install('aws_credentials')
  # aws credentials file format to ycsb recognized format
  vm.RemoteCommand('sed -i "s/aws_access_key_id/accessKey/g" {0}'.format(
      GetRemoteVMCredentialsFullPath(vm)))
  vm.RemoteCommand('sed -i "s/aws_secret_access_key/secretKey/g" {0}'.format(
      GetRemoteVMCredentialsFullPath(vm)))
<|endoftext|>


<commit_before># MIT License
# Copyright (c) 2017 MassChallenge, Inc.

from impact.models import UserRole
from impact.v1.events.base_user_role_grant_event import (
    BaseUserRoleGrantEvent,
)

class BaseUserBecameMentorEvent(BaseUserRoleGrantEvent):
    DESCRIPTION_FORMAT = "Became {role_name} for Program {name} ({id})"

    def description(self):
        program = self.program_role_grant.program_role.program
        return self.DESCRIPTION_FORMAT.format(role_name=self.ROLE_NAME,
                                              name=program.name,
                                              id=program.id)
<commit_msg>[AC-5147] remove unused import<commit_after># MIT License
# Copyright (c) 2017 MassChallenge, Inc.

from impact.v1.events.base_user_role_grant_event import (
    BaseUserRoleGrantEvent,
)

class BaseUserBecameMentorEvent(BaseUserRoleGrantEvent):
    DESCRIPTION_FORMAT = "Became {role_name} for Program {name} ({id})"

    def description(self):
        program = self.program_role_grant.program_role.program
        return self.DESCRIPTION_FORMAT.format(role_name=self.ROLE_NAME,
                                              name=program.name,
                                              id=program.id)
<|endoftext|>


<commit_before>from .models import (
    Dossier,
    Proposal,
    Vote
)

from rest_framework import (
    filters,
    viewsets,
)

from representatives_votes.serializers import (
    DossierDetailSerializer,
    DossierSerializer,
    ProposalDetailSerializer,
    ProposalSerializer,
    VoteSerializer,
)

class DossierViewSet(viewsets.ReadOnlyModelViewSet):
    """
    API endpoint that allows dossiers to be viewed.
    """

    queryset = Dossier.objects.all()
    serializer_class = DossierSerializer

    filter_backends = (
        filters.DjangoFilterBackend,
        filters.SearchFilter,
        filters.OrderingFilter
    )

    filter_fields = {
        'fingerprint': ['exact'],
        'title': ['exact', 'icontains'],
        'reference': ['exact', 'icontains'],
    }

    search_fields = (
        'title',
        'fingerprint',
        'reference',
        'text',
        'proposals__title'
    )

    ordering_fields = ('id', 'reference')

    def list(self, request):
        return super(DossierViewSet, self).list(request)

    def retrieve(self, request, pk=None):
        self.serializer_class = DossierDetailSerializer
        return super(DossierViewSet, self).retrieve(request, pk)

class ProposalViewSet(viewsets.ReadOnlyModelViewSet):
    """
    API endpoint that allows proposals to be viewed.
    """

    queryset = Proposal.objects.select_related('dossier')
    serializer_class = ProposalSerializer

    filter_backends = (
        filters.DjangoFilterBackend,
        filters.SearchFilter,
        filters.OrderingFilter
    )

    filter_fields = {
        'fingerprint': ['exact'],
        'dossier__fingerprint': ['exact'],
        'title': ['exact', 'icontains'],
        'description': ['icontains'],
        'reference': ['exact', 'icontains'],
        'datetime': ['exact', 'gte', 'lte'],
        'kind': ['exact'],
    }

    search_fields = (
        'title',
        'fingerprint', 'reference',
        'dossier__fingerprint',
        'dossier__title',
        'dossier__reference'
    )

    ordering_fields = ('id', 'reference')

    def list(self, request):
        return super(ProposalViewSet, self).list(request)

    def retrieve(self, request, pk=None):
        self.serializer_class = ProposalDetailSerializer
        return super(ProposalViewSet, self).retrieve(request, pk)

class VoteViewSet(viewsets.ReadOnlyModelViewSet):
    """
    API endpoint that allows proposals to be viewed.
    """

    queryset = Vote.objects.select_related('representative', 'proposal')
    serializer_class = VoteSerializer

    filter_backends = (
        filters.DjangoFilterBackend,
        filters.SearchFilter,
        filters.OrderingFilter
    )

    filter_fields = {
        'proposal__fingerprint': ['exact'],
        'position': ['exact'],
        'representative_name': ['exact', 'icontains'],
        'representative': ['exact']
    }

    def list(self, request):
        return super(VoteViewSet, self).list(request)

    def retrieve(self, request, pk=None):
        return super(VoteViewSet, self).retrieve(request, pk)
<commit_msg>Added default pagination for web browser<commit_after>from .models import (
    Dossier,
    Proposal,
    Vote
)

from rest_framework import (
    filters,
    viewsets,
)

from representatives.api import DefaultWebPagination

from representatives_votes.serializers import (
    DossierDetailSerializer,
    DossierSerializer,
    ProposalDetailSerializer,
    ProposalSerializer,
    VoteSerializer,
)

class DossierViewSet(viewsets.ReadOnlyModelViewSet):
    """
    API endpoint that allows dossiers to be viewed.
    """

    pagination_class = DefaultWebPagination
    queryset = Dossier.objects.all()
    serializer_class = DossierSerializer

    filter_backends = (
        filters.DjangoFilterBackend,
        filters.SearchFilter,
        filters.OrderingFilter
    )

    filter_fields = {
        'fingerprint': ['exact'],
        'title': ['exact', 'icontains'],
        'reference': ['exact', 'icontains'],
    }

    search_fields = (
        'title',
        'fingerprint',
        'reference',
        'text',
        'proposals__title'
    )

    ordering_fields = ('id', 'reference')

    def list(self, request):
        return super(DossierViewSet, self).list(request)

    def retrieve(self, request, pk=None):
        self.serializer_class = DossierDetailSerializer
        return super(DossierViewSet, self).retrieve(request, pk)

class ProposalViewSet(viewsets.ReadOnlyModelViewSet):
    """
    API endpoint that allows proposals to be viewed.
    """

    pagination_class = DefaultWebPagination
    queryset = Proposal.objects.select_related('dossier')
    serializer_class = ProposalSerializer

    filter_backends = (
        filters.DjangoFilterBackend,
        filters.SearchFilter,
        filters.OrderingFilter
    )

    filter_fields = {
        'fingerprint': ['exact'],
        'dossier__fingerprint': ['exact'],
        'title': ['exact', 'icontains'],
        'description': ['icontains'],
        'reference': ['exact', 'icontains'],
        'datetime': ['exact', 'gte', 'lte'],
        'kind': ['exact'],
    }

    search_fields = (
        'title',
        'fingerprint', 'reference',
        'dossier__fingerprint',
        'dossier__title',
        'dossier__reference'
    )

    ordering_fields = ('id', 'reference')

    def list(self, request):
        return super(ProposalViewSet, self).list(request)

    def retrieve(self, request, pk=None):
        self.serializer_class = ProposalDetailSerializer
        return super(ProposalViewSet, self).retrieve(request, pk)

class VoteViewSet(viewsets.ReadOnlyModelViewSet):
    """
    API endpoint that allows proposals to be viewed.
    """

    pagination_class = DefaultWebPagination
    queryset = Vote.objects.select_related('representative', 'proposal')
    serializer_class = VoteSerializer

    filter_backends = (
        filters.DjangoFilterBackend,
        filters.SearchFilter,
        filters.OrderingFilter
    )

    filter_fields = {
        'proposal__fingerprint': ['exact'],
        'position': ['exact'],
        'representative_name': ['exact', 'icontains'],
        'representative': ['exact']
    }

    def list(self, request):
        return super(VoteViewSet, self).list(request)

    def retrieve(self, request, pk=None):
        return super(VoteViewSet, self).retrieve(request, pk)
<|endoftext|>


<commit_before>from django import forms
from django.contrib.auth import get_user_model

from decisiontree.multitenancy.forms import TenancyModelForm

from .. import models
from .fields import TagField

class AnswerCreateUpdateForm(TenancyModelForm):

    class Meta:
        model = models.Answer
        fields = ['name', 'type', 'answer', 'description']

class AnswerSearchForm(forms.Form):
    # ANALYSIS_TYPES = (
    #     ('A', 'Mean'),
    #     ('R', 'Median'),
    #     ('C', 'Mode'),
    # )
    # answer = forms.ModelChoiceField(queryset=models.Answer.objects.none())
    # analysis = forms.ChoiceField(choices=ANALYSIS_TYPES)
    tag = forms.ModelChoiceField(
        required=False, empty_label="All Tags",
        queryset=models.Tag.objects.none())

    def __init__(self, *args, **kwargs):
        tree = kwargs.pop('tree')
        super(AnswerSearchForm, self).__init__(*args, **kwargs)
        # answers = models.Answer.objects.filter(transitions__entries__session__tree=tree)
        tags = models.Tag.objects.filter(entries__session__tree=tree).distinct()

        # self.fields['answer'].queryset = answers.distinct()
        self.fields['tag'].queryset = tags
        # self.fields['analysis'].label = 'Calculator'
        # self.fields['tag'].label = 'Calculator'

class EntryTagForm(TenancyModelForm):
    tags = TagField()

    class Meta:
        model = models.Entry
        fields = ['tags']

    def save(self):
        entry = super(EntryTagForm, self).save()
        # create tag notifications
        models.TagNotification.create_from_entry(entry)
        return entry

class PathCreateUpdateForm(TenancyModelForm):
    tags = TagField(required=False)

    class Meta:
        model = models.Transition
        fields = ['current_state', 'answer', 'next_state', 'tags']

    def __init__(self, *args, **kwargs):
        super(PathCreateUpdateForm, self).__init__(*args, **kwargs)
        states = models.TreeState.objects.select_related('question')
        states = states.order_by('question__text')
        self.fields['current_state'].queryset = states
        self.fields['current_state'].label = 'Current State'
        self.fields['answer'].label = 'Answer'
        self.fields['answer'].queryset = models.Answer.objects.order_by('answer')
        self.fields['next_state'].label = 'Next State'
        self.fields['next_state'].queryset = states
        self.fields['tags'].label = 'Auto tags'

class QuestionCreateUpdateForm(TenancyModelForm):

    class Meta:
        model = models.Question
        fields = ['text', 'error_response']

class StateCreateUpdateForm(TenancyModelForm):

    class Meta:
        model = models.TreeState
        fields = ['name', 'question', 'num_retries']

class SurveyCreateUpdateForm(TenancyModelForm):

    class Meta:
        model = models.Tree
        fields = ['trigger', 'root_state', 'completion_text', 'summary']

    def __init__(self, *args, **kwargs):
        super(SurveyCreateUpdateForm, self).__init__(*args, **kwargs)
        root_state = self.fields['root_state']
        root_state.label = 'First State'
        root_state.queryset = root_state.queryset.select_related('question')
        root_state.queryset = root_state.queryset.order_by('question__text')
        self.fields['summary'].widget = forms.Textarea()

class TagCreateUpdateForm(TenancyModelForm):

    class Meta:
        model = models.Tag
        fields = ['name', 'recipients']

    def __init__(self, *args, **kwargs):
        super(TagCreateUpdateForm, self).__init__(*args, **kwargs)
        self.fields['recipients'] = forms.ModelMultipleChoiceField(
            required=False, widget=forms.CheckboxSelectMultiple,
            queryset=get_user_model().objects.exclude(email=''))
<commit_msg>Use textarea widgets<commit_after>from django import forms
from django.contrib.auth import get_user_model

from decisiontree.multitenancy.forms import TenancyModelForm

from .. import models
from .fields import TagField

class AnswerCreateUpdateForm(TenancyModelForm):

    class Meta:
        model = models.Answer
        fields = ['name', 'type', 'answer', 'description']

class AnswerSearchForm(forms.Form):
    # ANALYSIS_TYPES = (
    #     ('A', 'Mean'),
    #     ('R', 'Median'),
    #     ('C', 'Mode'),
    # )
    # answer = forms.ModelChoiceField(queryset=models.Answer.objects.none())
    # analysis = forms.ChoiceField(choices=ANALYSIS_TYPES)
    tag = forms.ModelChoiceField(
        required=False, empty_label="All Tags",
        queryset=models.Tag.objects.none())

    def __init__(self, *args, **kwargs):
        tree = kwargs.pop('tree')
        super(AnswerSearchForm, self).__init__(*args, **kwargs)
        # answers = models.Answer.objects.filter(transitions__entries__session__tree=tree)
        tags = models.Tag.objects.filter(entries__session__tree=tree).distinct()

        # self.fields['answer'].queryset = answers.distinct()
        self.fields['tag'].queryset = tags
        # self.fields['analysis'].label = 'Calculator'
        # self.fields['tag'].label = 'Calculator'

class EntryTagForm(TenancyModelForm):
    tags = TagField()

    class Meta:
        model = models.Entry
        fields = ['tags']

    def save(self):
        entry = super(EntryTagForm, self).save()
        # create tag notifications
        models.TagNotification.create_from_entry(entry)
        return entry

class PathCreateUpdateForm(TenancyModelForm):
    tags = TagField(required=False)

    class Meta:
        model = models.Transition
        fields = ['current_state', 'answer', 'next_state', 'tags']

    def __init__(self, *args, **kwargs):
        super(PathCreateUpdateForm, self).__init__(*args, **kwargs)
        states = models.TreeState.objects.select_related('question')
        states = states.order_by('question__text')
        self.fields['current_state'].queryset = states
        self.fields['current_state'].label = 'Current State'
        self.fields['answer'].label = 'Answer'
        self.fields['answer'].queryset = models.Answer.objects.order_by('answer')
        self.fields['next_state'].label = 'Next State'
        self.fields['next_state'].queryset = states
        self.fields['tags'].label = 'Auto tags'

class QuestionCreateUpdateForm(TenancyModelForm):

    class Meta:
        model = models.Question
        fields = ['text', 'error_response']

    def __init__(self, *args, **kwargs):
        super(QuestionCreateUpdateForm, self).__init__(*args, **kwargs)
        self.fields['text'].widget = forms.Textarea()
        self.fields['error_response'].widget = forms.Textarea()

class StateCreateUpdateForm(TenancyModelForm):

    class Meta:
        model = models.TreeState
        fields = ['name', 'question', 'num_retries']

class SurveyCreateUpdateForm(TenancyModelForm):

    class Meta:
        model = models.Tree
        fields = ['trigger', 'root_state', 'completion_text', 'summary']

    def __init__(self, *args, **kwargs):
        super(SurveyCreateUpdateForm, self).__init__(*args, **kwargs)
        root_state = self.fields['root_state']
        root_state.label = 'First State'
        root_state.queryset = root_state.queryset.select_related('question')
        root_state.queryset = root_state.queryset.order_by('question__text')
        self.fields['completion_text'].widget = forms.Textarea()
        self.fields['summary'].widget = forms.Textarea()

class TagCreateUpdateForm(TenancyModelForm):

    class Meta:
        model = models.Tag
        fields = ['name', 'recipients']

    def __init__(self, *args, **kwargs):
        super(TagCreateUpdateForm, self).__init__(*args, **kwargs)
        self.fields['recipients'] = forms.ModelMultipleChoiceField(
            required=False, widget=forms.CheckboxSelectMultiple,
            queryset=get_user_model().objects.exclude(email=''))
<|endoftext|>


<commit_before>from django.http import HttpResponse
from rdrf.models.definition.models import RegistryForm
from rdrf.models.definition.models import Section
from rdrf.models.definition.models import CommonDataElement
from rdrf.models.definition.models import ContextFormGroup
from rdrf.models.definition.models import RDRFContext
from rdrf.models.definition.models import ClinicalData
from rdrf.helpers.utils import cde_completed
from rdrf.helpers.utils import format_date
from registry.patients.models import Patient
from registry.patients.models import ConsentValue
import logging
import csv

logger = logging.getLogger(__name__)

class SecurityException(Exception):
    pass

class ReportParserException(Exception):
    pass

class ColumnType:
    DEMOGRAPHICS = "demographics"
    CDE = "cde"
    COMPLETION = "completion"
    COMPLETION_PERCENTAGE = "%"
    CONSENT = "consent"
    CONSENT_DATE = "consent_date"
    FOLLOWUP_DATE = "followup_date"

demographics_transform_map = {"sex": {"1": "Male", "2": "Female", "3": "Indeterminate"},
                              "date_of_birth": format_date,
                              }

def aus_date(american_date):
    if not american_date:
        return ""
    year, month, day = american_date.split("-")
    return "%s-%s-%s" % (day, month, year)

def get_timestamp(clinical_data):
    from rdrf.helpers.utils import parse_iso_datetime
    if not clinical_data:
        return None
    if not clinical_data.data:
        return None

    return parse_iso_datetime(clinical_data.data.get("timestamp", None))

class ReportGenerator:
    def __init__(self, registry_model, report_name, report_spec, user):
        import json
        self.registry_model = registry_model
        self.report_name = report_name
        self.report_spec = json.loads(report_spec)
        self.user = user
        self.form_names = None
        self.start_date = None
        self.finish_date = None
        self.data = self._get_all_data()
        self.context_form_group = self._get_context_form_group()
        self.report = None

    def dump_csv(self, stream):
        writer = csv.writer(stream)
        writer.writerows(self.report)
        return stream

    def _get_context(self, patient_model):
        # get the fixed context associated with the context
        # form group specified in the report spec.
        # there should only be one for each patient
        context_models = RDRFContext.objects.filter(registry=self.registry_model,
                                                    object_id=patient_model.id,
                                                    context_form_group=self.context_form_group)
        num_contexts = len(context_models)
        if num_contexts == 1:
            return context_models[0]

    def _get_context_form_group(self):
        if "context_form_group" in self.report_spec:
            form_group_name = self.report_spec["context_form_group"]
            cfg = ContextFormGroup.objects.get(name=form_group_name,
                                               registry=self.registry_model,
                                               context_type="F")
            return cfg

    def _get_all_data(self):
        return []

    def generate_report(self):
        self._security_check()
        return self._run_report()

    def _run_report(self):
        rows = []
        rows.append(self._get_header())
        for patient_model in self._get_patients():
            # the context needs to be determined by the report spec
            # as it contains the context_form_group name
            context_model = self._get_context(patient_model)
            data = self._load_patient_data(patient_model, context_model.id)
            row = []
            for column in self.report_spec["columns"]:
                column_value = self._get_column_value(patient_model, data, column)
                row.append(column_value)
            rows.append(row)
        self.report = rows

    def _get_header(self):
        def header(col):
            if "label" in col:
                return col["label"]
            return col["name"]
        return [header(col) for col in self.report_spec["columns"]]

    def _get_column_value(self, patient_model, data, column):
        column_type = column["type"]
        if column_type == ColumnType.DEMOGRAPHICS:
            column_name = column["name"]
            return self._get_demographics_column(patient_model, column_name)
        if column_type == ColumnType.COMPLETION:
            form_name = column["name"]
            return self._completed(patient_model, form_name, data)
        if column_type == ColumnType.COMPLETION_PERCENTAGE:
            form_name = column["name"]
            return self._completed(patient_model, form_name, data, percentage=True)
        if column_type == ColumnType.CDE:
            cde_path = column["name"]
            try:
                return self._get_cde(patient_model, cde_path, data)
            except KeyError:
                return "[Not Entered]"
        if column_type == ColumnType.CONSENT:
            consent_section_code, consent_code = column["name"].split("/")
            return self._get_consent(patient_model,
                                     consent_section_code,
                                     consent_code)
        if column_type == ColumnType.CONSENT_DATE:
            consent_section_code, consent_code = column["name"].split("/")
            return self._get_consent(patient_model,
                                     consent_section_code,
                                     consent_code,
                                     get_date=True)
        if column_type == ColumnType.FOLLOWUP_DATE:
            context_form_group_name = column["context_form_group"]
            form_name = column["name"]
            return self._get_followup_date(patient_model,
                                           context_form_group_name,
                                           form_name)
        else:
            raise ReportParserException("Unknown column type: %s" % column_type)

    def _get_followup_date(self,
                           patient_model,
                           context_form_group_name,
                           form_name):
        from rdrf.models.definition.models import ContextFormGroup
        cfg = ContextFormGroup.objects.get(registry=self.registry_model,
                                           name=context_form_group_name)
        # find the last/latest context containing the form
        context_models = [c for c in patient_model.context_models
                          if c.registry.pk == self.registry_model.pk and c.context_form_group and c.context_form_group.pk == cfg.pk]
        if not context_models:
            return ""
        latest_date = None
        for context_model in context_models:
            # get the associated clinical data and check the timestamp
            try:
                clinical_data = ClinicalData.objects.get(context_id=context_model.pk,
                                                         django_model="Patient",
                                                         collection="cdes",
                                                         django_id=patient_model.pk)
                timestamp = get_timestamp(clinical_data)
                if latest_date is None or timestamp > latest_date:
                    latest_date = timestamp
            except ClinicalData.DoesNotExist:
                pass
        if latest_date is None:
            return ""
        return format_date(latest_date)

    def _get_consent(self,
                     patient_model,
                     consent_section_code,
                     consent_code,
                     get_date=False):
        for consent_section in self.registry_model.consent_sections.all():
            if consent_section.code == consent_section_code:
                for consent_question in consent_section.questions.all():
                    if consent_question.code == consent_code:
                        try:
                            consent_value = ConsentValue.objects.get(patient=patient_model,
                                                                     consent_question=consent_question)
                            if get_date:
                                first_save = consent_value.first_save
                                last_update = consent_value.last_update
                                if not last_update:
                                    return format_date(first_save)
                                return format_date(last_update)
                            return "True" if consent_value else "False"
                        except ConsentValue.DoesNotExist:
                            if get_date:
                                return ""
                            return "False"
        if get_date:
            return ""
        return "False"

    def _get_cde(self, patient_model, cde_path, data):
        if "/" in cde_path:
            form_name, section_code, cde_code = cde_path.split("/")
            form_model = RegistryForm.objects.get(name=form_name,
                                                  registry=self.registry_model)
            section_model = Section.objects.get(code=section_code)
            cde_model = CommonDataElement.objects.get(code=cde_code)
        else:
            form_model, section_model, cde_model = self._find_cde(cde_path)

        context_id = data["context_id"]
        raw_value = patient_model.get_form_value(self.registry_model.code,
                                                 form_model.name,
                                                 section_model.code,
                                                 cde_model.code,
                                                 context_id=context_id,
                                                 clinical_data=data,
                                                 flattened=False)
        if isinstance(raw_value, list):
            display_value = "|".join([str(cde_model.get_display_value(x)) for x in raw_value])
        else:
            display_value = cde_model.get_display_value(raw_value)
            if cde_model.datatype == "date":
                if not display_value:
                    return ""
                return format_date(display_value)

        return display_value

    def _find_cde(self, cde_code):
        for form_model in self.registry_model.forms:
            for section_model in form_model.section_models:
                for cde_model in section_model.cde_models:
                    if cde_model.code == cde_code:
                        return form_model, section_model, cde_model

        raise ReportParserException("Report cde %s not found" % cde_code)

    def _get_demographics_column(self, patient_model, column_name):
        transform = demographics_transform_map.get(column_name, None)
        raw_value = getattr(patient_model, column_name)
        if transform is None:
            return raw_value
        else:
            if isinstance(transform, dict):
                return transform[raw_value]
            return transform(raw_value)

    def _load_patient_data(self, patient_model, context_id):
        return patient_model.get_dynamic_data(self.registry_model,
                                              context_id=context_id)

    def _completed(self, patient_model, form_name, data, percentage=False):
        form_model = RegistryForm.objects.get(name=form_name,
                                              registry=self.registry_model)
        if not percentage:
            for section_model in form_model.section_models:
                if not section_model.allow_multiple:
                    for cde_model in section_model.cde_models:
                        if not cde_completed(self.registry_model,
                                             form_model,
                                             section_model,
                                             cde_model,
                                             patient_model,
                                             data):
                            return False
            return True
        # percentage
        num_cdes = 0.0
        num_completed = 0.0
        for section_model in form_model.section_models:
            if not section_model.allow_multiple:
                for cde_model in section_model.cde_models:
                    num_cdes += 1.0
                    if cde_completed(self.registry_model,
                                     form_model,
                                     section_model,
                                     cde_model,
                                     patient_model,
                                     data):
                        num_completed += 1.0
        value = 100.0 * (num_completed / num_cdes)
        return round(value, 0)

    def _get_patients(self):
        user_working_groups = self.user.working_groups.all()

        for patient_model in Patient.objects.filter(rdrf_registry__code__in=[self.registry_model.code],
                                                    working_groups__in=user_working_groups):
            yield patient_model

    def _security_check(self):
        if not self.user.in_registry(self.registry_model):
            raise SecurityException()

def execute(registry_model, report_name, report_spec, user):
    logger.info("running custom action report %s for %s" % (report_name,
                                                            user.username))
    parser = ReportGenerator(registry_model, report_name, report_spec, user)
    parser.generate_report()
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="Completion Report.csv"'
    return parser.dump_csv(response)
<commit_msg>rdrf #1278  fix typeerror<commit_after>from django.http import HttpResponse
from rdrf.models.definition.models import RegistryForm
from rdrf.models.definition.models import Section
from rdrf.models.definition.models import CommonDataElement
from rdrf.models.definition.models import ContextFormGroup
from rdrf.models.definition.models import RDRFContext
from rdrf.models.definition.models import ClinicalData
from rdrf.helpers.utils import cde_completed
from rdrf.helpers.utils import format_date
from registry.patients.models import Patient
from registry.patients.models import ConsentValue
import logging
import csv

logger = logging.getLogger(__name__)

class SecurityException(Exception):
    pass

class ReportParserException(Exception):
    pass

class ColumnType:
    DEMOGRAPHICS = "demographics"
    CDE = "cde"
    COMPLETION = "completion"
    COMPLETION_PERCENTAGE = "%"
    CONSENT = "consent"
    CONSENT_DATE = "consent_date"
    FOLLOWUP_DATE = "followup_date"

demographics_transform_map = {"sex": {"1": "Male", "2": "Female", "3": "Indeterminate"},
                              "date_of_birth": format_date,
                              }

def aus_date(american_date):
    if not american_date:
        return ""
    year, month, day = american_date.split("-")
    return "%s-%s-%s" % (day, month, year)

def get_timestamp(clinical_data):
    from rdrf.helpers.utils import parse_iso_datetime
    if not clinical_data:
        return None
    if not clinical_data.data:
        return None

    return parse_iso_datetime(clinical_data.data.get("timestamp", None))

class ReportGenerator:
    def __init__(self, registry_model, report_name, report_spec, user):
        import json
        self.registry_model = registry_model
        self.report_name = report_name
        self.report_spec = json.loads(report_spec)
        self.user = user
        self.form_names = None
        self.start_date = None
        self.finish_date = None
        self.data = self._get_all_data()
        self.context_form_group = self._get_context_form_group()
        self.report = None

    def dump_csv(self, stream):
        writer = csv.writer(stream)
        writer.writerows(self.report)
        return stream

    def _get_context(self, patient_model):
        # get the fixed context associated with the context
        # form group specified in the report spec.
        # there should only be one for each patient
        context_models = patient_model.context_models
        return self._get_fixed_context(context_models)

    def _get_fixed_context(self, context_models):
        for context_model in context_models:
            if context_model.context_form_group:
                if context_model.context_form_group.pk == self.context_form_group.pk:
                    return context_model

    def _get_context_form_group(self):
        if "context_form_group" in self.report_spec:
            form_group_name = self.report_spec["context_form_group"]
            cfg = ContextFormGroup.objects.get(name=form_group_name,
                                               registry=self.registry_model,
                                               context_type="F")
            return cfg

    def _get_all_data(self):
        return []

    def generate_report(self):
        self._security_check()
        return self._run_report()

    def _run_report(self):
        rows = []
        rows.append(self._get_header())
        for patient_model in self._get_patients():
            try:
                logger.info("getting data for patient %s" % patient_model.pk)
                # the context needs to be determined by the report spec
                # as it contains the context_form_group name
                context_model = self._get_context(patient_model)
                if not context_model:
                    logger.info("no context - skipping")
                    continue
                logger.info("context id = %s" % context_model.id)
                data = self._load_patient_data(patient_model, context_model.id)
                if not data:
                    logger.info("no data")
                else:
                    logger.info("data exists")
                    row = []
                    for column in self.report_spec["columns"]:
                        column_value = "" if not data else self._get_column_value(patient_model, data, column)
                        row.append(column_value)
                    rows.append(row)
            except Exception as ex:
                logger.error("Completion report error pid %s: %s" % (patient_model.pk,
                                                                     ex))
        self.report = rows

    def _get_header(self):
        def header(col):
            if "label" in col:
                return col["label"]
            return col["name"]
        return [header(col) for col in self.report_spec["columns"]]

    def _get_column_value(self, patient_model, data, column):
        column_type = column["type"]
        if column_type == ColumnType.DEMOGRAPHICS:
            column_name = column["name"]
            return self._get_demographics_column(patient_model, column_name)
        if column_type == ColumnType.COMPLETION:
            form_name = column["name"]
            return self._completed(patient_model, form_name, data)
        if column_type == ColumnType.COMPLETION_PERCENTAGE:
            form_name = column["name"]
            return self._completed(patient_model, form_name, data, percentage=True)
        if column_type == ColumnType.CDE:
            cde_path = column["name"]
            try:
                return self._get_cde(patient_model, cde_path, data)
            except KeyError:
                return "[Not Entered]"
        if column_type == ColumnType.CONSENT:
            consent_section_code, consent_code = column["name"].split("/")
            return self._get_consent(patient_model,
                                     consent_section_code,
                                     consent_code)
        if column_type == ColumnType.CONSENT_DATE:
            consent_section_code, consent_code = column["name"].split("/")
            return self._get_consent(patient_model,
                                     consent_section_code,
                                     consent_code,
                                     get_date=True)
        if column_type == ColumnType.FOLLOWUP_DATE:
            context_form_group_name = column["context_form_group"]
            form_name = column["name"]
            return self._get_followup_date(patient_model,
                                           context_form_group_name,
                                           form_name)
        else:
            raise ReportParserException("Unknown column type: %s" % column_type)

    def _get_followup_date(self,
                           patient_model,
                           context_form_group_name,
                           form_name):
        from rdrf.models.definition.models import ContextFormGroup
        cfg = ContextFormGroup.objects.get(registry=self.registry_model,
                                           name=context_form_group_name)
        # find the last/latest context containing the form
        context_models = [c for c in patient_model.context_models
                          if c.registry.pk == self.registry_model.pk and c.context_form_group and c.context_form_group.pk == cfg.pk]
        if not context_models:
            return ""
        latest_date = None
        for context_model in context_models:
            # get the associated clinical data and check the timestamp
            try:
                clinical_data = ClinicalData.objects.get(context_id=context_model.pk,
                                                         django_model="Patient",
                                                         collection="cdes",
                                                         django_id=patient_model.pk)
                timestamp = get_timestamp(clinical_data)
                if latest_date is None or timestamp > latest_date:
                    latest_date = timestamp
            except ClinicalData.DoesNotExist:
                pass
        if latest_date is None:
            return ""
        return format_date(latest_date)

    def _get_consent(self,
                     patient_model,
                     consent_section_code,
                     consent_code,
                     get_date=False):
        for consent_section in self.registry_model.consent_sections.all():
            if consent_section.code == consent_section_code:
                for consent_question in consent_section.questions.all():
                    if consent_question.code == consent_code:
                        try:
                            consent_value = ConsentValue.objects.get(patient=patient_model,
                                                                     consent_question=consent_question)
                            if get_date:
                                first_save = consent_value.first_save
                                last_update = consent_value.last_update
                                if not last_update:
                                    return format_date(first_save)
                                return format_date(last_update)
                            return "True" if consent_value else "False"
                        except ConsentValue.DoesNotExist:
                            if get_date:
                                return ""
                            return "False"
        if get_date:
            return ""
        return "False"

    def _get_cde(self, patient_model, cde_path, data):
        if "/" in cde_path:
            form_name, section_code, cde_code = cde_path.split("/")
            form_model = RegistryForm.objects.get(name=form_name,
                                                  registry=self.registry_model)
            section_model = Section.objects.get(code=section_code)
            cde_model = CommonDataElement.objects.get(code=cde_code)
        else:
            form_model, section_model, cde_model = self._find_cde(cde_path)

        context_id = data["context_id"]
        raw_value = patient_model.get_form_value(self.registry_model.code,
                                                 form_model.name,
                                                 section_model.code,
                                                 cde_model.code,
                                                 context_id=context_id,
                                                 clinical_data=data,
                                                 flattened=False)
        if isinstance(raw_value, list):
            display_value = "|".join([str(cde_model.get_display_value(x)) for x in raw_value])
        else:
            display_value = cde_model.get_display_value(raw_value)
            if cde_model.datatype == "date":
                if not display_value:
                    return ""
                return format_date(display_value)

        return display_value

    def _find_cde(self, cde_code):
        for form_model in self.registry_model.forms:
            for section_model in form_model.section_models:
                for cde_model in section_model.cde_models:
                    if cde_model.code == cde_code:
                        return form_model, section_model, cde_model

        raise ReportParserException("Report cde %s not found" % cde_code)

    def _get_demographics_column(self, patient_model, column_name):
        transform = demographics_transform_map.get(column_name, None)
        raw_value = getattr(patient_model, column_name)
        if transform is None:
            return raw_value
        else:
            if isinstance(transform, dict):
                return transform[raw_value]
            return transform(raw_value)

    def _load_patient_data(self, patient_model, context_id):
        return patient_model.get_dynamic_data(self.registry_model,
                                              context_id=context_id)

    def _completed(self, patient_model, form_name, data, percentage=False):
        form_model = RegistryForm.objects.get(name=form_name,
                                              registry=self.registry_model)
        if not percentage:
            for section_model in form_model.section_models:
                if not section_model.allow_multiple:
                    for cde_model in section_model.cde_models:
                        if not cde_completed(self.registry_model,
                                             form_model,
                                             section_model,
                                             cde_model,
                                             patient_model,
                                             data):
                            return False
            return True
        # percentage
        num_cdes = 0.0
        num_completed = 0.0
        for section_model in form_model.section_models:
            if not section_model.allow_multiple:
                for cde_model in section_model.cde_models:
                    num_cdes += 1.0
                    if cde_completed(self.registry_model,
                                     form_model,
                                     section_model,
                                     cde_model,
                                     patient_model,
                                     data):
                        num_completed += 1.0
        value = 100.0 * (num_completed / num_cdes)
        return round(value, 0)

    def _get_patients(self):
        user_working_groups = self.user.working_groups.all()

        for patient_model in Patient.objects.filter(rdrf_registry__code__in=[self.registry_model.code],
                                                    working_groups__in=user_working_groups):
            yield patient_model

    def _security_check(self):
        if not self.user.in_registry(self.registry_model):
            raise SecurityException()

def execute(registry_model, report_name, report_spec, user):
    logger.info("running custom action report %s for %s" % (report_name,
                                                            user.username))
    parser = ReportGenerator(registry_model, report_name, report_spec, user)
    parser.generate_report()
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="Completion Report.csv"'
    return parser.dump_csv(response)
<|endoftext|>


<commit_before>from __future__ import division

import argparse
import sys
import time

import chainer
from chainer import iterators

from chainercv.datasets import voc_detection_label_names
from chainercv.datasets import VOCDetectionDataset
from chainercv.evaluations import eval_detection_voc
from chainercv.links import FasterRCNNVGG16
from chainercv.links import SSD300
from chainercv.links import SSD512
from chainercv.utils import apply_prediction_to_iterator

class ProgressHook(object):

    def __init__(self, n_total):
        self.n_total = n_total
        self.start = time.time()
        self.n_processed = 0

    def __call__(self, imgs, pred_values, gt_values):
        self.n_processed += len(imgs)
        fps = self.n_processed / (time.time() - self.start)
        sys.stdout.write(
            '\r{:d} of {:d} images, {:.2f} FPS'.format(
                self.n_processed, self.n_total, fps))
        sys.stdout.flush()

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--model', choices=('faster_rcnn', 'ssd300', 'ssd512'),
        default='ssd300')
    parser.add_argument('--pretrained_model')
    parser.add_argument('--gpu', type=int, default=-1)
    parser.add_argument('--batchsize', type=int, default=32)
    args = parser.parse_args()

    if args.model == 'faster_rcnn':
        if args.pretrained_model:
            model = FasterRCNNVGG16(pretrained_model=args.pretrained_model)
        else:
            model = FasterRCNNVGG16(pretrained_model='voc07')
    elif args.model == 'ssd300':
        if args.pretrained_model:
            model = SSD300(pretrained_model=args.pretrained_model)
        else:
            model = SSD300(pretrained_model='voc0712')
    elif args.model == 'ssd512':
        if args.pretrained_model:
            model = SSD512(pretrained_model=args.pretrained_model)
        else:
            model = SSD512(pretrained_model='voc0712')

    if args.gpu >= 0:
        chainer.cuda.get_device(args.gpu).use()
        model.to_gpu()

    model.use_preset('evaluate')

    dataset = VOCDetectionDataset(
        year='2007', split='test', use_difficult=True, return_difficult=True)
    iterator = iterators.SerialIterator(
        dataset, args.batchsize, repeat=False, shuffle=False)

    imgs, pred_values, gt_values = apply_prediction_to_iterator(
        model.predict, iterator, hook=ProgressHook(len(dataset)))
    # delete unused iterator explicitly
    del imgs

    pred_bboxes, pred_labels, pred_scores = pred_values
    gt_bboxes, gt_labels, gt_difficults = gt_values

    eval_ = eval_detection_voc(
        pred_bboxes, pred_labels, pred_scores,
        gt_bboxes, gt_labels, gt_difficults,
        use_07_metric=True)

    print()
    print('mAP: {:f}'.format(eval_['map']))
    for l, name in enumerate(voc_detection_label_names):
        if l in eval_:
            print('{:s}: {:f}'.format(name, eval_[l]['ap']))
        else:
            print('{:s}: -'.format(name))

if __name__ == '__main__':
    main()
<commit_msg>update eval_demo<commit_after>from __future__ import division

import argparse
import sys
import time

import chainer
from chainer import iterators

from chainercv.datasets import voc_detection_label_names
from chainercv.datasets import VOCDetectionDataset
from chainercv.evaluations import eval_detection_voc_ap
from chainercv.links import FasterRCNNVGG16
from chainercv.links import SSD300
from chainercv.links import SSD512
from chainercv.utils import apply_prediction_to_iterator

class ProgressHook(object):

    def __init__(self, n_total):
        self.n_total = n_total
        self.start = time.time()
        self.n_processed = 0

    def __call__(self, imgs, pred_values, gt_values):
        self.n_processed += len(imgs)
        fps = self.n_processed / (time.time() - self.start)
        sys.stdout.write(
            '\r{:d} of {:d} images, {:.2f} FPS'.format(
                self.n_processed, self.n_total, fps))
        sys.stdout.flush()

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--model', choices=('faster_rcnn', 'ssd300', 'ssd512'),
        default='ssd300')
    parser.add_argument('--pretrained_model')
    parser.add_argument('--gpu', type=int, default=-1)
    parser.add_argument('--batchsize', type=int, default=32)
    args = parser.parse_args()

    if args.model == 'faster_rcnn':
        if args.pretrained_model:
            model = FasterRCNNVGG16(pretrained_model=args.pretrained_model)
        else:
            model = FasterRCNNVGG16(pretrained_model='voc07')
    elif args.model == 'ssd300':
        if args.pretrained_model:
            model = SSD300(pretrained_model=args.pretrained_model)
        else:
            model = SSD300(pretrained_model='voc0712')
    elif args.model == 'ssd512':
        if args.pretrained_model:
            model = SSD512(pretrained_model=args.pretrained_model)
        else:
            model = SSD512(pretrained_model='voc0712')

    if args.gpu >= 0:
        chainer.cuda.get_device(args.gpu).use()
        model.to_gpu()

    model.use_preset('evaluate')

    dataset = VOCDetectionDataset(
        year='2007', split='test', use_difficult=True, return_difficult=True)
    iterator = iterators.SerialIterator(
        dataset, args.batchsize, repeat=False, shuffle=False)

    imgs, pred_values, gt_values = apply_prediction_to_iterator(
        model.predict, iterator, hook=ProgressHook(len(dataset)))
    # delete unused iterator explicitly
    del imgs

    pred_bboxes, pred_labels, pred_scores = pred_values
    gt_bboxes, gt_labels, gt_difficults = gt_values

    ap = eval_detection_voc_ap(
        pred_bboxes, pred_labels, pred_scores,
        gt_bboxes, gt_labels, gt_difficults,
        use_07_metric=True)
    map_ = sum(ap_l for ap_l in ap if ap is not None)

    print()
    print('mAP: {:f}'.format(map_))
    for l, name in enumerate(voc_detection_label_names):
        if ap[l]:
            print('{:s}: {:f}'.format(name, ap[l]))
        else:
            print('{:s}: -'.format(name))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from keras.models import Sequential
from keras.layers.core import Dense, Activation
from keras.layers.recurrent import LSTM
from keras.datasets.data_utils import get_file
import numpy as np
import random, sys

'''
    Example script to generate text from Nietzsche's writings.

    At least 20 epochs are required before the generated text
    starts sounding coherent.

    It is recommended to run this script on GPU, as recurrent
    networks are quite computationally intensive.

    If you try this script on new data, make sure your corpus
    has at least ~100k characters. ~1M is better.
'''

path = get_file('nietzsche.txt', origin="https://s3.amazonaws.com/text-datasets/nietzsche.txt")
text = open(path).read().lower()
print('corpus length:', len(text))

chars = set(text)
print('total chars:', len(chars))
char_indices = dict((c, i) for i, c in enumerate(chars))
indices_char = dict((i, c) for i, c in enumerate(chars))

# cut the text in semi-redundant sequences of 30 characters
maxlen = 30
step = 5
sentences = []
next_chars = []
for i in range(0, len(text) - maxlen, step):
    sentences.append(text[i : i + maxlen])
    next_chars.append(text[i + maxlen])
print('nb sequences:', len(sentences))

print('Vectorization...')
X = np.zeros((len(sentences), maxlen, len(chars)))
y = np.zeros((len(sentences), len(chars)))
for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        X[i, t, char_indices[char]] = 1.
    y[i, char_indices[next_chars[i]]] = 1.

# build the model: 2 stacked LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(len(chars), 512, return_sequences=True))
model.add(LSTM(512, 512, return_sequences=False))
model.add(Dense(512, len(chars)))
model.add(Activation('softmax'))

model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

# helper function to sample an index from a probability array
def sample(a, diversity=0.75):
    if random.random() > diversity:
        return np.argmax(a)
    while 1:
        i = random.randint(0, len(a)-1)
        if a[i] > random.random():
            return i

# train the model, output generated text after each epoch
for iteration in range(1, 30):
    print()
    print('-' * 50)
    print('Iteration', iteration)
    model.fit(X, y, batch_size=128, nb_epoch=1)

    start_index = random.randint(0, len(text) - maxlen - 1)

    for diversity in [0.4, 0.7, 1.]:
        print()
        print('----- diversity:', diversity)

        generated = ''
        sentence = text[start_index : start_index + maxlen]
        generated += sentence
        print('----- Generating with seed: "' + sentence + '"')
        sys.stdout.write(generated)

        for iteration in range(400):
            x = np.zeros((1, maxlen, len(chars)))
            for t, char in enumerate(sentence):
                x[0, t, char_indices[char]] = 1.

            preds = model.predict(x, verbose=0)[0]
            next_index = sample(preds, diversity)
            next_char = indices_char[next_index]

            generated += next_char
            sentence = sentence[1:] + next_char

            sys.stdout.write(next_char)
            sys.stdout.flush()
        print()

<commit_msg>Fix printing for Python2 in LSTM example<commit_after>from __future__ import print_function
from keras.models import Sequential
from keras.layers.core import Dense, Activation
from keras.layers.recurrent import LSTM
from keras.datasets.data_utils import get_file
import numpy as np
import random, sys

'''
    Example script to generate text from Nietzsche's writings.

    At least 20 epochs are required before the generated text
    starts sounding coherent.

    It is recommended to run this script on GPU, as recurrent
    networks are quite computationally intensive.

    If you try this script on new data, make sure your corpus
    has at least ~100k characters. ~1M is better.
'''

path = get_file('nietzsche.txt', origin="https://s3.amazonaws.com/text-datasets/nietzsche.txt")
text = open(path).read().lower()
print('corpus length:', len(text))

chars = set(text)
print('total chars:', len(chars))
char_indices = dict((c, i) for i, c in enumerate(chars))
indices_char = dict((i, c) for i, c in enumerate(chars))

# cut the text in semi-redundant sequences of 30 characters
maxlen = 30
step = 5
sentences = []
next_chars = []
for i in range(0, len(text) - maxlen, step):
    sentences.append(text[i : i + maxlen])
    next_chars.append(text[i + maxlen])
print('nb sequences:', len(sentences))

print('Vectorization...')
X = np.zeros((len(sentences), maxlen, len(chars)))
y = np.zeros((len(sentences), len(chars)))
for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        X[i, t, char_indices[char]] = 1.
    y[i, char_indices[next_chars[i]]] = 1.

# build the model: 2 stacked LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(len(chars), 512, return_sequences=True))
model.add(LSTM(512, 512, return_sequences=False))
model.add(Dense(512, len(chars)))
model.add(Activation('softmax'))

model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

# helper function to sample an index from a probability array
def sample(a, diversity=0.75):
    if random.random() > diversity:
        return np.argmax(a)
    while 1:
        i = random.randint(0, len(a)-1)
        if a[i] > random.random():
            return i

# train the model, output generated text after each epoch
for iteration in range(1, 30):
    print()
    print('-' * 50)
    print('Iteration', iteration)
    model.fit(X, y, batch_size=128, nb_epoch=1)

    start_index = random.randint(0, len(text) - maxlen - 1)

    for diversity in [0.4, 0.7, 1.]:
        print()
        print('----- diversity:', diversity)

        generated = ''
        sentence = text[start_index : start_index + maxlen]
        generated += sentence
        print('----- Generating with seed: "' + sentence + '"')
        sys.stdout.write(generated)

        for iteration in range(400):
            x = np.zeros((1, maxlen, len(chars)))
            for t, char in enumerate(sentence):
                x[0, t, char_indices[char]] = 1.

            preds = model.predict(x, verbose=0)[0]
            next_index = sample(preds, diversity)
            next_char = indices_char[next_index]

            generated += next_char
            sentence = sentence[1:] + next_char

            sys.stdout.write(next_char)
            sys.stdout.flush()
        print()

<|endoftext|>


<commit_before>"""Handlers of KycoBuffers"""
import logging

from struct import unpack

log = logging.getLogger('Kyco')

def raw_event_handler(raw_buffer, msg_in_buffer, app_buffer):
    log.info("Raw Event Handler started")
    while True:
        event = raw_buffer.get()
        log.debug('KycoRawEvent read...')
        log.debug(event.context)
        # data = unpack('!B', data)[0]
        # addr = writer.get_extra_info('peername')
        # log.debug("[Message Handler] New OF Message '%s' received from %s",
        # str(data), addr)
        # message = (addr, data)
        # msg_in_buffer.put(message)

def msg_in_event_handler(msg_in_buffer):
    log.info("Message In Event Handler started")
    # msg = "[Message Processor] MessageInEvenet %s"
    while True:
        event = msg_in_buffer.get()
        log.debug(event.context)
        # apps = self._events_listeners[0]
        # executor = ThreadPoolExecutor(max_workers=len(apps))
        # self.executors.append(executor)
        # [executor.submit(app.handle_event, message_event) for app in apps]

def msg_out_event_handler(msg_out_buffer):
    log.info("Message Out Event Handler started")
    # msg = "[Message Processor] MessageInEvenet %s"
    while True:
        event = msg_out_buffer.get()
        log.debug(event.context)
        # apps = self._events_listeners[0]
        # executor = ThreadPoolExecutor(max_workers=len(apps))
        # self.executors.append(executor)
        # [executor.submit(app.handle_event, message_event) for app in apps]

def app_event_handler(app_buffer):
    log.info("App Event Handler started")
    # msg = "[Message Processor] MessageInEvenet %s"
    while True:
        event = app_buffer.get()
        log.debug(event.context)
        # apps = self._events_listeners[0]
        # executor = ThreadPoolExecutor(max_workers=len(apps))
        # self.executors.append(executor)
        # [executor.submit(app.handle_event, message_event) for app in apps]
<commit_msg>Some fixes on event_handlers<commit_after>"""Handlers of KycoBuffers"""
import logging

from struct import unpack
from kyco.core.events import KycoRawOpenFlowMessageIn
log = logging.getLogger('Kyco')

def raw_event_handler(listeners, raw_buffer, msg_in_buffer, app_buffer):
    log.info("Raw Event Handler started")
    while True:
        event = raw_buffer.get()
        log.debug("\n#######################################################\n"
                  "                 RawEvent handler called\n"
                  "#######################################################\n")
        log.debug('KycoRawEvent read...')
        # log.debug("%s: %s", event.context, event.content)
        if isinstance(event, KycoRawOpenFlowMessageIn):
            for listener in listeners['KycoRawOpenFlowMessageIn']:
                listener(event)

        # data = unpack('!B', data)[0]
        # addr = writer.get_extra_info('peername')
        # log.debug("[Message Handler] New OF Message '%s' received from %s",
        # str(data), addr)
        # message = (addr, data)
        # msg_in_buffer.put(message)

def msg_in_event_handler(listeners, msg_in_buffer):
    log.info("Message In Event Handler started")
    # msg = "[Message Processor] MessageInEvenet %s"
    while True:
        event = msg_in_buffer.get()
        log.debug("\n#######################################################\n"
                  "                 MsgInEvent handler called\n"
                  "#######################################################\n")
        log.debug("%s: %s", event.context, event.content)
        # apps = self._events_listeners[0]
        # executor = ThreadPoolExecutor(max_workers=len(apps))
        # self.executors.append(executor)
        # [executor.submit(app.handle_event, message_event) for app in apps]

def msg_out_event_handler(listeners, msg_out_buffer):
    log.info("Message Out Event Handler started")
    # msg = "[Message Processor] MessageInEvenet %s"
    while True:
        event = msg_out_buffer.get()
        log.debug("\n#######################################################\n"
                  "                 MsgOutEvent handler called\n"
                  "#######################################################\n")
        log.debug("%s: %s", event.context, event.content)

        # apps = self._events_listeners[0]
        # executor = ThreadPoolExecutor(max_workers=len(apps))
        # self.executors.append(executor)
        # [executor.submit(app.handle_event, message_event) for app in apps]

def app_event_handler(listeners, app_buffer):
    log.info("App Event Handler started")
    # msg = "[Message Processor] MessageInEvenet %s"
    while True:
        event = app_buffer.get()
        log.debug("\n#######################################################\n"
                  "                 AppEvent handler called\n"
                  "#######################################################\n")
        log.debug("%s: %s", event.context, event.content)
        # apps = self._events_listeners[0]
        # executor = ThreadPoolExecutor(max_workers=len(apps))
        # self.executors.append(executor)
        # [executor.submit(app.handle_event, message_event) for app in apps]
<|endoftext|>


<commit_before>

"""
Run some experiments using SyntheticDataset1
"""

import os
import sys
import errno
import logging
import numpy
import argparse
from apgl.graph import *
from exp.recommendexp.RecommendExpHelper import RecommendExpHelper
from exp.recommendexp.SyntheticDataset1 import SyntheticDataset1

#if __debug__:
#    raise RuntimeError("Must run python with -O flag")

numpy.random.seed(21)
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
numpy.set_printoptions(suppress=True, linewidth=150)
numpy.seterr("raise", under="ignore")

# Arguments related to the dataset
dataArgs = argparse.Namespace()

# Arguments related to the algorithm
defaultAlgoArgs = argparse.Namespace()
defaultAlgoArgs.ks = numpy.array(2**numpy.arange(4, 7, 0.5), numpy.int)
defaultAlgoArgs.rhos = numpy.linspace(0.5, 0.0, 10)
defaultAlgoArgs.folds = 3

# init (reading/writting command line arguments)
# data args parser #
dataParser = argparse.ArgumentParser(description="", add_help=False)
dataParser.add_argument("-h", "--help", action="store_true", help="show this help message and exit")
devNull, remainingArgs = dataParser.parse_known_args(namespace=dataArgs)
if dataArgs.help:
    helpParser  = argparse.ArgumentParser(description="", add_help=False, parents=[dataParser, RecommendExpHelper.newAlgoParser(defaultAlgoArgs)])
    helpParser.print_help()
    exit()

dataArgs.extendedDirName = ""
dataArgs.extendedDirName += "SyntheticDataset1"

# print args #
logging.info("Running on SyntheticDataset1")
logging.info("Data params:")
keys = list(vars(dataArgs).keys())
keys.sort()
for key in keys:
    logging.info("    " + str(key) + ": " + str(dataArgs.__getattribute__(key)))

# data
generator = SyntheticDataset1(startM=5000, endM=6000, startN=1000, endN=1200, pnz=0.01)

# run
logging.info("Creating the exp-runner")
recommendExpHelper = RecommendExpHelper(generator.getTrainIteratorFunc(), generator.getTestIteratorFunc(), remainingArgs, defaultAlgoArgs, dataArgs.extendedDirName)
recommendExpHelper.printAlgoArgs()
#    os.makedirs(resultsDir, exist_ok=True) # for python 3.2
try:
    os.makedirs(recommendExpHelper.resultsDir)
except OSError as err:
    if err.errno != errno.EEXIST:
        raise

recommendExpHelper.runExperiment()
<commit_msg>param tweak <commit_after>

"""
Run some experiments using SyntheticDataset1
"""

import os
import sys
import errno
import logging
import numpy
import argparse
from apgl.graph import *
from exp.recommendexp.RecommendExpHelper import RecommendExpHelper
from exp.recommendexp.SyntheticDataset1 import SyntheticDataset1

#if __debug__:
#    raise RuntimeError("Must run python with -O flag")

numpy.random.seed(21)
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
numpy.set_printoptions(suppress=True, linewidth=150)
numpy.seterr("raise", under="ignore")

# Arguments related to the dataset
dataArgs = argparse.Namespace()

# Arguments related to the algorithm
defaultAlgoArgs = argparse.Namespace()
defaultAlgoArgs.ks = numpy.array(2**numpy.arange(4, 7, 0.5), numpy.int)
defaultAlgoArgs.rhos = numpy.linspace(0.5, 0.0, 10)
defaultAlgoArgs.folds = 3

# init (reading/writting command line arguments)
# data args parser #
dataParser = argparse.ArgumentParser(description="", add_help=False)
dataParser.add_argument("-h", "--help", action="store_true", help="show this help message and exit")
devNull, remainingArgs = dataParser.parse_known_args(namespace=dataArgs)
if dataArgs.help:
    helpParser  = argparse.ArgumentParser(description="", add_help=False, parents=[dataParser, RecommendExpHelper.newAlgoParser(defaultAlgoArgs)])
    helpParser.print_help()
    exit()

dataArgs.extendedDirName = ""
dataArgs.extendedDirName += "SyntheticDataset1"

# print args #
logging.info("Running on SyntheticDataset1")
logging.info("Data params:")
keys = list(vars(dataArgs).keys())
keys.sort()
for key in keys:
    logging.info("    " + str(key) + ": " + str(dataArgs.__getattribute__(key)))

# data
generator = SyntheticDataset1(startM=5000, endM=10000, startN=1000, endN=1500, pnz=0.02, noise=0.05)

# run
logging.info("Creating the exp-runner")
recommendExpHelper = RecommendExpHelper(generator.getTrainIteratorFunc(), generator.getTestIteratorFunc(), remainingArgs, defaultAlgoArgs, dataArgs.extendedDirName)
recommendExpHelper.printAlgoArgs()
#    os.makedirs(resultsDir, exist_ok=True) # for python 3.2
try:
    os.makedirs(recommendExpHelper.resultsDir)
except OSError as err:
    if err.errno != errno.EEXIST:
        raise

recommendExpHelper.runExperiment()
<|endoftext|>


<commit_before>import datetime
import functools
import sys
from concurrent.futures import ProcessPoolExecutor, as_completed
from threading import Thread
from os import makedirs, getcwd
import os.path
import logging

from ..common import chdir, run
from ..specs import load_all_specs, check_dependencies
from .find_update import update_available
from .process_student import process_student
from .args import process_args, compute_stogit_url
from .progress_bar import progress_bar
from .save_recordings import save_recordings, gist_recordings
from .tabulate import tabulate
from ..webapp import server

def make_progress_bar(students, no_progress=False):
    if no_progress:
        return lambda _: None

    size = len(students)
    remaining = set(students)
    invocation_count = 0

    def increment(username):
        nonlocal remaining
        nonlocal invocation_count
        remaining.remove(username)
        invocation_count += 1
        msg = ', '.join(sorted(remaining))
        progress_bar(size, invocation_count, message=msg)

    msg = ', '.join(sorted(remaining))
    progress_bar(size, invocation_count, message=msg)
    return increment

def run_server(basedir):
    server.exe_name = '"{}/server/server_file"'.format(basedir)
    server.run_server()
    return

def main():
    basedir = getcwd()
    args, usernames, assignments, stogit_url = process_args()
    ci = args['ci']
    clean = args['clean']
    date = args['date']
    debug = args['debug']
    gist = args['gist']
    highlight_partials = args['highlight_partials']
    interact = args['interact']
    no_check = args['no_check']
    no_update = args['no_update']
    no_progress = args['no_progress']
    quiet = args['quiet']
    skip_update_check = args['skip_update_check']
    sort_by = args['sort_by']
    web = args['web']
    workers = args['workers']

    if debug or interact or web:
        workers = 1

    current_version, new_version = update_available(skip_update_check=skip_update_check)
    if new_version:
        print(('v{} is available: you have v{}. '
               'Try "pip3 install --no-cache --user --upgrade cs251tk" '
               'to update.').format(new_version, current_version), file=sys.stderr)

    if date:
        logging.debug('Checking out {}'.format(date))

    if not os.path.exists("data"):
        print('data directory not found', file=sys.stderr)
        download = input("Download specs? (Y/N)")
        if download and download.lower()[0] == "y":
            repo = input("Which class? (SD/HD)")
            if repo and repo.lower()[0] == 's':
                with chdir(basedir):
                    run(['git', 'clone', 'https://github.com/StoDevX/cs251-specs.git', 'data'])
                    if not args['stogit']:
                        stogit_url = compute_stogit_url(course="sd", stogit=None, _now=datetime.date.today())
            elif repo and repo.lower()[0] == "h":
                with chdir(basedir):
                    run(['git', 'clone', 'https://github.com/StoDevX/cs241-specs.git', 'data'])
                    if not args['stogit']:
                        stogit_url = compute_stogit_url(course="hd", stogit=None, _now=datetime.date.today())
            else:
                print("Class not recognized", file=sys.stderr)
                sys.exit(1)
        else:
            sys.exit(1)

    specs = load_all_specs(basedir=os.path.join(basedir, 'data'), skip_update_check=skip_update_check)
    if not specs:
        print('no specs loaded!')
        sys.exit(1)

    if assignments:
        available_specs = set(assignments)

        for spec_to_use in assignments:
            try:
                check_dependencies(specs[spec_to_use])
            except KeyError:
                print('Spec {} does not exist'.format(spec_to_use), file=sys.stderr)
                available_specs.remove(spec_to_use)

        assignments = available_specs

        if not assignments:
            print('no valid specs remaining', file=sys.stderr)
            sys.exit(1)

    results = []
    records = []
    makedirs('./students', exist_ok=True)
    with chdir('./students'):
        single = functools.partial(
            process_student,
            assignments=assignments,
            basedir=basedir,
            ci=ci,
            clean=clean,
            date=date,
            debug=debug,
            interact=interact,
            no_check=no_check,
            no_update=no_update,
            specs=specs,
            stogit_url=stogit_url,
            web=web
        )

        if workers > 1:
            print_progress = make_progress_bar(usernames, no_progress=no_progress)
            with ProcessPoolExecutor(max_workers=workers) as pool:
                futures = [pool.submit(single, name) for name in usernames]
                for future in as_completed(futures):
                    result, recording = future.result()
                    print_progress(result['username'])
                    results.append(result)
                    records.extend(recording)
        elif web:
            Thread(target=run_server, args=(basedir,), daemon=True).start()
            for student in usernames:
                print("\nStudent: {}".format(student))
                result, recording = single(student)
                results.append(result)
                records.extend(recording)
        else:
            for student in usernames:
                logging.debug('Processing {}'.format(student))
                result, recording = single(student)
                results.append(result)
                records.extend(recording)

    if not quiet:
        table = tabulate(results, sort_by=sort_by, highlight_partials=highlight_partials)
        print('\n' + table)

    if gist:
        table = tabulate(results, sort_by=sort_by, highlight_partials=highlight_partials)
        gist_recordings(records, table, debug=debug)
    else:
        save_recordings(records, debug=debug)
<commit_msg>Download specs automatically if ci and course flags are present<commit_after>import datetime
import functools
import sys
from concurrent.futures import ProcessPoolExecutor, as_completed
from threading import Thread
from os import makedirs, getcwd
import os.path
import logging

from ..common import chdir, run
from ..specs import load_all_specs, check_dependencies
from .find_update import update_available
from .process_student import process_student
from .args import process_args, compute_stogit_url
from .progress_bar import progress_bar
from .save_recordings import save_recordings, gist_recordings
from .tabulate import tabulate
from ..webapp import server

def make_progress_bar(students, no_progress=False):
    if no_progress:
        return lambda _: None

    size = len(students)
    remaining = set(students)
    invocation_count = 0

    def increment(username):
        nonlocal remaining
        nonlocal invocation_count
        remaining.remove(username)
        invocation_count += 1
        msg = ', '.join(sorted(remaining))
        progress_bar(size, invocation_count, message=msg)

    msg = ', '.join(sorted(remaining))
    progress_bar(size, invocation_count, message=msg)
    return increment

def run_server(basedir):
    server.exe_name = '"{}/server/server_file"'.format(basedir)
    server.run_server()
    return

def download_specs(course, basedir, stogit, stogit_url):
    spec_urls = {
        'sd':  'https://github.com/StoDevX/cs251-specs.git',
        'hd':  'https://github.com/StoDevX/cs241-specs.git',
        'ads': 'https://github.com/Jedmeyer/cs253-specs.git'
    }
    course = course.lower()
    try:
        url = spec_urls[course]
    except KeyError:
        print("Course {} not recognized".format(course))
        sys.exit(1)
    with chdir(basedir):
        run(['git', 'clone', 'https://github.com/StoDevX/cs251-specs.git', 'data'])

def main():
    basedir = getcwd()
    args, usernames, assignments, stogit_url = process_args()
    ci = args['ci']
    clean = args['clean']
    date = args['date']
    debug = args['debug']
    gist = args['gist']
    highlight_partials = args['highlight_partials']
    interact = args['interact']
    no_check = args['no_check']
    no_update = args['no_update']
    no_progress = args['no_progress']
    quiet = args['quiet']
    skip_update_check = args['skip_update_check']
    sort_by = args['sort_by']
    web = args['web']
    workers = args['workers']

    if debug or interact or web:
        workers = 1

    current_version, new_version = update_available(skip_update_check=skip_update_check)
    if new_version:
        print(('v{} is available: you have v{}. '
               'Try "pip3 install --no-cache --user --upgrade cs251tk" '
               'to update.').format(new_version, current_version), file=sys.stderr)

    if date:
        logging.debug('Checking out {}'.format(date))

    if not os.path.exists("data"):
        if args['ci']:
            if args['course']:
                download_specs(args['course'], basedir, args['stogit'], stogit_url)
            else:
                print("data directory not found and no course specified")
                sys.exit(1)

        else:
            if args['course']:
                print('data directory not found', file=sys.stderr)
                download = input("Download specs for {}? (Y/N)".format(args['course'].upper()))
                if download and download.lower()[0] == "y":
                    download_specs(args['course'], basedir, args['stogit'], stogit_url)
                else:
                    sys.exit(1)
            else:
                print('data directory not found', file=sys.stderr)
                download = input("Download specs? (Y/N)")
                if download and download.lower()[0] == "y":
                    repo = input("Which class? (SD/HD/ADS)")
                    if repo:
                        download_specs(repo, basedir, args['stogit'], stogit_url)
                    else:
                        sys.exit(1)
                else:
                    sys.exit(1)

    specs = load_all_specs(basedir=os.path.join(basedir, 'data'), skip_update_check=skip_update_check)
    if not specs:
        print('no specs loaded!')
        sys.exit(1)

    if assignments:
        available_specs = set(assignments)

        for spec_to_use in assignments:
            try:
                check_dependencies(specs[spec_to_use])
            except KeyError:
                print('Spec {} does not exist'.format(spec_to_use), file=sys.stderr)
                available_specs.remove(spec_to_use)

        assignments = available_specs

        if not assignments:
            print('no valid specs remaining', file=sys.stderr)
            sys.exit(1)

    results = []
    records = []
    makedirs('./students', exist_ok=True)
    with chdir('./students'):
        single = functools.partial(
            process_student,
            assignments=assignments,
            basedir=basedir,
            ci=ci,
            clean=clean,
            date=date,
            debug=debug,
            interact=interact,
            no_check=no_check,
            no_update=no_update,
            specs=specs,
            stogit_url=stogit_url,
            web=web
        )

        if workers > 1:
            print_progress = make_progress_bar(usernames, no_progress=no_progress)
            with ProcessPoolExecutor(max_workers=workers) as pool:
                futures = [pool.submit(single, name) for name in usernames]
                for future in as_completed(futures):
                    result, recording = future.result()
                    print_progress(result['username'])
                    results.append(result)
                    records.extend(recording)
        elif web:
            Thread(target=run_server, args=(basedir,), daemon=True).start()
            for student in usernames:
                print("\nStudent: {}".format(student))
                result, recording = single(student)
                results.append(result)
                records.extend(recording)
        else:
            for student in usernames:
                logging.debug('Processing {}'.format(student))
                result, recording = single(student)
                results.append(result)
                records.extend(recording)

    if ci or not quiet:
        table = tabulate(results, sort_by=sort_by, highlight_partials=highlight_partials)
        if ci:
            print('\n' + table)
            if '[31m' in table:
                print("Files missing")
                sys.exit(1)
        elif not quiet:
            print('\n' + table)

    if gist:
        table = tabulate(results, sort_by=sort_by, highlight_partials=highlight_partials)
        gist_recordings(records, table, debug=debug)
    else:
        save_recordings(records, debug=debug)
<|endoftext|>