<commit_before>#    Copyright 2012 OpenStack Foundation
#    Copyright 2014 Mirantis Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_log import log
import six

from manila.share import driver

LOG = log.getLogger(__name__)

class FakeShareDriver(driver.ShareDriver):
    """Fake share driver."""

    def __init__(self, *args, **kwargs):
        super(FakeShareDriver, self).__init__([True, False], *args, **kwargs)

    def manage_existing(self, share, driver_options):
        LOG.debug("Fake share driver: manage")
        LOG.debug("Fake share driver: driver options: %s",
                  six.text_type(driver_options))
        return {'size': 1}

    def unmanage(self, share):
        LOG.debug("Fake share driver: unmanage")

    @property
    def driver_handles_share_servers(self):
        if not isinstance(self.configuration.safe_get(
                'driver_handles_share_servers'), bool):
            return True

        return self.configuration.driver_handles_share_servers

    def create_snapshot(self, context, snapshot, share_server=None):
        pass

    def delete_snapshot(self, context, snapshot, share_server=None):
        pass

    def create_share(self, context, share, share_server=None):
        return ['/fake/path', '/fake/path2']

    def create_share_from_snapshot(self, context, share, snapshot,
                                   share_server=None):
        return ['/fake/path', '/fake/path2']

    def delete_share(self, context, share, share_server=None):
        pass

    def ensure_share(self, context, share, share_server=None):
        pass

    def allow_access(self, context, share, access, share_server=None):
        pass

    def deny_access(self, context, share, access, share_server=None):
        pass

    def get_share_stats(self, refresh=False):
        return None

    def do_setup(self, context):
        pass

    def setup_server(self, *args, **kwargs):
        pass

    def teardown_server(self, *args, **kwargs):
        pass

    def get_network_allocations_number(self):
        # NOTE(vponomaryov): Simulate drivers that use share servers and
        # do not use 'service_instance' module.
        return 2

    def _verify_share_server_handling(self, driver_handles_share_servers):
        return super(FakeShareDriver, self)._verify_share_server_handling(
            driver_handles_share_servers)

    def create_consistency_group(self, context, cg_id):
        pass

    def delete_consistency_group(self, context, cg_id):
        pass
<commit_msg>Add hint how to configure fake_driver in manila-share<commit_after>#    Copyright 2012 OpenStack Foundation
#    Copyright 2014 Mirantis Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_log import log
import six

from manila.share import driver

LOG = log.getLogger(__name__)

class FakeShareDriver(driver.ShareDriver):
    """Fake share driver.

    This fake driver can be also used as a test driver within a real
    running manila-share instance. To activate it use this in manila.conf::

        enabled_share_backends = fake

        [fake]
        driver_handles_share_servers = True
        share_backend_name = fake
        share_driver = manila.tests.fake_driver.FakeShareDriver

    With it you basically mocked all backend driver calls but e.g. networking
    will still be activated.
    """

    def __init__(self, *args, **kwargs):
        super(FakeShareDriver, self).__init__([True, False], *args, **kwargs)

    def manage_existing(self, share, driver_options):
        LOG.debug("Fake share driver: manage")
        LOG.debug("Fake share driver: driver options: %s",
                  six.text_type(driver_options))
        return {'size': 1}

    def unmanage(self, share):
        LOG.debug("Fake share driver: unmanage")

    @property
    def driver_handles_share_servers(self):
        if not isinstance(self.configuration.safe_get(
                'driver_handles_share_servers'), bool):
            return True

        return self.configuration.driver_handles_share_servers

    def create_snapshot(self, context, snapshot, share_server=None):
        pass

    def delete_snapshot(self, context, snapshot, share_server=None):
        pass

    def create_share(self, context, share, share_server=None):
        return ['/fake/path', '/fake/path2']

    def create_share_from_snapshot(self, context, share, snapshot,
                                   share_server=None):
        return ['/fake/path', '/fake/path2']

    def delete_share(self, context, share, share_server=None):
        pass

    def ensure_share(self, context, share, share_server=None):
        pass

    def allow_access(self, context, share, access, share_server=None):
        pass

    def deny_access(self, context, share, access, share_server=None):
        pass

    def get_share_stats(self, refresh=False):
        return None

    def do_setup(self, context):
        pass

    def setup_server(self, *args, **kwargs):
        pass

    def teardown_server(self, *args, **kwargs):
        pass

    def get_network_allocations_number(self):
        # NOTE(vponomaryov): Simulate drivers that use share servers and
        # do not use 'service_instance' module.
        return 2

    def _verify_share_server_handling(self, driver_handles_share_servers):
        return super(FakeShareDriver, self)._verify_share_server_handling(
            driver_handles_share_servers)

    def create_consistency_group(self, context, cg_id):
        pass

    def delete_consistency_group(self, context, cg_id):
        pass
<|endoftext|>


<commit_before>import time

__author__ = 'esteele'

import random, re, threading
from decimal import Decimal

class SensorReadingProvider(threading.Thread):
    # 'Humidity: 62.80 %\tTemperature: 20.00 *C\r\n'
    humidity_temp_string_re = re.compile("Humidity: (?P<humidity_perc>[0-9]{1,2}\.[0-9]{2}).*Temperature: (?P<temp_celcius>[0-9]{1,2}\.[0-9]{2}).*$")

    def __init__(self, serial_port):
        threading.Thread.__init__(self)
        self.last_humidity_reading = None
        self.last_temperature_reading = None
        self.arduino_serial_port = serial_port
        # Kick off the recorder
        self.start()

    def _record_sensor_reading(self):
        # Goes directly to the sensor and updates class variables
        # TODO - handle when the DHT returns an error (see the arduino source) and return something that can be used upstream to indicate an error
        line = self.arduino_serial_port.readline()
        mo = self.humidity_temp_string_re.match(line)
        if mo:
            humidity = mo.group('humidity_perc')
            temperature = mo.group('temp_celcius')
            #print "Matched. Humidity is %s and temp is %s" % (humidity, temperature)
        else:
            humidity = temperature = None
            print "No match for line->%s<-" % (line,)

        self.last_humidity_reading = humidity
        self.last_temperature_reading = temperature

    def run(self):
       while 1:
           self._record_sensor_reading()

    def get_latest_temperature(self):
        return self.last_temperature_reading

    def get_latest_humidity(self):
        return self.last_humidity_reading

class SimulatedSensorReadingProvider(SensorReadingProvider):
    INITIAL_TEMPERATURE_CELSIUS = Decimal("18.00")
    INITIAL_HUMIDITY_PERC = Decimal("85.00")

    def __init__(self, serial_port):
        self.srg = self._sensor_reading_generator(self.INITIAL_TEMPERATURE_CELSIUS, self.INITIAL_HUMIDITY_PERC)
        SensorReadingProvider.__init__(self, serial_port)

    def _sensor_reading_generator(self, t_celsius, h_percent):
        """
        Generates sensor readings that are relatively stable, moving by + or -0.05 for each calling
        """
        while True:
            t_celsius += Decimal(str(random.choice((-1,1))/20.0))
            h_percent += Decimal(str(random.choice((-1,1))/20.0))
            yield [t_celsius, h_percent]

    def _record_sensor_reading(self):
        """
        Simulates a request and response from a sensor, including the time taken to obtain the reading.
        """
        self.last_temperature_reading, self.last_humidity_reading = self.srg.next()
        time.sleep(0.25)

<commit_msg>Cap humidity at 99% to avoid borking the db<commit_after>import time

__author__ = 'esteele'

import random, re, threading
from decimal import Decimal

class SensorReadingProvider(threading.Thread):
    # 'Humidity: 62.80 %\tTemperature: 20.00 *C\r\n'
    humidity_temp_string_re = re.compile("Humidity: (?P<humidity_perc>[0-9]{1,2}\.[0-9]{2}).*Temperature: (?P<temp_celcius>[0-9]{1,2}\.[0-9]{2}).*$")

    def __init__(self, serial_port):
        threading.Thread.__init__(self)
        self.last_humidity_reading = None
        self.last_temperature_reading = None
        self.arduino_serial_port = serial_port
        # Kick off the recorder
        self.start()

    def _record_sensor_reading(self):
        # Goes directly to the sensor and updates class variables
        # TODO - handle when the DHT returns an error (see the arduino source) and return something that can be used upstream to indicate an error
        line = self.arduino_serial_port.readline()
        mo = self.humidity_temp_string_re.match(line)
        if mo:
            humidity = mo.group('humidity_perc')
            temperature = mo.group('temp_celcius')
            #print "Matched. Humidity is %s and temp is %s" % (humidity, temperature)
        else:
            humidity = temperature = None
            print "No match for line->%s<-" % (line,)

        self.last_humidity_reading = humidity
        self.last_temperature_reading = temperature

    def run(self):
       while 1:
           self._record_sensor_reading()

    def get_latest_temperature(self):
        return self.last_temperature_reading

    def get_latest_humidity(self):
        return self.last_humidity_reading

class SimulatedSensorReadingProvider(SensorReadingProvider):
    INITIAL_TEMPERATURE_CELSIUS = Decimal("18.00")
    INITIAL_HUMIDITY_PERC = Decimal("85.00")

    def __init__(self, serial_port):
        self.srg = self._sensor_reading_generator(self.INITIAL_TEMPERATURE_CELSIUS, self.INITIAL_HUMIDITY_PERC)
        SensorReadingProvider.__init__(self, serial_port)

    def _sensor_reading_generator(self, t_celsius, h_percent):
        """
        Generates sensor readings that are relatively stable, moving by + or -0.05 for each calling
        """
        while True:
            t_celsius += Decimal(str(random.choice((-1,1))/20.0))
            h_percent += Decimal(str(random.choice((-1,1))/20.0))
            # Cap humidity at 99% - it'll bork the db otherwise
            h_percent = min(Decimal("99"), h_percent)
            yield [t_celsius, h_percent]

    def _record_sensor_reading(self):
        """
        Simulates a request and response from a sensor, including the time taken to obtain the reading.
        """
        self.last_temperature_reading, self.last_humidity_reading = self.srg.next()
        time.sleep(0.25)

<|endoftext|>


<commit_before>from django.db import models
from django_extensions.db.fields import AutoSlugField

from django.utils.translation import ugettext_lazy as _

from jsonfield import JSONField

from sboard.models import NodeForeignKey
from sboard.models import couch
from couchdbkit.exceptions import ResourceNotFound

from manoseimas.utils import reify

import manoseimas.common.utils.words as words_utils

class CrawledItem(models.Model):
    source = models.URLField()
    created_at = models.DateTimeField(auto_now_add=True)
    modified_at = models.DateTimeField(auto_now=True)

    class Meta:
        abstract = True

def get_mp_full_name(mp):
    return mp.full_name

def prepare_positions(node):
    from manoseimas.compat.models import PersonPosition

    def position_to_dict(position):
        return {
            'node_ref': position.node.ref,
            'permalink': position.node.ref.permalink(),
            'formatted': position.format_position(),
            'title': position.node.ref.title,
            'position': position.position,
            'klass': position.klass,
        }

    position_list = list(PersonPosition.objects.filter(profile=node))
    position_list.sort(key=lambda pp: abs(pp.position), reverse=True)

    positions = {'for': [], 'against': [], 'neutral': []}
    for position in position_list:
        if abs(position.position) < 0.2:
            positions['neutral'].append(position_to_dict(position))
        elif position.position > 0:
            positions['for'].append(position_to_dict(position))
        else:
            positions['against'].append(position_to_dict(position))
    return positions

class ParliamentMember(CrawledItem):
    slug = AutoSlugField(populate_from=('first_name', 'last_name'),
                         max_length=120)
    source_id = models.CharField(max_length=16)
    first_name = models.CharField(max_length=128)
    last_name = models.CharField(max_length=128)
    date_of_birth = models.CharField(max_length=16, blank=True, null=True)
    email = models.EmailField(blank=True, null=True)
    phone = models.CharField(max_length=32, blank=True, null=True)
    candidate_page = models.URLField(blank=True, null=True)
    raised_by = models.ForeignKey('PoliticalParty', blank=True, null=True)
    photo = models.ImageField(upload_to='profile_images',
                              blank=True, null=True)
    term_of_office = models.CharField(max_length=32, blank=True, null=True)

    office_address = models.TextField(blank=True, null=True)
    constituency = models.CharField(max_length=128, blank=True, null=True)
    party_candidate = models.BooleanField(default=True)
    groups = models.ManyToManyField('Group', through='GroupMembership',
                                    related_name='members')

    biography = models.TextField(blank=True, null=True)

    # Precomputed stats fields
    statement_count = models.PositiveIntegerField(blank=True, null=True)
    long_statement_count = models.PositiveIntegerField(blank=True, null=True)
    vote_percentage = models.FloatField(blank=True, null=True)
    discussion_contribution_percentage = models.FloatField(blank=True,
                                                           null=True)
    positions = JSONField(default=None, blank=True, null=True)

    precomputed_fields = (
        ('statement_count', 'get_statement_count'),
        ('long_statement_count', 'get_long_statement_count'),
        ('vote_percentage', 'get_vote_percentage'),
        ('discussion_contribution_percentage',
         'get_discussion_contribution_percentage'),
        ('positions', 'get_positions'),
    )
    precomputation_depends_on = ('StenogramStatement',)

    @property
    def full_name(self):
        return u' '.join([self.first_name, self.last_name])

    def __unicode__(self):
        return self.full_name

    @property
    def fractions(self):
        return self.groups.filter(type=Group.TYPE_FRACTION)

    @property
    def fraction(self):
        ''' Current parliamentarian's fraction. '''
        if getattr(self, '_fraction'):
            return self._fraction[0].group
        else:
            membership = GroupMembership.objects.filter(
                member=self,
                group__type=Group.TYPE_FRACTION,
                until=None
            )[:]

            if membership:
                self._fraction = membership
                return membership[0].group
            else:
                return None

    def get_statement_count(self):
        return self.statements.filter(as_chairperson=False).count()

    def get_long_statement_count(self):
        return self.statements.filter(as_chairperson=False).\
            filter(word_count__gte=50).count()

    def get_long_statement_percentage(self):
        statements = self.get_statement_count()
        long_statements = self.get_long_statement_count()
        return (float(long_statements) / statements * 100
                if statements else 0.0)

    def get_discussion_contribution_percentage(self):
        all_discussions = StenogramTopic.objects.count()
        contributed_discusions = StenogramStatement.objects.\
            filter(speaker=self, as_chairperson=False).\
            aggregate(topics=models.Count('topic_id',
                                          distinct=True))
        return (float(contributed_discusions['topics'])
                / all_discussions * 100.0) if all_discussions else 0.0

    @property
    def votes(self):
        # Avoiding circular imports
        from manoseimas.votings.models import get_mp_votes
        return get_mp_votes(self.source_id)

    def get_vote_percentage(self):
        from manoseimas.votings.models import get_total_votes
        votes = sum(self.votes.values()) if self.votes else 0
        total_votes = get_total_votes()
        vote_percentage = float(votes) / total_votes * 100.0
        return vote_percentage

    def get_positions(self):
        try:
            mp_node = couch.view('sboard/by_slug', key=self.slug).one()
            return prepare_positions(mp_node)
        except ResourceNotFound:
            return None

    @property
    def all_statements(self):
        return self.statements.all()

class PoliticalParty(CrawledItem):
    name = models.CharField(max_length=128, unique=True)

    def __unicode__(self):
        return self.name

class Group(CrawledItem):
    TYPE_GROUP = 'group'
    TYPE_COMMITTEE = 'committee'
    TYPE_COMMISSION = 'commission'
    TYPE_FRACTION = 'fraction'
    TYPE_PARLIAMENT = 'parliament'

    GROUP_TYPES = (
        (TYPE_GROUP, _('Group')),
        (TYPE_COMMITTEE, _('Committee')),
        (TYPE_COMMISSION, _('Commission')),
        (TYPE_FRACTION, _('Fraction')),
        (TYPE_PARLIAMENT, _('Parliament')),
    )

    name = models.CharField(max_length=255, unique=True)
    slug = AutoSlugField(populate_from='name', max_length=120)
    type = models.CharField(max_length=64,
                            choices=GROUP_TYPES)
    displayed = models.BooleanField(default=True)
    logo = models.ImageField(upload_to='fraction_logos',
                             blank=True, null=True)

    # Precomputed stats fields
    avg_statement_count = models.FloatField(blank=True, null=True)
    avg_long_statement_count = models.FloatField(blank=True, null=True)
    avg_vote_percentage = models.FloatField(blank=True, null=True)
    avg_discussion_contribution_percentage = models.FloatField(blank=True,
                                                               null=True)
    positions = JSONField(default=None, blank=True, null=True)

    precomputed_fields = (
        ('avg_statement_count', 'get_avg_statement_count'),
        ('avg_long_statement_count', 'get_avg_long_statement_count'),
        ('avg_vote_percentage', 'get_avg_vote_percentage'),
        ('avg_discussion_contribution_percentage',
         'get_avg_discussion_contribution_percentage'),
        ('positions', 'get_positions'),
    )
    precomputation_filter = {
        'type': TYPE_FRACTION,
    }
    precomputation_depends_on = ('ParliamentMember',)

    class Meta:
        unique_together = (('name', 'type'))

    def __unicode__(self):
        return u'{} ({})'.format(self.name, self.type)

    @property
    def active_members(self):
        return self.members.filter(groupmembership__until=None)

    @reify
    def active_member_count(self):
        return self.active_members.count()

    def get_avg_statement_count(self):
        agg = self.active_members.filter(
            statements__as_chairperson=False,
        ).annotate(
            models.Count('statements')
        ).aggregate(
            avg_statements=models.Avg('statements__count')
        )
        return agg['avg_statements']

    def get_avg_long_statement_count(self):
        agg = self.active_members.filter(
            statements__word_count__gte=50,
            statements__as_chairperson=False,
        ).annotate(
            models.Count('statements')
        ).aggregate(
            avg_statements=models.Avg('statements__count')
        )
        return agg['avg_statements']

    def get_avg_vote_percentage(self):
        total_percentage = 0.0
        for member in self.active_members:
            total_percentage += member.vote_percentage
        return (total_percentage / self.active_member_count
                if self.active_member_count else 0.0)

    def get_avg_discussion_contribution_percentage(self):
        agg = self.active_members.aggregate(
            avg_contrib=models.Avg('discussion_contribution_percentage')
        )
        return agg['avg_contrib']

    def get_positions(self):
        fraction_node = couch.view('sboard/by_slug', key=self.slug).one()
        return prepare_positions(fraction_node)

class GroupMembership(CrawledItem):
    member = models.ForeignKey(ParliamentMember,
                               related_name='groupmembership')
    group = models.ForeignKey(Group)
    position = models.CharField(max_length=128)
    since = models.DateField(blank=True, null=True)
    until = models.DateField(blank=True, null=True)

    def __unicode__(self):
        return u'{} - {} ({})'.format(self.group.name,
                                      self.member.full_name,
                                      self.position)

class Stenogram(CrawledItem):
    source_id = models.CharField(max_length=16, db_index=True)
    date = models.DateField()
    sitting_no = models.IntegerField()
    sitting_name = models.TextField(blank=True, null=True)
    session = models.CharField(max_length=64, blank=True, null=True)

    def __unicode__(self):
        return u'{} Nr. {}'.format(self.date, self.sitting_no)

class StenogramTopic(CrawledItem):
    stenogram = models.ForeignKey(Stenogram, related_name='topics')
    title = models.TextField()
    timestamp = models.DateTimeField()
    presenters = models.ManyToManyField(ParliamentMember,
                                        related_name='presented_topics')

    def __unicode__(self):
        return self.title[:160]

class StenogramStatement(CrawledItem):
    topic = models.ForeignKey(StenogramTopic, related_name='statements')
    speaker = models.ForeignKey(ParliamentMember, related_name='statements',
                                blank=True, null=True)
    speaker_name = models.CharField(max_length=64)
    as_chairperson = models.BooleanField(default=False)
    text = models.TextField()
    word_count = models.PositiveIntegerField(default=0)

    precomputed_fields = (
        ('word_count', 'get_word_count'),
    )

    def get_speaker_name(self):
        return self.speaker.full_name if self.speaker else self.speaker_name

    def __unicode__(self):
        return u'{}: {}'.format(self.get_speaker_name(), self.text[:160])

    def get_word_count(self):
        return words_utils.get_word_count(self.text)

class Voting(models.Model):
    stenogram_topic = models.ForeignKey(StenogramTopic, related_name='votings')
    node = NodeForeignKey()
    timestamp = models.DateTimeField()

    class Meta:
        unique_together = ('stenogram_topic', 'node')

def percentile_property(attr):
    def inner_fn(self):
        total = self.__class__.objects.count()
        return int((total - getattr(self, attr) + 1.0)
                   / total * 100 + 0.5)
    return property(inner_fn)

class Ranking(models.Model):
    created_at = models.DateTimeField(auto_now_add=True)
    modified_at = models.DateTimeField(auto_now=True)
    votes_rank = models.IntegerField(default=0)
    statement_count_rank = models.IntegerField(default=0)
    long_statement_count_rank = models.IntegerField(default=0)
    discusion_contribution_percentage_rank = models.IntegerField(default=0)

    class Meta:
        abstract = True

    votes_percentile = percentile_property(
        'votes_rank')
    statement_count_percentile = percentile_property(
        'statement_count_rank')
    long_statement_count_percentile = percentile_property(
        'long_statement_count_rank')
    discusion_contribution_percentage_percentile = percentile_property(
        'discusion_contribution_percentage_rank')

class MPRanking(Ranking):
    target = models.OneToOneField(ParliamentMember,
                                  related_name='ranking')

class GroupRanking(Ranking):
    target = models.OneToOneField(Group,
                                  related_name='ranking')

class LawProject(CrawledItem):
    source_id = models.CharField(max_length=16, db_index=True, unique=True)
    date = models.DateField()
    project_name = models.TextField()
    project_number = models.CharField(max_length=32)
    project_url = models.URLField()

    proposers = models.ManyToManyField(ParliamentMember,
                                       related_name='law_projects')

    date_passed = models.DateField(blank=True, null=True)
    passing_source_id = models.CharField(max_length=16, blank=True,
                                         null=True, db_index=True)
    passing_doc_number = models.CharField(max_length=32, blank=True, null=True)
    passing_doc_url = models.URLField(blank=True, null=True)
<commit_msg>Compute stats for whole parliament.<commit_after>from django.db import models
from django_extensions.db.fields import AutoSlugField

from django.utils.translation import ugettext_lazy as _

from jsonfield import JSONField

from sboard.models import NodeForeignKey
from sboard.models import couch
from couchdbkit.exceptions import ResourceNotFound

from manoseimas.utils import reify

import manoseimas.common.utils.words as words_utils

class CrawledItem(models.Model):
    source = models.URLField()
    created_at = models.DateTimeField(auto_now_add=True)
    modified_at = models.DateTimeField(auto_now=True)

    class Meta:
        abstract = True

def get_mp_full_name(mp):
    return mp.full_name

def prepare_positions(node):
    from manoseimas.compat.models import PersonPosition

    def position_to_dict(position):
        return {
            'node_ref': position.node.ref,
            'permalink': position.node.ref.permalink(),
            'formatted': position.format_position(),
            'title': position.node.ref.title,
            'position': position.position,
            'klass': position.klass,
        }

    position_list = list(PersonPosition.objects.filter(profile=node))
    position_list.sort(key=lambda pp: abs(pp.position), reverse=True)

    positions = {'for': [], 'against': [], 'neutral': []}
    for position in position_list:
        if abs(position.position) < 0.2:
            positions['neutral'].append(position_to_dict(position))
        elif position.position > 0:
            positions['for'].append(position_to_dict(position))
        else:
            positions['against'].append(position_to_dict(position))
    return positions

class ParliamentMember(CrawledItem):
    slug = AutoSlugField(populate_from=('first_name', 'last_name'),
                         max_length=120)
    source_id = models.CharField(max_length=16)
    first_name = models.CharField(max_length=128)
    last_name = models.CharField(max_length=128)
    date_of_birth = models.CharField(max_length=16, blank=True, null=True)
    email = models.EmailField(blank=True, null=True)
    phone = models.CharField(max_length=32, blank=True, null=True)
    candidate_page = models.URLField(blank=True, null=True)
    raised_by = models.ForeignKey('PoliticalParty', blank=True, null=True)
    photo = models.ImageField(upload_to='profile_images',
                              blank=True, null=True)
    term_of_office = models.CharField(max_length=32, blank=True, null=True)

    office_address = models.TextField(blank=True, null=True)
    constituency = models.CharField(max_length=128, blank=True, null=True)
    party_candidate = models.BooleanField(default=True)
    groups = models.ManyToManyField('Group', through='GroupMembership',
                                    related_name='members')

    biography = models.TextField(blank=True, null=True)

    # Precomputed stats fields
    statement_count = models.PositiveIntegerField(blank=True, null=True)
    long_statement_count = models.PositiveIntegerField(blank=True, null=True)
    vote_percentage = models.FloatField(blank=True, null=True)
    discussion_contribution_percentage = models.FloatField(blank=True,
                                                           null=True)
    positions = JSONField(default=None, blank=True, null=True)

    precomputed_fields = (
        ('statement_count', 'get_statement_count'),
        ('long_statement_count', 'get_long_statement_count'),
        ('vote_percentage', 'get_vote_percentage'),
        ('discussion_contribution_percentage',
         'get_discussion_contribution_percentage'),
        ('positions', 'get_positions'),
    )
    precomputation_depends_on = ('StenogramStatement',)

    @property
    def full_name(self):
        return u' '.join([self.first_name, self.last_name])

    def __unicode__(self):
        return self.full_name

    @property
    def fractions(self):
        return self.groups.filter(type=Group.TYPE_FRACTION)

    @property
    def fraction(self):
        ''' Current parliamentarian's fraction. '''
        if getattr(self, '_fraction'):
            return self._fraction[0].group
        else:
            membership = GroupMembership.objects.filter(
                member=self,
                group__type=Group.TYPE_FRACTION,
                until=None
            )[:]

            if membership:
                self._fraction = membership
                return membership[0].group
            else:
                return None

    def get_statement_count(self):
        return self.statements.filter(as_chairperson=False).count()

    def get_long_statement_count(self):
        return self.statements.filter(as_chairperson=False).\
            filter(word_count__gte=50).count()

    def get_long_statement_percentage(self):
        statements = self.get_statement_count()
        long_statements = self.get_long_statement_count()
        return (float(long_statements) / statements * 100
                if statements else 0.0)

    def get_discussion_contribution_percentage(self):
        all_discussions = StenogramTopic.objects.count()
        contributed_discusions = StenogramStatement.objects.\
            filter(speaker=self, as_chairperson=False).\
            aggregate(topics=models.Count('topic_id',
                                          distinct=True))
        return (float(contributed_discusions['topics'])
                / all_discussions * 100.0) if all_discussions else 0.0

    @property
    def votes(self):
        # Avoiding circular imports
        from manoseimas.votings.models import get_mp_votes
        return get_mp_votes(self.source_id)

    def get_vote_percentage(self):
        from manoseimas.votings.models import get_total_votes
        votes = sum(self.votes.values()) if self.votes else 0
        total_votes = get_total_votes()
        vote_percentage = float(votes) / total_votes * 100.0
        return vote_percentage

    def get_positions(self):
        try:
            mp_node = couch.view('sboard/by_slug', key=self.slug).one()
            return prepare_positions(mp_node)
        except ResourceNotFound:
            return None

    @property
    def all_statements(self):
        return self.statements.all()

class PoliticalParty(CrawledItem):
    name = models.CharField(max_length=128, unique=True)

    def __unicode__(self):
        return self.name

class Group(CrawledItem):
    TYPE_GROUP = 'group'
    TYPE_COMMITTEE = 'committee'
    TYPE_COMMISSION = 'commission'
    TYPE_FRACTION = 'fraction'
    TYPE_PARLIAMENT = 'parliament'

    GROUP_TYPES = (
        (TYPE_GROUP, _('Group')),
        (TYPE_COMMITTEE, _('Committee')),
        (TYPE_COMMISSION, _('Commission')),
        (TYPE_FRACTION, _('Fraction')),
        (TYPE_PARLIAMENT, _('Parliament')),
    )

    name = models.CharField(max_length=255, unique=True)
    slug = AutoSlugField(populate_from='name', max_length=120)
    type = models.CharField(max_length=64,
                            choices=GROUP_TYPES)
    displayed = models.BooleanField(default=True)
    logo = models.ImageField(upload_to='fraction_logos',
                             blank=True, null=True)

    # Precomputed stats fields
    avg_statement_count = models.FloatField(blank=True, null=True)
    avg_long_statement_count = models.FloatField(blank=True, null=True)
    avg_vote_percentage = models.FloatField(blank=True, null=True)
    avg_discussion_contribution_percentage = models.FloatField(blank=True,
                                                               null=True)
    positions = JSONField(default=None, blank=True, null=True)

    precomputed_fields = (
        ('avg_statement_count', 'get_avg_statement_count'),
        ('avg_long_statement_count', 'get_avg_long_statement_count'),
        ('avg_vote_percentage', 'get_avg_vote_percentage'),
        ('avg_discussion_contribution_percentage',
         'get_avg_discussion_contribution_percentage'),
        ('positions', 'get_positions'),
    )
    precomputation_filter = {
        'type__in': (TYPE_FRACTION, TYPE_PARLIAMENT),
    }
    precomputation_depends_on = ('ParliamentMember',)

    class Meta:
        unique_together = (('name', 'type'))

    def __unicode__(self):
        return u'{} ({})'.format(self.name, self.type)

    @property
    def active_members(self):
        return self.members.filter(groupmembership__until=None)

    @reify
    def active_member_count(self):
        return self.active_members.count()

    def get_avg_statement_count(self):
        agg = self.active_members.filter(
            statements__as_chairperson=False,
        ).annotate(
            models.Count('statements')
        ).aggregate(
            avg_statements=models.Avg('statements__count')
        )
        return agg['avg_statements']

    def get_avg_long_statement_count(self):
        agg = self.active_members.filter(
            statements__word_count__gte=50,
            statements__as_chairperson=False,
        ).annotate(
            models.Count('statements')
        ).aggregate(
            avg_statements=models.Avg('statements__count')
        )
        return agg['avg_statements']

    def get_avg_vote_percentage(self):
        total_percentage = 0.0
        for member in self.active_members:
            total_percentage += member.vote_percentage
        return (total_percentage / self.active_member_count
                if self.active_member_count else 0.0)

    def get_avg_discussion_contribution_percentage(self):
        agg = self.active_members.aggregate(
            avg_contrib=models.Avg('discussion_contribution_percentage')
        )
        return agg['avg_contrib']

    def get_positions(self):
        fraction_node = couch.view('sboard/by_slug', key=self.slug).one()
        return prepare_positions(fraction_node)

class GroupMembership(CrawledItem):
    member = models.ForeignKey(ParliamentMember,
                               related_name='groupmembership')
    group = models.ForeignKey(Group)
    position = models.CharField(max_length=128)
    since = models.DateField(blank=True, null=True)
    until = models.DateField(blank=True, null=True)

    def __unicode__(self):
        return u'{} - {} ({})'.format(self.group.name,
                                      self.member.full_name,
                                      self.position)

class Stenogram(CrawledItem):
    source_id = models.CharField(max_length=16, db_index=True)
    date = models.DateField()
    sitting_no = models.IntegerField()
    sitting_name = models.TextField(blank=True, null=True)
    session = models.CharField(max_length=64, blank=True, null=True)

    def __unicode__(self):
        return u'{} Nr. {}'.format(self.date, self.sitting_no)

class StenogramTopic(CrawledItem):
    stenogram = models.ForeignKey(Stenogram, related_name='topics')
    title = models.TextField()
    timestamp = models.DateTimeField()
    presenters = models.ManyToManyField(ParliamentMember,
                                        related_name='presented_topics')

    def __unicode__(self):
        return self.title[:160]

class StenogramStatement(CrawledItem):
    topic = models.ForeignKey(StenogramTopic, related_name='statements')
    speaker = models.ForeignKey(ParliamentMember, related_name='statements',
                                blank=True, null=True)
    speaker_name = models.CharField(max_length=64)
    as_chairperson = models.BooleanField(default=False)
    text = models.TextField()
    word_count = models.PositiveIntegerField(default=0)

    precomputed_fields = (
        ('word_count', 'get_word_count'),
    )

    def get_speaker_name(self):
        return self.speaker.full_name if self.speaker else self.speaker_name

    def __unicode__(self):
        return u'{}: {}'.format(self.get_speaker_name(), self.text[:160])

    def get_word_count(self):
        return words_utils.get_word_count(self.text)

class Voting(models.Model):
    stenogram_topic = models.ForeignKey(StenogramTopic, related_name='votings')
    node = NodeForeignKey()
    timestamp = models.DateTimeField()

    class Meta:
        unique_together = ('stenogram_topic', 'node')

def percentile_property(attr):
    def inner_fn(self):
        total = self.__class__.objects.count()
        return int((total - getattr(self, attr) + 1.0)
                   / total * 100 + 0.5)
    return property(inner_fn)

class Ranking(models.Model):
    created_at = models.DateTimeField(auto_now_add=True)
    modified_at = models.DateTimeField(auto_now=True)
    votes_rank = models.IntegerField(default=0)
    statement_count_rank = models.IntegerField(default=0)
    long_statement_count_rank = models.IntegerField(default=0)
    discusion_contribution_percentage_rank = models.IntegerField(default=0)

    class Meta:
        abstract = True

    votes_percentile = percentile_property(
        'votes_rank')
    statement_count_percentile = percentile_property(
        'statement_count_rank')
    long_statement_count_percentile = percentile_property(
        'long_statement_count_rank')
    discusion_contribution_percentage_percentile = percentile_property(
        'discusion_contribution_percentage_rank')

class MPRanking(Ranking):
    target = models.OneToOneField(ParliamentMember,
                                  related_name='ranking')

class GroupRanking(Ranking):
    target = models.OneToOneField(Group,
                                  related_name='ranking')

class LawProject(CrawledItem):
    source_id = models.CharField(max_length=16, db_index=True, unique=True)
    date = models.DateField()
    project_name = models.TextField()
    project_number = models.CharField(max_length=32)
    project_url = models.URLField()

    proposers = models.ManyToManyField(ParliamentMember,
                                       related_name='law_projects')

    date_passed = models.DateField(blank=True, null=True)
    passing_source_id = models.CharField(max_length=16, blank=True,
                                         null=True, db_index=True)
    passing_doc_number = models.CharField(max_length=32, blank=True, null=True)
    passing_doc_url = models.URLField(blank=True, null=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# dns.nsupdate - rfc2136 based challenge handler
# Copyright (c) Rudolf Mayerhofer, 2019
# available under the ISC license, see LICENSE
import datetime
import ipaddress
import re
import socket
import io

import dns
import dns.query
import dns.resolver
import dns.tsigkeyring
import dns.update

from modes.dns.abstract import DNSChallengeHandler

DEFAULT_KEY_ALGORITHM = "HMAC-MD5.SIG-ALG.REG.INT"

class ChallengeHandler(DNSChallengeHandler):
    @staticmethod
    def _find_first_key_name(tsig_key_file):
        try:
            with io.open(tsig_key_file) as key_file:
                key_struct = key_file.read()
                return re.search(r"key \"?([^\"{ ]+?)\"? {.*};", key_struct, re.DOTALL).group(1)
        except IOError as exc:
            print(exc)
            raise Exception(
                "A problem was encountered opening your keyfile, %s." % tsig_key_file)
        except AttributeError as exc:
            print(exc)
            raise Exception("Failed to find first key name")

    @staticmethod
    def _read_tsigkey(tsig_key_file, key_name):
        try:
            with io.open(tsig_key_file) as key_file:
                key_struct = key_file.read()
                key_data = re.search(r"key \"?%s\"? {(.*?)};" % key_name, key_struct, re.DOTALL).group(1)
                algorithm = re.search(r"algorithm ([a-zA-Z0-9_-]+?);", key_data, re.DOTALL).group(1)
                tsig_secret = re.search(r"secret \"(.*?)\"", key_data, re.DOTALL).group(1)
        except IOError as exc:
            print(exc)
            raise Exception(
                "A problem was encountered opening your keyfile, %s." % tsig_key_file)
        except AttributeError as exc:
            print(exc)
            raise Exception("Unable to decipher the keyname and secret from your keyfile.")

        keyring = dns.tsigkeyring.from_text({
            key_name: tsig_secret
        })

        if not algorithm:
            algorithm = DEFAULT_KEY_ALGORITHM

        return keyring, algorithm

    @staticmethod
    def _lookup_dns_server(domain_or_ip):
        try:
            return str(ipaddress.ip_address(domain_or_ip))
        except ValueError:
            result = socket.getaddrinfo(domain_or_ip, 53)
            if len(result) > 0:
                return result[0][4][0]
            else:
                raise ValueError("Could not lookup dns ip for {}".format(domain_or_ip))

    @staticmethod
    def _get_soa(domain, nameserver=None):
        if nameserver:
            nameservers = [nameserver]
        else:
            nameservers = dns.resolver.get_default_resolver().nameservers

        domain = dns.name.from_text(domain)
        if not domain.is_absolute():
            domain = domain.concatenate(dns.name.root)

        while domain.parent() != dns.name.root:
            request = dns.message.make_query(domain, dns.rdatatype.SOA)
            for nameserver in nameservers:
                try:
                    response = dns.query.udp(request, nameserver)
                    if response.rcode() == dns.rcode.NOERROR:
                        for answer in response.answer:
                            for item in answer:
                                if item.rdtype == dns.rdatatype.SOA:
                                    zone = domain.to_text()
                                    authoritative_ns = item.mname.to_text().split(' ')[0]
                                    return zone, authoritative_ns
                    else:
                        break
                except dns.exception.Timeout:
                    # Go to next nameserver on timeout
                    continue
                except dns.exception.DNSException:
                    # Break loop on any other error
                    break
            domain = domain.parent()
        raise Exception('Could not find Zone SOA for "{0}"'.format(domain))

    @staticmethod
    def get_challenge_type():
        return "dns-01"

    def __init__(self, config):
        DNSChallengeHandler.__init__(self, config)
        if 'nsupdate_keyfile' in config:
            if 'nsupdate_keyname' in config:
                nsupdate_keyname = config.get("nsupdate_keyname")
            else:
                nsupdate_keyname = self._find_first_key_name(config.get("nsupdate_keyfile"))

            self.keyring, self.keyalgorithm = self._read_tsigkey(config.get("nsupdate_keyfile"), nsupdate_keyname)
        else:
            self.keyring = dns.tsigkeyring.from_text({
                config.get("nsupdate_keyname"): config.get("nsupdate_keyvalue")
            })
            self.keyalgorithm = config.get("nsupdate_keyalgorithm", DEFAULT_KEY_ALGORITHM)
        self.dns_server = config.get("nsupdate_server")
        self.dns_ttl = int(config.get("nsupdate_ttl", "60"))

    def _determine_zone_and_nameserverip(self, domain):
        nameserver = self.dns_server
        if nameserver:
            nameserverip = self._lookup_dns_server(nameserver)
            zone, _ = self._get_soa(domain, nameserverip)
        else:
            zone, nameserver = self._get_soa(domain)
            nameserverip = self._lookup_dns_server(nameserver)

        return zone, nameserverip

    def add_dns_record(self, domain, txtvalue):
        zone, nameserverip = self._determine_zone_and_nameserverip(domain)
        update = dns.update.Update(zone, keyring=self.keyring, keyalgorithm=self.keyalgorithm)
        update.add(domain, self.dns_ttl, 'TXT', txtvalue)
        dns.query.tcp(update, nameserverip)
        print('Added \'{} 60 IN TXT "{}"\' to {}'.format(domain, txtvalue, nameserverip))
        return datetime.datetime.now() + datetime.timedelta(seconds=2 * self.dns_ttl)

    def remove_dns_record(self, domain, txtvalue):
        zone, nameserverip = self._determine_zone_and_nameserverip(domain)
        update = dns.update.Update(zone, keyring=self.keyring, keyalgorithm=self.keyalgorithm)
        update.delete(domain, 'TXT', txtvalue)
        dns.query.tcp(update, nameserverip)
        print('Deleted \'{} 60 IN TXT "{}"\' from {}'.format(domain, txtvalue, nameserverip))
<commit_msg>dns.nsupdate: Change status output order for better debugging<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# dns.nsupdate - rfc2136 based challenge handler
# Copyright (c) Rudolf Mayerhofer, 2019
# available under the ISC license, see LICENSE
import datetime
import ipaddress
import re
import socket
import io

import dns
import dns.query
import dns.resolver
import dns.tsigkeyring
import dns.update

from modes.dns.abstract import DNSChallengeHandler

DEFAULT_KEY_ALGORITHM = "HMAC-MD5.SIG-ALG.REG.INT"

class ChallengeHandler(DNSChallengeHandler):
    @staticmethod
    def _find_first_key_name(tsig_key_file):
        try:
            with io.open(tsig_key_file) as key_file:
                key_struct = key_file.read()
                return re.search(r"key \"?([^\"{ ]+?)\"? {.*};", key_struct, re.DOTALL).group(1)
        except IOError as exc:
            print(exc)
            raise Exception(
                "A problem was encountered opening your keyfile, %s." % tsig_key_file)
        except AttributeError as exc:
            print(exc)
            raise Exception("Failed to find first key name")

    @staticmethod
    def _read_tsigkey(tsig_key_file, key_name):
        try:
            with io.open(tsig_key_file) as key_file:
                key_struct = key_file.read()
                key_data = re.search(r"key \"?%s\"? {(.*?)};" % key_name, key_struct, re.DOTALL).group(1)
                algorithm = re.search(r"algorithm ([a-zA-Z0-9_-]+?);", key_data, re.DOTALL).group(1)
                tsig_secret = re.search(r"secret \"(.*?)\"", key_data, re.DOTALL).group(1)
        except IOError as exc:
            print(exc)
            raise Exception(
                "A problem was encountered opening your keyfile, %s." % tsig_key_file)
        except AttributeError as exc:
            print(exc)
            raise Exception("Unable to decipher the keyname and secret from your keyfile.")

        keyring = dns.tsigkeyring.from_text({
            key_name: tsig_secret
        })

        if not algorithm:
            algorithm = DEFAULT_KEY_ALGORITHM

        return keyring, algorithm

    @staticmethod
    def _lookup_dns_server(domain_or_ip):
        try:
            return str(ipaddress.ip_address(domain_or_ip))
        except ValueError:
            result = socket.getaddrinfo(domain_or_ip, 53)
            if len(result) > 0:
                return result[0][4][0]
            else:
                raise ValueError("Could not lookup dns ip for {}".format(domain_or_ip))

    @staticmethod
    def _get_soa(domain, nameserver=None):
        if nameserver:
            nameservers = [nameserver]
        else:
            nameservers = dns.resolver.get_default_resolver().nameservers

        domain = dns.name.from_text(domain)
        if not domain.is_absolute():
            domain = domain.concatenate(dns.name.root)

        while domain.parent() != dns.name.root:
            request = dns.message.make_query(domain, dns.rdatatype.SOA)
            for nameserver in nameservers:
                try:
                    response = dns.query.udp(request, nameserver)
                    if response.rcode() == dns.rcode.NOERROR:
                        for answer in response.answer:
                            for item in answer:
                                if item.rdtype == dns.rdatatype.SOA:
                                    zone = domain.to_text()
                                    authoritative_ns = item.mname.to_text().split(' ')[0]
                                    return zone, authoritative_ns
                    else:
                        break
                except dns.exception.Timeout:
                    # Go to next nameserver on timeout
                    continue
                except dns.exception.DNSException:
                    # Break loop on any other error
                    break
            domain = domain.parent()
        raise Exception('Could not find Zone SOA for "{0}"'.format(domain))

    @staticmethod
    def get_challenge_type():
        return "dns-01"

    def __init__(self, config):
        DNSChallengeHandler.__init__(self, config)
        if 'nsupdate_keyfile' in config:
            if 'nsupdate_keyname' in config:
                nsupdate_keyname = config.get("nsupdate_keyname")
            else:
                nsupdate_keyname = self._find_first_key_name(config.get("nsupdate_keyfile"))

            self.keyring, self.keyalgorithm = self._read_tsigkey(config.get("nsupdate_keyfile"), nsupdate_keyname)
        else:
            self.keyring = dns.tsigkeyring.from_text({
                config.get("nsupdate_keyname"): config.get("nsupdate_keyvalue")
            })
            self.keyalgorithm = config.get("nsupdate_keyalgorithm", DEFAULT_KEY_ALGORITHM)
        self.dns_server = config.get("nsupdate_server")
        self.dns_ttl = int(config.get("nsupdate_ttl", "60"))

    def _determine_zone_and_nameserverip(self, domain):
        nameserver = self.dns_server
        if nameserver:
            nameserverip = self._lookup_dns_server(nameserver)
            zone, _ = self._get_soa(domain, nameserverip)
        else:
            zone, nameserver = self._get_soa(domain)
            nameserverip = self._lookup_dns_server(nameserver)

        return zone, nameserverip

    def add_dns_record(self, domain, txtvalue):
        zone, nameserverip = self._determine_zone_and_nameserverip(domain)
        update = dns.update.Update(zone, keyring=self.keyring, keyalgorithm=self.keyalgorithm)
        update.add(domain, self.dns_ttl, 'TXT', txtvalue)
        print('Adding \'{} 60 IN TXT "{}"\' to {}'.format(domain, txtvalue, nameserverip))
        dns.query.tcp(update, nameserverip)
        return datetime.datetime.now() + datetime.timedelta(seconds=2 * self.dns_ttl)

    def remove_dns_record(self, domain, txtvalue):
        zone, nameserverip = self._determine_zone_and_nameserverip(domain)
        update = dns.update.Update(zone, keyring=self.keyring, keyalgorithm=self.keyalgorithm)
        update.delete(domain, 'TXT', txtvalue)
        print('Deleting \'{} 60 IN TXT "{}"\' from {}'.format(domain, txtvalue, nameserverip))
        dns.query.tcp(update, nameserverip)
<|endoftext|>


<commit_before># This file is part of the MapProxy project.
# Copyright (C) 2010 Omniscale <http://omniscale.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
Configuration loading and system initializing.
"""
from __future__ import with_statement

import types
import pkg_resources

import logging
log = logging.getLogger(__name__)

from mapproxy.core.conf_loader import Source, CacheSource
from mapproxy.core.client import auth_data_from_url
from mapproxy.wms.cache import WMSTileSource
from mapproxy.wms.layer import (DirectLayer, WMSCacheLayer, DebugLayer, VLayer,
                                 FeatureInfoSource, MultiLayer, AttributionLayer)
from mapproxy.core.layer import LayerMetaData
from mapproxy.core.grid import tile_grid_for_epsg
from mapproxy.wms.server import WMSServer
from mapproxy.wms.request import WMS100MapRequest, WMS111MapRequest, WMS130MapRequest,\
                                  WMS100FeatureInfoRequest, WMS111FeatureInfoRequest,\
                                  WMS130FeatureInfoRequest
from mapproxy.wms.client import WMSClient
from mapproxy.core.config import base_config
from mapproxy.core.request import split_mime_type
from mapproxy.core.odict import odict

def load_request_parser():
    request_parsers = {}
    for entry_point in pkg_resources.iter_entry_points('mapproxy.wms.request_parser'):
        request_parsers[entry_point.name] = entry_point
    req_parser_name = base_config().wms.request_parser
    request_parser = None
    if req_parser_name != 'default':
        if req_parser_name in request_parsers:
            request_parser = request_parsers[req_parser_name].load()
        else:
            log.warn("configured wms request parser ('%s') not found", req_parser_name)
    return request_parser

request_parser = load_request_parser()
del load_request_parser

def load_client_request():
    client_requests = {}
    for entry_point in pkg_resources.iter_entry_points('mapproxy.wms.client_request'):
        client_requests[entry_point.name] = entry_point
    req_name = base_config().wms.client_request
    client_request = None
    if req_name != 'default':
        if req_name in client_requests:
            client_request = client_requests[req_name].load()
        else:
            log.warn("configured wms client request ('%s') not found", req_name)
    return client_request
client_request = load_client_request()
del load_client_request

def create_wms_server(proxy_conf):
    layers = odict()
    for name, layer in proxy_conf.layer_confs.iteritems():
        layers[name] = configured_layer(layer)

    layers['__debug__'] = DebugLayer()
    return WMSServer(layers, proxy_conf.service_md, request_parser=request_parser)

def configured_layer(layer_conf):
    """
    Returns a configured layer (WMSCacheLayer, MultiLayer, VLayer, etc).
    """
    attribution_layer = get_attribution_layer(layer_conf)
    if layer_conf.multi_layer:
        layers = [_configured_layer(layer_conf, source, attribution_layer)
                  for source in layer_conf.sources]
        md = layer_conf.layer['md'].copy()
        md['name'] = layer_conf.name
        return MultiLayer(layers, md)
    else:
        return _configured_layer(layer_conf, layer_conf.sources, attribution_layer)

def get_attribution_layer(layer_conf):
    attribution_text = layer_conf.layer.get('attribution', {}).get('text', None)
    if attribution_text is None:
        attribution_text = layer_conf.service.get('attribution', {}).get('text', None)
    if attribution_text is None or attribution_text is '':
        return None
    attribution_inverse = layer_conf.layer.get('attribution', {}).get('inverse', None)
    if attribution_inverse is None:
        attrib = layer_conf.service.get('attribution', {})
        attribution_inverse = attrib.get('inverse', False)

    if isinstance(attribution_inverse, basestring):
        if attribution_inverse.lower() == 'true':
            attribution_inverse = True
        else:
            attribution_inverse = False
    return AttributionLayer(attribution_text, inverse=attribution_inverse)

def _configured_layer(layer_conf, sources, attribution_layer=None):
    layer = []
    for source in sources:
        try:
            layer.append(source.configured_layer())
        except NotImplementedError:
            layer.append(WMSCacheLayer(source.configured_cache()))
    layer.reverse()
    if attribution_layer is not None:
        layer.append(attribution_layer)
    md = layer_conf.layer['md'].copy()
    md['name'] = layer_conf.name
    if len(layer) == 1:
        layer = layer[0]
        layer.md = LayerMetaData(md)
        return layer
    return VLayer(md, layer)

class DirectSource(Source):
    def __init__(self, layer_conf, source, param=None):
        Source.__init__(self, layer_conf, source, param)
        self.requests = [create_request(self.source['req'], self.param)]
    def has_featureinfo(self):
        return self.source.get('wms_opts', {}).get('featureinfo', False)
    def configured_layer(self):
        clients = wms_clients_for_requests(self.requests)
        return DirectLayer(clients[0], queryable=self.has_featureinfo())

class DebugSource(Source):
    def configured_layer(self):
        return DebugLayer()

class WMSCacheSource(CacheSource):
    def __init__(self, layer_conf, source, param=None):
        CacheSource.__init__(self, layer_conf, source, param)
        version = self.wms_client_version()
        self.requests = [create_request(self.source['req'], self.param,
                                        req_type='map', version=version)]
        self.fi_requests = []
        if self.has_featureinfo():
            self.fi_requests.append(create_request(self.source['req'], self.param,
                                                   req_type='featureinfo',
                                                   version=version))

    def wms_client_version(self):
        return self.source.get('wms_opts', {}).get('version', '1.1.1')

    def has_featureinfo(self):
        return self.source.get('wms_opts', {}).get('featureinfo', False)

    def configured_layer(self):
        fi_source = None
        if self.fi_requests:
            fi_clients = wms_clients_for_requests(self.fi_requests)
            fi_source = FeatureInfoSource(fi_clients)
        return WMSCacheLayer(self.configured_cache(), fi_source=fi_source)

    def init_grid(self):
        req = self.requests[0]
        bbox = req.params.bbox
        self.transparent = req.params.transparent
        res = self.param['res']
        srs = self.param['srs']
        tile_size = self.layer_conf.layer.get('param', {}).get('tile_size', (256, 256))
        self.grid = tile_grid_for_epsg(epsg=srs, tile_size=tile_size, bbox=bbox, res=res)
    def init_tile_source(self):
        clients = wms_clients_for_requests(self.requests[::-1])
        format = self.layer_conf.layer.get('param', {}).get('format', None)
        if format is not None:
            _mime_class, format, _options = split_mime_type(format)
        self.src = WMSTileSource(self.grid, clients, format=format,
                                 meta_size=base_config().cache.meta_size,
                                 meta_buffer=base_config().cache.meta_buffer)

    def merge(self, other):
        if isinstance(other, WMSCacheSource) and self.param == other.param:
            self.requests.extend(other.requests)
            self.fi_requests.extend(other.fi_requests)
            return self
        else:
            return None

def wms_clients_for_requests(requests):
    clients = []
    for req in requests:
        http_client = None
        url, (username, password) = auth_data_from_url(req.url)
        if username and password:
            req.url = url
            http_client = HTTPClient(url, username, password)
        clients.append(WMSClient(req, client_request, http_client=http_client))
    return clients

wms_version_requests = {'1.0.0': {'featureinfo': WMS100FeatureInfoRequest,
                                  'map': WMS100MapRequest,},
                        '1.1.1': {'featureinfo': WMS111FeatureInfoRequest,
                                  'map': WMS111MapRequest,},
                        '1.3.0': {'featureinfo': WMS130FeatureInfoRequest,
                                  'map': WMS130MapRequest,},
                       }

def create_request(req_data, param, req_type='map', version='1.1.1'):
    url = req_data['url']
    req_data = req_data.copy()
    del req_data['url']
    if 'request_format' in param:
        req_data['format'] = param['request_format']
    else:
        req_data['format'] = param['format']
    req_data['bbox'] = param['bbox']
    if isinstance(req_data['bbox'], types.ListType):
        req_data['bbox'] = ','.join(str(x) for x in req_data['bbox'])
    req_data['srs'] = param['srs']

    return wms_version_requests[version][req_type](url=url, param=req_data)
<commit_msg>fixed missing import<commit_after># This file is part of the MapProxy project.
# Copyright (C) 2010 Omniscale <http://omniscale.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
Configuration loading and system initializing.
"""
from __future__ import with_statement

import types
import pkg_resources

import logging
log = logging.getLogger(__name__)

from mapproxy.core.conf_loader import Source, CacheSource
from mapproxy.core.client import auth_data_from_url, HTTPClient
from mapproxy.wms.cache import WMSTileSource
from mapproxy.wms.layer import (DirectLayer, WMSCacheLayer, DebugLayer, VLayer,
                                 FeatureInfoSource, MultiLayer, AttributionLayer)
from mapproxy.core.layer import LayerMetaData
from mapproxy.core.grid import tile_grid_for_epsg
from mapproxy.wms.server import WMSServer
from mapproxy.wms.request import WMS100MapRequest, WMS111MapRequest, WMS130MapRequest,\
                                  WMS100FeatureInfoRequest, WMS111FeatureInfoRequest,\
                                  WMS130FeatureInfoRequest
from mapproxy.wms.client import WMSClient
from mapproxy.core.config import base_config
from mapproxy.core.request import split_mime_type
from mapproxy.core.odict import odict

def load_request_parser():
    request_parsers = {}
    for entry_point in pkg_resources.iter_entry_points('mapproxy.wms.request_parser'):
        request_parsers[entry_point.name] = entry_point
    req_parser_name = base_config().wms.request_parser
    request_parser = None
    if req_parser_name != 'default':
        if req_parser_name in request_parsers:
            request_parser = request_parsers[req_parser_name].load()
        else:
            log.warn("configured wms request parser ('%s') not found", req_parser_name)
    return request_parser

request_parser = load_request_parser()
del load_request_parser

def load_client_request():
    client_requests = {}
    for entry_point in pkg_resources.iter_entry_points('mapproxy.wms.client_request'):
        client_requests[entry_point.name] = entry_point
    req_name = base_config().wms.client_request
    client_request = None
    if req_name != 'default':
        if req_name in client_requests:
            client_request = client_requests[req_name].load()
        else:
            log.warn("configured wms client request ('%s') not found", req_name)
    return client_request
client_request = load_client_request()
del load_client_request

def create_wms_server(proxy_conf):
    layers = odict()
    for name, layer in proxy_conf.layer_confs.iteritems():
        layers[name] = configured_layer(layer)

    layers['__debug__'] = DebugLayer()
    return WMSServer(layers, proxy_conf.service_md, request_parser=request_parser)

def configured_layer(layer_conf):
    """
    Returns a configured layer (WMSCacheLayer, MultiLayer, VLayer, etc).
    """
    attribution_layer = get_attribution_layer(layer_conf)
    if layer_conf.multi_layer:
        layers = [_configured_layer(layer_conf, source, attribution_layer)
                  for source in layer_conf.sources]
        md = layer_conf.layer['md'].copy()
        md['name'] = layer_conf.name
        return MultiLayer(layers, md)
    else:
        return _configured_layer(layer_conf, layer_conf.sources, attribution_layer)

def get_attribution_layer(layer_conf):
    attribution_text = layer_conf.layer.get('attribution', {}).get('text', None)
    if attribution_text is None:
        attribution_text = layer_conf.service.get('attribution', {}).get('text', None)
    if attribution_text is None or attribution_text is '':
        return None
    attribution_inverse = layer_conf.layer.get('attribution', {}).get('inverse', None)
    if attribution_inverse is None:
        attrib = layer_conf.service.get('attribution', {})
        attribution_inverse = attrib.get('inverse', False)

    if isinstance(attribution_inverse, basestring):
        if attribution_inverse.lower() == 'true':
            attribution_inverse = True
        else:
            attribution_inverse = False
    return AttributionLayer(attribution_text, inverse=attribution_inverse)

def _configured_layer(layer_conf, sources, attribution_layer=None):
    layer = []
    for source in sources:
        try:
            layer.append(source.configured_layer())
        except NotImplementedError:
            layer.append(WMSCacheLayer(source.configured_cache()))
    layer.reverse()
    if attribution_layer is not None:
        layer.append(attribution_layer)
    md = layer_conf.layer['md'].copy()
    md['name'] = layer_conf.name
    if len(layer) == 1:
        layer = layer[0]
        layer.md = LayerMetaData(md)
        return layer
    return VLayer(md, layer)

class DirectSource(Source):
    def __init__(self, layer_conf, source, param=None):
        Source.__init__(self, layer_conf, source, param)
        self.requests = [create_request(self.source['req'], self.param)]
    def has_featureinfo(self):
        return self.source.get('wms_opts', {}).get('featureinfo', False)
    def configured_layer(self):
        clients = wms_clients_for_requests(self.requests)
        return DirectLayer(clients[0], queryable=self.has_featureinfo())

class DebugSource(Source):
    def configured_layer(self):
        return DebugLayer()

class WMSCacheSource(CacheSource):
    def __init__(self, layer_conf, source, param=None):
        CacheSource.__init__(self, layer_conf, source, param)
        version = self.wms_client_version()
        self.requests = [create_request(self.source['req'], self.param,
                                        req_type='map', version=version)]
        self.fi_requests = []
        if self.has_featureinfo():
            self.fi_requests.append(create_request(self.source['req'], self.param,
                                                   req_type='featureinfo',
                                                   version=version))

    def wms_client_version(self):
        return self.source.get('wms_opts', {}).get('version', '1.1.1')

    def has_featureinfo(self):
        return self.source.get('wms_opts', {}).get('featureinfo', False)

    def configured_layer(self):
        fi_source = None
        if self.fi_requests:
            fi_clients = wms_clients_for_requests(self.fi_requests)
            fi_source = FeatureInfoSource(fi_clients)
        return WMSCacheLayer(self.configured_cache(), fi_source=fi_source)

    def init_grid(self):
        req = self.requests[0]
        bbox = req.params.bbox
        self.transparent = req.params.transparent
        res = self.param['res']
        srs = self.param['srs']
        tile_size = self.layer_conf.layer.get('param', {}).get('tile_size', (256, 256))
        self.grid = tile_grid_for_epsg(epsg=srs, tile_size=tile_size, bbox=bbox, res=res)
    def init_tile_source(self):
        clients = wms_clients_for_requests(self.requests[::-1])
        format = self.layer_conf.layer.get('param', {}).get('format', None)
        if format is not None:
            _mime_class, format, _options = split_mime_type(format)
        self.src = WMSTileSource(self.grid, clients, format=format,
                                 meta_size=base_config().cache.meta_size,
                                 meta_buffer=base_config().cache.meta_buffer)

    def merge(self, other):
        if isinstance(other, WMSCacheSource) and self.param == other.param:
            self.requests.extend(other.requests)
            self.fi_requests.extend(other.fi_requests)
            return self
        else:
            return None

def wms_clients_for_requests(requests):
    clients = []
    for req in requests:
        http_client = None
        url, (username, password) = auth_data_from_url(req.url)
        if username and password:
            req.url = url
            http_client = HTTPClient(url, username, password)
        clients.append(WMSClient(req, client_request, http_client=http_client))
    return clients

wms_version_requests = {'1.0.0': {'featureinfo': WMS100FeatureInfoRequest,
                                  'map': WMS100MapRequest,},
                        '1.1.1': {'featureinfo': WMS111FeatureInfoRequest,
                                  'map': WMS111MapRequest,},
                        '1.3.0': {'featureinfo': WMS130FeatureInfoRequest,
                                  'map': WMS130MapRequest,},
                       }

def create_request(req_data, param, req_type='map', version='1.1.1'):
    url = req_data['url']
    req_data = req_data.copy()
    del req_data['url']
    if 'request_format' in param:
        req_data['format'] = param['request_format']
    else:
        req_data['format'] = param['format']
    req_data['bbox'] = param['bbox']
    if isinstance(req_data['bbox'], types.ListType):
        req_data['bbox'] = ','.join(str(x) for x in req_data['bbox'])
    req_data['srs'] = param['srs']

    return wms_version_requests[version][req_type](url=url, param=req_data)
<|endoftext|>


<commit_before>import matplotlib.pyplot as pyplot
import numpy as np
import sys
import math
from Trip import Trip
import os
import random
from sklearn.metrics import roc_auc_score
#from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.externals import joblib
from sklearn.svm import SVC
import random
#from sklearn.feature_selection import SelectKBest
#from sklearn.feature_selection import chi2

num_selfTrips = 140
num_testTrips = 60
num_NOTselfTrips = 140
size = num_testTrips+num_selfTrips

class Driver(object):

	def __init__(self, driverName):
		self.name = driverName

	#we might have to change the rounding at some point, but it's a good way to start
	def calculateResults(self,predicted, true):
		tp = 0
		tn = 0
		fp = 0
		fn = 0

		for i in range (len(true)):
			if (true[i] == 1 and round(predicted[i]) == 1):
				tp+=1
			if (true[i] == 1 and round(predicted[i]) == 0):
				fn+=1
			if (true[i] == 0 and round(predicted[i]) == 1):
				fp+=1
			if (true[i] == 0 and round(predicted[i]) == 0):
				tn+=1

		#print tp, tn, fp, fn
		prec = float(tp)/(tp+fp)
		recall = float(tp)/(tp+fn)
		acc = float (tp+tn)/(tp+tn+fp+fn)
		#print 'Precision: ', prec
		#print 'Recall: ', recall
		auc = roc_auc_score(true, predicted)
		#auc = 0
		return (prec, recall, auc, acc)

	def splitData(self, data, labels, k):

		traintrips = []
		target = []
		testtrips = []
		testtarget =[]
		inc = size/5
		#print len (data)
		#print size*2
		for i in range (size*2):

			if (i>=(k*inc) and i<(k+1)*inc) or (i>=((k+5)*inc) and i<((k+6)*inc)):
				testtrips.append(data[i])
				testtarget.append(labels[i])
			else:
				traintrips.append(data[i])
				target.append(labels[i])

		return traintrips, target, testtrips, testtarget

	def classify(self):

		#get training trips for this driver
		f = open("driver_stats/"+str(self.name)+"_training.csv")
		#f.readline() #skip header labels
		#traintrips
		dataset = np.genfromtxt(f, delimiter=',')
		f.close()

		#get list of labels for the trips in traintrips
		g = open("driver_stats/trainingLabels.csv")
		#target
		labels = np.genfromtxt(g, delimiter=',')
		g.close()

		"""
		#get test trips for this driver
		h = open("driver_stats/"+str(self.name)+"_test.csv")
		testtrips = np.genfromtxt(h, delimiter=',')
		h.close()
		k = open("driver_stats/testLabels.csv")
		test_target = np.genfromtxt(k, delimiter=',')
		k.close()
		"""

		inc = size/5
		res = []
		for k in range(0,5):

			traintrips, target, testtrips, testtarget = self.splitData(dataset, labels, k)

			clf = RandomForestClassifier(n_estimators=500)
			#(random_state=1, n_estimators=500, n_jobs=1, min_samples_leaf=3)

			#(n_estimators=500)

			#print traintrips[0]
			clf.fit(traintrips, target)
			predLabels = clf.predict (testtrips)
			#print predLabels
			#print testtarget
			res.append(self.calculateResults(predLabels, testtarget))
			#print self.calculateResults(predLabels, testtarget)

		return res

		#print clf.score(testtrips, test_target)

		#joblib.dump(clf, "driver_stats/"+str(self.name)+"_clf.pkl")
	"""
	def writeCSV(self):
		g = open ("driver_stats/"+str(self.name)+"_trips.csv", "w")
		#a header and then the features for each trip
		g.write("advSpeed,tripDist\n")
		for i in range (1,num_selfTrips+1):
			t = Trip("../drivers/"+str(self.name)+"/"+str(i)+".csv")
			g.write(t.printFeatures())
		g.close()
	"""
	def getRandomDriverTrips(self, numtrips, numNotDrivers, feat):
		notDrivers = os.listdir("../drivers/")
		copy = notDrivers[1:]
		random.shuffle(copy)
		notDrivers[1:] = copy
		if numNotDrivers == 0 or numNotDrivers >= len(notDrivers):
			numNotDrivers = len(notDrivers)-1

		tripList = []
		for i in range(numtrips):
			dnum = notDrivers[random.randint(1, numNotDrivers)] #sample a random driver
			while dnum == self.name: #don't sample from self
				dnum = notDrivers[random.randint(1, numNotDrivers)]
			tnum = random.randint(1,200)#sample a random trip
			t = Trip("../drivers/"+str(dnum)+"/"+str(tnum)+".csv", feat)
			tripList.append(t.printFeatures())
		return tripList
	"""
	def writeCSV_notDriver(self):
		#list other drivers in directory, since their numbers skip around
		notDrivers = os.listdir("../drivers/")

		g = open ("driver_stats/"+str(self.name)+"_NOTtrips.csv", "w")
		tripList = self.getRandomDriverTrips(num_NOTselfTrips+num_testTrips)
		for other in tripList:
			g.write(other)
		g.close()
	"""

	def writeCSV_training(self, order, numNotDrivers, feat):
		g = open ("driver_stats/"+str(self.name)+"_training.csv", "w")
		#a header and then the features for each trip
			#g.write("advSpeed,tripDist\n")
		#first trips from this driver
		for i in range (0,num_selfTrips+num_testTrips):
			#print i
			t = Trip("../drivers/"+str(self.name)+"/"+str(order[i])+".csv", feat)
			g.write(t.printFeatures())
		#trips from other drivers
		tripList = self.getRandomDriverTrips(num_NOTselfTrips+num_testTrips, numNotDrivers, feat)
		for other in tripList:
			g.write(other)
		g.close()

	def writeCSV_labels(self):
		#file containing training labels, same for any driver
		h = open ("driver_stats/"+"trainingLabels.csv", "w")
		for i in range(num_selfTrips+num_testTrips):
			h.write(str(1)+"\n")
		for i in range(num_NOTselfTrips+num_testTrips):
			h.write(str(0)+"\n")
		h.close()

	def createDataSets(self, numNotDrivers, feat):
		order = [i for i in range(1, 201)]
		random.shuffle(order)
		self.writeCSV_training(order, numNotDrivers, feat)
		self.writeCSV_labels()
		#self.writeCSV_test(order)
		#self.writeCSV_testlabels()

#d1 = Driver(sys.argv[1])
#d1.createDataSets()
#print d1.classify()

"""d2 = Driver(sys.argv[2])

pyplot.hist(d1.advSpeed, 10, color='blue')
pyplot.hist(d2.advSpeed, 10, color='red')
pyplot.show()"""

"""
fig = pyplot.figure()
f_dist = fig.add_subplot(111)
f_dist.scatter(d1.distance, d1.advSpeed, c='b')
f_dist.scatter(d2.distance, d2.advSpeed, c='r')
pyplot.show()"""<commit_msg>minor changes<commit_after>import matplotlib.pyplot as pyplot
import numpy as np
import sys
import math
from Trip import Trip
import os
import random
from sklearn.metrics import roc_auc_score
#from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.externals import joblib
from sklearn.svm import SVC
import random
#from sklearn.feature_selection import SelectKBest
#from sklearn.feature_selection import chi2

num_selfTrips = 140
num_testTrips = 60
num_NOTselfTrips = 140
size = num_testTrips+num_selfTrips

class Driver(object):

	def __init__(self, driverName):
		self.name = driverName

	#we might have to change the rounding at some point, but it's a good way to start
	def calculateResults(self,predicted, true):
		tp = 0
		tn = 0
		fp = 0
		fn = 0

		for i in range (len(true)):
			if (true[i] == 1 and round(predicted[i]) == 1):
				tp+=1
			if (true[i] == 1 and round(predicted[i]) == 0):
				fn+=1
			if (true[i] == 0 and round(predicted[i]) == 1):
				fp+=1
			if (true[i] == 0 and round(predicted[i]) == 0):
				tn+=1

		#print tp, tn, fp, fn
		prec = float(tp)/(tp+fp)
		recall = float(tp)/(tp+fn)
		acc = float (tp+tn)/(tp+tn+fp+fn)
		#print 'Precision: ', prec
		#print 'Recall: ', recall
		auc = roc_auc_score(true, predicted)
		#auc = 0
		return (prec, recall, auc, acc)

	def splitData(self, data, labels, k):

		traintrips = []
		target = []
		testtrips = []
		testtarget =[]
		inc = size/5
		#print len (data)
		#print size*2
		for i in range (size*2):

			if (i>=(k*inc) and i<(k+1)*inc) or (i>=((k+5)*inc) and i<((k+6)*inc)):
				testtrips.append(data[i])
				testtarget.append(labels[i])
			else:
				traintrips.append(data[i])
				target.append(labels[i])

		return traintrips, target, testtrips, testtarget

	def classify(self):

		#get training trips for this driver
		f = open("driver_stats/"+str(self.name)+"_training.csv")
		#f.readline() #skip header labels
		#traintrips
		dataset = np.genfromtxt(f, delimiter=',')
		f.close()

		#get list of labels for the trips in traintrips
		g = open("driver_stats/trainingLabels.csv")
		#target
		labels = np.genfromtxt(g, delimiter=',')
		g.close()

		print dataset[0]
		"""
		#get test trips for this driver
		h = open("driver_stats/"+str(self.name)+"_test.csv")
		testtrips = np.genfromtxt(h, delimiter=',')
		h.close()
		k = open("driver_stats/testLabels.csv")
		test_target = np.genfromtxt(k, delimiter=',')
		k.close()
		"""

		inc = size/5
		res = []
		for k in range(0,5):

			traintrips, target, testtrips, testtarget = self.splitData(dataset, labels, k)

			clf = RandomForestClassifier(n_estimators=500)
			#(random_state=1, n_estimators=500, n_jobs=1, min_samples_leaf=3)

			#(n_estimators=500)

			#print traintrips[0]
			clf.fit(traintrips, target)
			predLabels = clf.predict (testtrips)
			#print predLabels
			#print testtarget
			res.append(self.calculateResults(predLabels, testtarget))
			#print self.calculateResults(predLabels, testtarget)

		return res

		#print clf.score(testtrips, test_target)

		#joblib.dump(clf, "driver_stats/"+str(self.name)+"_clf.pkl")
	"""
	def writeCSV(self):
		g = open ("driver_stats/"+str(self.name)+"_trips.csv", "w")
		#a header and then the features for each trip
		g.write("advSpeed,tripDist\n")
		for i in range (1,num_selfTrips+1):
			t = Trip("../drivers/"+str(self.name)+"/"+str(i)+".csv")
			g.write(t.printFeatures())
		g.close()
	"""
	def getRandomDriverTrips(self, numtrips, numNotDrivers, feat):
		notDrivers = os.listdir("../drivers/")
		copy = notDrivers[1:]
		random.shuffle(copy)
		notDrivers[1:] = copy
		if numNotDrivers == 0 or numNotDrivers >= len(notDrivers):
			numNotDrivers = len(notDrivers)-1

		tripList = []
		for i in range(numtrips):
			dnum = notDrivers[random.randint(1, numNotDrivers)] #sample a random driver
			while dnum == self.name: #don't sample from self
				dnum = notDrivers[random.randint(1, numNotDrivers)]
			tnum = random.randint(1,200)#sample a random trip
			t = Trip("../drivers/"+str(dnum)+"/"+str(tnum)+".csv", feat)
			tripList.append(t.printFeatures())
		return tripList
	"""
	def writeCSV_notDriver(self):
		#list other drivers in directory, since their numbers skip around
		notDrivers = os.listdir("../drivers/")

		g = open ("driver_stats/"+str(self.name)+"_NOTtrips.csv", "w")
		tripList = self.getRandomDriverTrips(num_NOTselfTrips+num_testTrips)
		for other in tripList:
			g.write(other)
		g.close()
	"""

	def writeCSV_training(self, order, numNotDrivers, feat):
		g = open ("driver_stats/"+str(self.name)+"_training.csv", "w")
		#a header and then the features for each trip
			#g.write("advSpeed,tripDist\n")
		#first trips from this driver
		for i in range (0,num_selfTrips+num_testTrips):
			#print i
			t = Trip("../drivers/"+str(self.name)+"/"+str(order[i])+".csv", feat)
			g.write(t.printFeatures())
		#trips from other drivers
		tripList = self.getRandomDriverTrips(num_NOTselfTrips+num_testTrips, numNotDrivers, feat)
		for other in tripList:
			g.write(other)
		g.close()

	def writeCSV_labels(self):
		#file containing training labels, same for any driver
		h = open ("driver_stats/"+"trainingLabels.csv", "w")
		for i in range(num_selfTrips+num_testTrips):
			h.write(str(1)+"\n")
		for i in range(num_NOTselfTrips+num_testTrips):
			h.write(str(0)+"\n")
		h.close()

	def createDataSets(self, numNotDrivers, feat):
		order = [i for i in range(1, 201)]
		random.shuffle(order)
		self.writeCSV_training(order, numNotDrivers, feat)
		self.writeCSV_labels()
		#self.writeCSV_test(order)
		#self.writeCSV_testlabels()

#d1 = Driver(sys.argv[1])
#d1.createDataSets()
#print d1.classify()

"""d2 = Driver(sys.argv[2])

pyplot.hist(d1.advSpeed, 10, color='blue')
pyplot.hist(d2.advSpeed, 10, color='red')
pyplot.show()"""

"""
fig = pyplot.figure()
f_dist = fig.add_subplot(111)
f_dist.scatter(d1.distance, d1.advSpeed, c='b')
f_dist.scatter(d2.distance, d2.advSpeed, c='r')
pyplot.show()"""<|endoftext|>


<commit_before>#!/usr/bin/env python

import json
from mimetypes import guess_type
import subprocess

from flask import abort, Blueprint

import app_config
import copytext
from render_utils import flatten_app_config

static_app = Blueprint('static_app', __name__)

@static_app.route('/js/templates.js')
def _templates_js():
    """
    Render JST templates.
    """
    r = subprocess.check_output(["node_modules/universal-jst/bin/jst.js", "--template", "underscore", "jst"])

    return r, 200, { 'Content-Type': 'application/javascript' }

def less(filename, static_path=''):
    """
    Render LESS files.
    """
    r = subprocess.check_output(["node_modules/less/bin/lessc", "%s/less/%s" % (static_path, filename)])

    return r, 200, { 'Content-Type': 'text/css' }

@static_app.route('/less/<string:filename>')
def _less(filename):
    return less(filename)

@static_app.route('/js/app_config.js')
def _app_config_js():
    """
    Render app configuration to javascript.
    """
    config = flatten_app_config()
    js = 'window.APP_CONFIG = ' + json.dumps(config)

    return js, 200, { 'Content-Type': 'application/javascript' }

@static_app.route('/js/copy.js')
def _copy_js():
    """
    Render copytext to javascript.
    """
    copy = 'window.COPY = ' + copytext.Copy(app_config.COPY_PATH).json()

    return copy, 200, { 'Content-Type': 'application/javascript' }

def static(path, static_path=''):
    """
    Serve arbitrary files.
    """
    real_path = '%s/www/%s' % (static_path, path)
    print real_path

    try:
        with open(real_path) as f:
            return f.read(), 200, { 'Content-Type': guess_type(real_path)[0] }
    except IOError:
        abort(404)

@static_app.route('/<path:path>')
def _static(path):
    return static(path)
<commit_msg>Fix paths I broke.<commit_after>#!/usr/bin/env python

import json
from mimetypes import guess_type
import subprocess

from flask import abort, Blueprint

import app_config
import copytext
from render_utils import flatten_app_config

static_app = Blueprint('static_app', __name__)

@static_app.route('/js/templates.js')
def _templates_js():
    """
    Render JST templates.
    """
    r = subprocess.check_output(["node_modules/universal-jst/bin/jst.js", "--template", "underscore", "jst"])

    return r, 200, { 'Content-Type': 'application/javascript' }

def less(filename, static_path=''):
    """
    Render LESS files.
    """
    r = subprocess.check_output(["node_modules/less/bin/lessc", ('%s/less/%s' % (static_path, filename)).lstrip('/')])

    return r, 200, { 'Content-Type': 'text/css' }

@static_app.route('/less/<string:filename>')
def _less(filename):
    return less(filename)

@static_app.route('/js/app_config.js')
def _app_config_js():
    """
    Render app configuration to javascript.
    """
    config = flatten_app_config()
    js = 'window.APP_CONFIG = ' + json.dumps(config)

    return js, 200, { 'Content-Type': 'application/javascript' }

@static_app.route('/js/copy.js')
def _copy_js():
    """
    Render copytext to javascript.
    """
    copy = 'window.COPY = ' + copytext.Copy(app_config.COPY_PATH).json()

    return copy, 200, { 'Content-Type': 'application/javascript' }

def static(path, static_path=''):
    """
    Serve arbitrary files.
    """
    real_path = ('%s/www/%s' % (static_path, path)).lstrip('/')

    try:
        with open(real_path) as f:
            return f.read(), 200, { 'Content-Type': guess_type(real_path)[0] }
    except IOError:
        abort(404)

@static_app.route('/<path:path>')
def _static(path):
    return static(path)
<|endoftext|>