<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str

import pickle
import numpy as np
import matplotlib as mpl
mpl.use('Agg')
from matplotlib import pyplot as plt
from datetime import datetime

from indra.util.get_version import get_git_info
from indra.tools.reading.util.reporter import Reporter

class DbAwsStatReporter(Reporter):
    """A class to handle generating the reports made at the end of a job."""
    def __init__(self, job_name, s3_log_prefix, s3, bucket_name):
        super(DbAwsStatReporter, self).__init__('%s_summary' % job_name)
        self.s3_prefix = s3_log_prefix + 'statistics/'
        self.bucket_name = bucket_name
        self.s3 = s3
        self.summary_dict = {}
        self.hist_dict = {}
        self.set_title("Report of Database Reading Batch Job")
        self.sections = {'Summary Statistics': [], 'Plots': [], 'Git Info': [],
                         'Job Info': []}
        self._make_job_line('Job Name', job_name)
        self._make_job_line('Job s3 prefix', s3_log_prefix)
        self._get_git_info()
        self.set_section_order(['Job Info', 'Git Info', 'Summary Statistics',
                                'Plots'])
        return

    def _get_git_info(self):
        git_info_dict = get_git_info()
        text_file_content = ''
        for key, val in git_info_dict.items():
            label = key.replace('_', ' ').capitalize()
            text_file_content += '%s: %s\n' % (label, val)
            self.add_story_text('%s: %s' % (label, val), section='Git Info')
        self.s3.put_object(Key=self.s3_prefix + 'git_info.txt',
                           Body=text_file_content, Bucket=self.bucket_name)
        return

    def _plot_hist(self, agged, agg_over, data):
        fig = plt.figure()
        plt.hist(data, bins=np.arange(len(data)))
        plt.xlabel('Number of %s for %s' % (agged, agg_over))
        plt.ylabel('Number of %s with a given number of %s' % (agg_over, agged))
        fname = '%s_per_%s.png' % (agged, agg_over)
        fig.set_size_inches(6, 4)
        fig.savefig(fname)
        with open(fname, 'rb') as f:
            s3_key = self.s3_prefix + fname
            self.s3.put_object(Key=s3_key, Body=f.read(),
                               Bucket=self.bucket_name)
        self.add_story_image(fname, width=6, height=4, section='Plots')
        return

    def _make_timing_report(self, starts, ends):
        # Report on the timing
        timing_str = ''
        for step in ['reading', 'statement production', 'stats']:
            time_taken = ends[step] - starts[step]
            timing_str += ('%22s: start: %s, end: %s, duration: %s\n'
                           % (step, starts[step], ends[step], time_taken))

        self.s3.put_object(Key=self.s3_prefix + 'timing.txt', Body=timing_str,
                           Bucket=self.bucket_name)
        return

    def _stash_data(self):
        """Store the data in pickle files. This should be done last."""
        self.s3.put_object(Key=self.s3_prefix + 'hist_data.pkl',
                           Body=pickle.dumps(self.hist_dict),
                           Bucket=self.bucket_name)
        self.s3.put_object(Key=self.s3_prefix + 'sum_data.pkl',
                           Bucket=self.bucket_name,
                           Body=pickle.dumps(self.summary_dict))
        return

    def _populate_hist_data(self, readings_with_stmts, readings_with_no_stmts):
        # Do a bunch of aggregation
        tc_rd_dict = {}
        tc_stmt_dict = {}
        rd_stmt_dict = {}
        reader_stmts = {}
        reader_tcids = {}
        reader_rids = {}
        for rid, tcid, reader, stmts in readings_with_stmts:
            # Handle things keyed by tcid
            if tcid not in tc_rd_dict.keys():
                tc_rd_dict[tcid] = {rid}
                tc_stmt_dict[tcid] = set(stmts)
            else:
                tc_rd_dict[tcid].add(rid)
                tc_stmt_dict[tcid] |= set(stmts)

            # Handle things keyed by rid
            if rid not in rd_stmt_dict.keys():
                rd_stmt_dict[rid] = set(stmts)
            else:
                rd_stmt_dict[rid] |= set(stmts)  # this shouldn't really happen.

            # Handle things keyed by reader
            if reader not in reader_stmts.keys():
                reader_stmts[reader] = set(stmts)
                reader_tcids[reader] = {tcid}
                reader_rids[reader] = {rid}
            else:
                reader_stmts[reader] |= set(stmts)
                reader_tcids[reader].add(tcid)
                reader_rids[reader].add(rid)

        for rid, tcid, reader, _ in readings_with_no_stmts:
            # Handle things keyed by tcid
            if tcid not in tc_rd_dict.keys():
                tc_rd_dict[tcid] = {rid}
            else:
                tc_rd_dict[tcid].add(rid)

            # Handle things keyed by reader
            if reader not in reader_stmts.keys():
                reader_tcids[reader] = {tcid}
                reader_rids[reader] = {rid}
            else:
                reader_tcids[reader].add(tcid)
                reader_rids[reader].add(rid)

        # Produce some numpy count arrays.
        self.hist_dict[('readings', 'text content')] = \
            np.array([len(rid_set) for rid_set in tc_rd_dict.values()])
        self.hist_dict[('stmts', 'text content')] = \
            np.array([len(stmts) for stmts in tc_stmt_dict.values()])
        self.hist_dict[('stmts', 'readings')] = \
            np.array([len(stmts) for stmts in rd_stmt_dict.values()])
        self.hist_dict[('stmts', 'readers')] = \
            np.array([len(stmts) for stmts in reader_stmts.values()])
        self.hist_dict[('text content', 'reader')] = \
            np.array([len(tcid_set) for tcid_set in reader_tcids.values()])
        self.hist_dict[('readings', 'reader')] = \
            np.array([len(rid_set) for rid_set in reader_rids.values()])
        return

    def _make_histograms(self):
        # Produce the histograms
        for (agged, agg_over), data in self.hist_dict.items():
            self._plot_hist(agged, agg_over, data)
            label = '%s per %s' % (agged, agg_over)
            stat_dict = {'mean': data.mean(), 'std': data.std(),
                         'median': np.median(data)}
            self.add_story_text(str(stat_dict), style='Code', section='Plots')
            self.summary_dict[label.capitalize()] = {'mean': data.mean(),
                                                     'std': data.std(),
                                                     'median': np.median(data)}
        return

    def _make_text_summary(self):
        text_report_str = ''
        top_labels = ['Total readings', 'Content processed',
                      'Statements produced']
        for label in top_labels:
            text_str = '%s: %d\n' % (label, self.summary_dict[label])
            self.add_story_text(text_str, section='Summary Statistics')
            text_report_str += '%s: %d\n' % (label, self.summary_dict[label])

        for label, data in self.summary_dict.items():
            if label in top_labels:
                continue
            if isinstance(data, dict):
                text_report_str += '%s:\n' % label
                text_report_str += '\n'.join(['\t%s: %d' % (k, v)
                                              for k, v in data.items()])
                text_report_str += '\n'
            else:
                text_str = '%s: %d\n' % (label, data)
                self.add_story_text(text_str, section='Summary Statistics')
                text_report_str += text_str
        self.s3.put_object(Key=self.s3_prefix + 'summary.txt',
                           Body=text_report_str, Bucket=self.bucket_name)
        return

    def _make_job_line(self, key, value):
        self.add_story_text(key, section='Job Info', space=(1, 6))
        self.add_story_text(value, section='Job Info', style='Code')

    def report_statistics(self, reading_outputs, stmt_outputs, starts, ends):
        starts['stats'] = datetime.now()
        for k, end in ends.items():
            self._make_job_line(k + ' start', str(starts[k]))
            self._make_job_line(k + ' end', str(end))
            self._make_job_line(k + ' duration', str(end-starts[k]))

        self.summary_dict['Total readings'] = len(reading_outputs)
        reading_stmts = [(rd.reading_id, rd.tcid, rd.reader, rd.get_statements())
                         for rd in reading_outputs]
        self.summary_dict['Content processed'] = \
            len({t[1] for t in reading_stmts})
        self.summary_dict['Statements produced'] = len(stmt_outputs)
        self.s3.put_object(Key=self.s3_prefix + 'raw_tuples.pkl',
                           Body=pickle.dumps([t[:-1] + (len(t[-1]),)
                                              for t in reading_stmts]),
                           Bucket=self.bucket_name)

        readings_with_stmts = []
        readings_with_no_stmts = []
        for t in reading_stmts:
            if t[-1]:
                readings_with_stmts.append(t)
            else:
                readings_with_no_stmts.append(t)

        self.summary_dict['Readings with no statements'] = \
            len(readings_with_no_stmts)

        self._populate_hist_data(readings_with_stmts, readings_with_no_stmts)
        self._make_histograms()
        self._make_text_summary()
        ends['stats'] = datetime.now()

        fname = self.make_report()
        with open(fname, 'rb') as f:
            self.s3.put_object(Key=self.s3_prefix + fname, Body=f.read(),
                               Bucket=self.bucket_name)

        self._make_timing_report(starts, ends)
        self._stash_data()
        return
<commit_msg>Add more doc strings to DbAwsReporter.<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str

import pickle
import numpy as np
import matplotlib as mpl
mpl.use('Agg')
from matplotlib import pyplot as plt
from datetime import datetime

from indra.util.get_version import get_git_info
from indra.tools.reading.util.reporter import Reporter

class DbAwsStatReporter(Reporter):
    """A class to handle generating the reports made at the end of a job.

    Parameters
    ----------
    job_name : str
        The name of the job about which this report is being produced.
    s3_log_prefix: str
        The desired prefix location in which the information about the job is
        stored on s3.
    s3 : boto3.client('s3') instance
        An s3 client from boto3.
    bucket_name : str
        The name of the root s3 bucket.
    """
    def __init__(self, job_name, s3_log_prefix, s3, bucket_name):
        super(DbAwsStatReporter, self).__init__('%s_summary' % job_name)
        self.s3_prefix = s3_log_prefix + 'statistics/'
        self.bucket_name = bucket_name
        self.s3 = s3
        self.summary_dict = {}
        self.hist_dict = {}
        self.set_title("Report of Database Reading Batch Job")
        self.sections = {'Summary Statistics': [], 'Plots': [], 'Git Info': [],
                         'Job Info': []}
        self._make_job_line('Job Name', job_name)
        self._make_job_line('Job s3 prefix', s3_log_prefix)
        self._get_git_info()
        self.set_section_order(['Job Info', 'Git Info', 'Summary Statistics',
                                'Plots'])
        return

    def _get_git_info(self):
        """Stash the git info in a text file."""
        git_info_dict = get_git_info()
        text_file_content = ''
        for key, val in git_info_dict.items():
            label = key.replace('_', ' ').capitalize()
            text_file_content += '%s: %s\n' % (label, val)
            self.add_story_text('%s: %s' % (label, val), section='Git Info')
        self.s3.put_object(Key=self.s3_prefix + 'git_info.txt',
                           Body=text_file_content, Bucket=self.bucket_name)
        return

    def _plot_hist(self, agged, agg_over, data):
        """Make and stash a figure for histogram-like data."""
        fig = plt.figure()
        plt.hist(data, bins=np.arange(len(data)))
        plt.xlabel('Number of %s for %s' % (agged, agg_over))
        plt.ylabel('Number of %s with a given number of %s' % (agg_over, agged))
        fname = '%s_per_%s.png' % (agged, agg_over)
        fig.set_size_inches(6, 4)
        fig.savefig(fname)
        with open(fname, 'rb') as f:
            s3_key = self.s3_prefix + fname
            self.s3.put_object(Key=s3_key, Body=f.read(),
                               Bucket=self.bucket_name)
        self.add_story_image(fname, width=6, height=4, section='Plots')
        return

    def _make_timing_report(self, starts, ends):
        """Stash a text file with the timings: start, end, and duration."""
        # Report on the timing
        timing_str = ''
        for step in ['reading', 'statement production', 'stats']:
            time_taken = ends[step] - starts[step]
            timing_str += ('%22s: start: %s, end: %s, duration: %s\n'
                           % (step, starts[step], ends[step], time_taken))

        self.s3.put_object(Key=self.s3_prefix + 'timing.txt', Body=timing_str,
                           Bucket=self.bucket_name)
        return

    def _stash_data(self):
        """Store the data in pickle files. This should be done last."""
        self.s3.put_object(Key=self.s3_prefix + 'hist_data.pkl',
                           Body=pickle.dumps(self.hist_dict),
                           Bucket=self.bucket_name)
        self.s3.put_object(Key=self.s3_prefix + 'sum_data.pkl',
                           Bucket=self.bucket_name,
                           Body=pickle.dumps(self.summary_dict))
        return

    def _populate_hist_data(self, readings_with_stmts, readings_with_no_stmts):
        """Aggregate data from the various readings into arrays."""
        # Do a bunch of aggregation
        tc_rd_dict = {}
        tc_stmt_dict = {}
        rd_stmt_dict = {}
        reader_stmts = {}
        reader_tcids = {}
        reader_rids = {}
        for rid, tcid, reader, stmts in readings_with_stmts:
            # Handle things keyed by tcid
            if tcid not in tc_rd_dict.keys():
                tc_rd_dict[tcid] = {rid}
                tc_stmt_dict[tcid] = set(stmts)
            else:
                tc_rd_dict[tcid].add(rid)
                tc_stmt_dict[tcid] |= set(stmts)

            # Handle things keyed by rid
            if rid not in rd_stmt_dict.keys():
                rd_stmt_dict[rid] = set(stmts)
            else:
                rd_stmt_dict[rid] |= set(stmts)  # this shouldn't really happen.

            # Handle things keyed by reader
            if reader not in reader_stmts.keys():
                reader_stmts[reader] = set(stmts)
                reader_tcids[reader] = {tcid}
                reader_rids[reader] = {rid}
            else:
                reader_stmts[reader] |= set(stmts)
                reader_tcids[reader].add(tcid)
                reader_rids[reader].add(rid)

        for rid, tcid, reader, _ in readings_with_no_stmts:
            # Handle things keyed by tcid
            if tcid not in tc_rd_dict.keys():
                tc_rd_dict[tcid] = {rid}
            else:
                tc_rd_dict[tcid].add(rid)

            # Handle things keyed by reader
            if reader not in reader_stmts.keys():
                reader_tcids[reader] = {tcid}
                reader_rids[reader] = {rid}
            else:
                reader_tcids[reader].add(tcid)
                reader_rids[reader].add(rid)

        # Produce some numpy count arrays.
        self.hist_dict[('readings', 'text content')] = \
            np.array([len(rid_set) for rid_set in tc_rd_dict.values()])
        self.hist_dict[('stmts', 'text content')] = \
            np.array([len(stmts) for stmts in tc_stmt_dict.values()])
        self.hist_dict[('stmts', 'readings')] = \
            np.array([len(stmts) for stmts in rd_stmt_dict.values()])
        self.hist_dict[('stmts', 'readers')] = \
            np.array([len(stmts) for stmts in reader_stmts.values()])
        self.hist_dict[('text content', 'reader')] = \
            np.array([len(tcid_set) for tcid_set in reader_tcids.values()])
        self.hist_dict[('readings', 'reader')] = \
            np.array([len(rid_set) for rid_set in reader_rids.values()])
        return

    def _make_histograms(self):
        """Plot all the histograms."""
        # Produce the histograms
        for (agged, agg_over), data in self.hist_dict.items():
            self._plot_hist(agged, agg_over, data)
            label = '%s per %s' % (agged, agg_over)
            stat_dict = {'mean': data.mean(), 'std': data.std(),
                         'median': np.median(data)}
            self.add_story_text(str(stat_dict), style='Code', section='Plots')
            self.summary_dict[label.capitalize()] = {'mean': data.mean(),
                                                     'std': data.std(),
                                                     'median': np.median(data)}
        return

    def _make_text_summary(self):
        """Stash a text file summary of totals."""
        text_report_str = ''
        top_labels = ['Total readings', 'Content processed',
                      'Statements produced']
        for label in top_labels:
            text_str = '%s: %d\n' % (label, self.summary_dict[label])
            self.add_story_text(text_str, section='Summary Statistics')
            text_report_str += '%s: %d\n' % (label, self.summary_dict[label])

        for label, data in self.summary_dict.items():
            if label in top_labels:
                continue
            if isinstance(data, dict):
                text_report_str += '%s:\n' % label
                text_report_str += '\n'.join(['\t%s: %d' % (k, v)
                                              for k, v in data.items()])
                text_report_str += '\n'
            else:
                text_str = '%s: %d\n' % (label, data)
                self.add_story_text(text_str, section='Summary Statistics')
                text_report_str += text_str
        self.s3.put_object(Key=self.s3_prefix + 'summary.txt',
                           Body=text_report_str, Bucket=self.bucket_name)
        return

    def _make_job_line(self, key, value):
        """For job info section, produce one line."""
        self.add_story_text(key, section='Job Info', space=(1, 6))
        self.add_story_text(value, section='Job Info', style='Code')

    def report_statistics(self, reading_outputs, stmt_outputs, starts, ends):
        """Grab low-hanging-statistics and produce a pdf report.

        All the data generated is also stashed for future use in text and pickle
        files alongside the pdf summary on s3.

        Parameters
        ----------
        reading_outputs : list [ReadingData]
            A list of the ReadingData results output by `produce_readings`.
        stmt_outputs : list [StatementData]
            A list of the StatementData results output by `produce_statements`.
        starts : dict {<stage> : <datetime.datetime instance>}
            A dict of the start times of different parts of the job
            (e.g. statement production).
        ends : dict {<stage> : <datetime.datetime instance>}
            A dict of the end times of different parts of the job (like starts).
        """
        starts['stats'] = datetime.now()
        for k, end in ends.items():
            self._make_job_line(k + ' start', str(starts[k]))
            self._make_job_line(k + ' end', str(end))
            self._make_job_line(k + ' duration', str(end-starts[k]))

        self.summary_dict['Total readings'] = len(reading_outputs)
        reading_stmts = [(rd.reading_id, rd.tcid, rd.reader, rd.get_statements())
                         for rd in reading_outputs]
        self.summary_dict['Content processed'] = \
            len({t[1] for t in reading_stmts})
        self.summary_dict['Statements produced'] = len(stmt_outputs)
        self.s3.put_object(Key=self.s3_prefix + 'raw_tuples.pkl',
                           Body=pickle.dumps([t[:-1] + (len(t[-1]),)
                                              for t in reading_stmts]),
                           Bucket=self.bucket_name)

        readings_with_stmts = []
        readings_with_no_stmts = []
        for t in reading_stmts:
            if t[-1]:
                readings_with_stmts.append(t)
            else:
                readings_with_no_stmts.append(t)

        self.summary_dict['Readings with no statements'] = \
            len(readings_with_no_stmts)

        self._populate_hist_data(readings_with_stmts, readings_with_no_stmts)
        self._make_histograms()
        self._make_text_summary()
        ends['stats'] = datetime.now()

        fname = self.make_report()
        with open(fname, 'rb') as f:
            self.s3.put_object(Key=self.s3_prefix + fname, Body=f.read(),
                               Bucket=self.bucket_name)

        self._make_timing_report(starts, ends)
        self._stash_data()
        return
<|endoftext|>


<commit_before>import collections
import datetime
import functools
import urllib.parse

from django.conf import settings
from django.core.urlresolvers import reverse, reverse_lazy
from django.db import models
from django.utils.timezone import make_naive
from django.utils.translation import ugettext, ugettext_lazy as _

from core.models import BigForeignKey, ConferenceRelated, EventInfo
from core.utils import format_html_lazy
from proposals.models import TalkProposal

DAY_1 = datetime.date(2016, 6, 3)
DAY_2 = datetime.date(2016, 6, 4)
DAY_3 = datetime.date(2016, 6, 5)
DAY_NAMES = collections.OrderedDict([
    (DAY_1, _('Day 1')),
    (DAY_2, _('Day 2')),
    (DAY_3, _('Day 3')),
])

class TimeManager(models.Manager):
    def get(self, value):
        """We only has one field, so let's make it available without keyword.
        """
        return super().get(value=value)

@functools.total_ordering
class Time(models.Model):

    value = models.DateTimeField(
        primary_key=True,
        verbose_name=_('value'),
    )

    objects = TimeManager()

    class Meta:
        verbose_name = _('time')
        verbose_name_plural = _('times')
        ordering = ['value']

    def __str__(self):
        return str(make_naive(self.value))

    def __lt__(self, other):
        if not isinstance(other, Time):
            return NotImplemented
        if (not isinstance(self.value, datetime.datetime) or
                not isinstance(other.value, datetime.datetime)):
            return NotImplemented
        return self.value < other.value

class Location:
    """All possible location combinations.

    The numbering prefix helps to order events by locations. We need this
    information when resolving events in the same time period.

    Rules:

    1. The R3 events are put first.
    2. Belt and partial belt events are next, in that order.
    3. Block events in R0-2 are next, in that order.
    """
    R3   = '1-r3'
    ALL  = '2-all'
    R012 = '3-r012'
    R0   = '4-r0'
    R1   = '5-r1'
    R2   = '6-r2'

    @classmethod
    def get_md_width(cls, value):
        return {
            '2-all': 4,
            '3-r012': 3,
            '4-r0': 1,
            '5-r1': 1,
            '6-r2': 1,
            '1-r3': 1,
        }[value]

class BaseEvent(ConferenceRelated):
    """Base interface for all events in the schedule.
    """
    LOCATION_CHOICES = [
        (Location.ALL,  _('All rooms')),
        (Location.R012, _('R0, R1, R2')),
        (Location.R0,   _('R0')),
        (Location.R1,   _('R1')),
        (Location.R2,   _('R2')),
        (Location.R3,   _('R3')),
    ]
    location = models.CharField(
        max_length=6,
        choices=LOCATION_CHOICES,
        blank=True,
        null=True,
        db_index=True,
        verbose_name=_('location'),
    )

    begin_time = models.ForeignKey(
        to=Time,
        blank=True,
        null=True,
        related_name='begined_%(class)s_set',
        verbose_name=_('begin time'),
    )

    end_time = models.ForeignKey(
        to=Time,
        blank=True,
        null=True,
        related_name='ended_%(class)s_set',
        verbose_name=_('end time'),
    )

    class Meta:
        abstract = True

class CustomEvent(BaseEvent):

    title = models.CharField(
        max_length=140,
        verbose_name=_('title'),
    )

    class Meta:
        verbose_name = _('custom event')
        verbose_name_plural = _('custom events')

    def __str__(self):
        return self.title

class KeynoteEvent(BaseEvent):

    speaker_name = models.CharField(
        verbose_name=_('speaker name'),
        max_length=100,
    )
    slug = models.SlugField(
        verbose_name=_('slug'),
        help_text=format_html_lazy(
            _("This is used to link to the speaker's introduction on the "
              "Keynote page, e.g. 'liang2' will link to "
              "'{link}#keynote-speaker-liang2'."),
            link=reverse_lazy('page', kwargs={'path': 'events/keynotes'}),
        )
    )

    class Meta:
        verbose_name = _('keynote event')
        verbose_name_plural = _('keynote events')

    def __str__(self):
        return ugettext('Keynote: {speaker}'.format(
            speaker=self.speaker_name,
        ))

    def get_absolute_url(self):
        url = reverse('page', kwargs={'path': 'events/keynotes'})
        split = urllib.parse.urlsplit(url)
        frag = 'keynote-speaker-{slug}'.format(slug=self.slug)
        return urllib.parse.urlunsplit(split._replace(fragment=frag))

class SponsoredEvent(EventInfo, BaseEvent):

    host = BigForeignKey(
        to=settings.AUTH_USER_MODEL,
        verbose_name=_('host'),
    )
    slug = models.SlugField(
        allow_unicode=True,
        verbose_name=_('slug'),
    )

    class Meta:
        verbose_name = _('sponsored event')
        verbose_name_plural = _('sponsored events')

    def get_absolute_url(self):
        return reverse('events_sponsored_event_detail', kwargs={
            'slug': self.slug,
        })

class ProposedTalkEventManager(models.Manager):
    def get_queryset(self):
        """We almost always need the proposal info, so let's always JOIN it.
        """
        return super().get_queryset().select_related('proposal')

class ProposedTalkEvent(BaseEvent):

    proposal = BigForeignKey(
        to=TalkProposal,
        limit_choices_to={'accepted': True},
        verbose_name=_('proposal'),
    )

    objects = ProposedTalkEventManager()

    class Meta:
        verbose_name = _('talk event')
        verbose_name_plural = _('talk events')

    def __str__(self):
        return self.proposal.title

    def get_absolute_url(self):
        return reverse('events_talk_detail', kwargs={'pk': self.proposal.pk})

class Schedule(models.Model):

    html = models.TextField(
        verbose_name=_('HTML'),
    )
    created_at = models.DateTimeField(
        verbose_name=_('created at'),
        auto_now_add=True,
    )

    class Meta:
        verbose_name = _('Schedule')
        verbose_name_plural = _('Schedules')
        ordering = ['-created_at']
        get_latest_by = 'created_at'

    def __str__(self):
        return ugettext('Schedule created at {}').format(self.created_at)
<commit_msg>Fix filtering for ProposedTalkEvent<commit_after>import collections
import datetime
import functools
import urllib.parse

from django.conf import settings
from django.core.urlresolvers import reverse, reverse_lazy
from django.db import models
from django.utils.timezone import make_naive
from django.utils.translation import ugettext, ugettext_lazy as _

from core.models import (
    BigForeignKey, EventInfo,
    ConferenceRelated, DefaultConferenceManagerMixin,
)
from core.utils import format_html_lazy
from proposals.models import TalkProposal

DAY_1 = datetime.date(2016, 6, 3)
DAY_2 = datetime.date(2016, 6, 4)
DAY_3 = datetime.date(2016, 6, 5)
DAY_NAMES = collections.OrderedDict([
    (DAY_1, _('Day 1')),
    (DAY_2, _('Day 2')),
    (DAY_3, _('Day 3')),
])

class TimeManager(models.Manager):
    def get(self, value):
        """We only has one field, so let's make it available without keyword.
        """
        return super().get(value=value)

@functools.total_ordering
class Time(models.Model):

    value = models.DateTimeField(
        primary_key=True,
        verbose_name=_('value'),
    )

    objects = TimeManager()

    class Meta:
        verbose_name = _('time')
        verbose_name_plural = _('times')
        ordering = ['value']

    def __str__(self):
        return str(make_naive(self.value))

    def __lt__(self, other):
        if not isinstance(other, Time):
            return NotImplemented
        if (not isinstance(self.value, datetime.datetime) or
                not isinstance(other.value, datetime.datetime)):
            return NotImplemented
        return self.value < other.value

class Location:
    """All possible location combinations.

    The numbering prefix helps to order events by locations. We need this
    information when resolving events in the same time period.

    Rules:

    1. The R3 events are put first.
    2. Belt and partial belt events are next, in that order.
    3. Block events in R0-2 are next, in that order.
    """
    R3   = '1-r3'
    ALL  = '2-all'
    R012 = '3-r012'
    R0   = '4-r0'
    R1   = '5-r1'
    R2   = '6-r2'

    @classmethod
    def get_md_width(cls, value):
        return {
            '2-all': 4,
            '3-r012': 3,
            '4-r0': 1,
            '5-r1': 1,
            '6-r2': 1,
            '1-r3': 1,
        }[value]

class BaseEvent(ConferenceRelated):
    """Base interface for all events in the schedule.
    """
    LOCATION_CHOICES = [
        (Location.ALL,  _('All rooms')),
        (Location.R012, _('R0, R1, R2')),
        (Location.R0,   _('R0')),
        (Location.R1,   _('R1')),
        (Location.R2,   _('R2')),
        (Location.R3,   _('R3')),
    ]
    location = models.CharField(
        max_length=6,
        choices=LOCATION_CHOICES,
        blank=True,
        null=True,
        db_index=True,
        verbose_name=_('location'),
    )

    begin_time = models.ForeignKey(
        to=Time,
        blank=True,
        null=True,
        related_name='begined_%(class)s_set',
        verbose_name=_('begin time'),
    )

    end_time = models.ForeignKey(
        to=Time,
        blank=True,
        null=True,
        related_name='ended_%(class)s_set',
        verbose_name=_('end time'),
    )

    class Meta:
        abstract = True

class CustomEvent(BaseEvent):

    title = models.CharField(
        max_length=140,
        verbose_name=_('title'),
    )

    class Meta:
        verbose_name = _('custom event')
        verbose_name_plural = _('custom events')

    def __str__(self):
        return self.title

class KeynoteEvent(BaseEvent):

    speaker_name = models.CharField(
        verbose_name=_('speaker name'),
        max_length=100,
    )
    slug = models.SlugField(
        verbose_name=_('slug'),
        help_text=format_html_lazy(
            _("This is used to link to the speaker's introduction on the "
              "Keynote page, e.g. 'liang2' will link to "
              "'{link}#keynote-speaker-liang2'."),
            link=reverse_lazy('page', kwargs={'path': 'events/keynotes'}),
        )
    )

    class Meta:
        verbose_name = _('keynote event')
        verbose_name_plural = _('keynote events')

    def __str__(self):
        return ugettext('Keynote: {speaker}'.format(
            speaker=self.speaker_name,
        ))

    def get_absolute_url(self):
        url = reverse('page', kwargs={'path': 'events/keynotes'})
        split = urllib.parse.urlsplit(url)
        frag = 'keynote-speaker-{slug}'.format(slug=self.slug)
        return urllib.parse.urlunsplit(split._replace(fragment=frag))

class SponsoredEvent(EventInfo, BaseEvent):

    host = BigForeignKey(
        to=settings.AUTH_USER_MODEL,
        verbose_name=_('host'),
    )
    slug = models.SlugField(
        allow_unicode=True,
        verbose_name=_('slug'),
    )

    class Meta:
        verbose_name = _('sponsored event')
        verbose_name_plural = _('sponsored events')

    def get_absolute_url(self):
        return reverse('events_sponsored_event_detail', kwargs={
            'slug': self.slug,
        })

class ProposedTalkEventManager(DefaultConferenceManagerMixin, models.Manager):

    conference_attr = 'proposal__conference'

    def get_queryset(self):
        """We almost always need the proposal info, so let's always JOIN it.
        """
        return super().get_queryset().select_related('proposal')

class ProposedTalkEvent(BaseEvent):

    proposal = BigForeignKey(
        to=TalkProposal,
        limit_choices_to={'accepted': True},
        verbose_name=_('proposal'),
    )

    objects = ProposedTalkEventManager()

    class Meta:
        verbose_name = _('talk event')
        verbose_name_plural = _('talk events')

    def __str__(self):
        return self.proposal.title

    def get_absolute_url(self):
        return reverse('events_talk_detail', kwargs={'pk': self.proposal.pk})

class Schedule(models.Model):

    html = models.TextField(
        verbose_name=_('HTML'),
    )
    created_at = models.DateTimeField(
        verbose_name=_('created at'),
        auto_now_add=True,
    )

    class Meta:
        verbose_name = _('Schedule')
        verbose_name_plural = _('Schedules')
        ordering = ['-created_at']
        get_latest_by = 'created_at'

    def __str__(self):
        return ugettext('Schedule created at {}').format(self.created_at)
<|endoftext|>


<commit_before>"""
Test DataTable interface.
"""
import os, unittest
import opensim as osim

class TestDataTable(unittest.TestCase):
    def test_DataTable(self):
        table = osim.DataTable()
        # Set column labels.
        table.setColumnLabels(['0', '1', '2', '3'])
        assert table.getColumnLabels() == ('0', '1', '2', '3')
        assert table.hasColumn('0')
        assert table.hasColumn('2')
        assert not table.hasColumn('not-found')
        table.setColumnLabel(0, 'zero')
        table.setColumnLabel(2, 'two')
        assert table.getColumnLabel(0) == 'zero'
        assert table.getColumnLabel(2) == 'two'
        assert table.getColumnIndex('zero') == 0
        assert table.getColumnIndex('two')  == 2
        # Append a row to the table.
        row = osim.RowVector([1, 2, 3, 4])
        table.appendRow(0.1, row)
        assert table.getNumRows() == 1
        assert table.getNumColumns() == 4
        row0 = table.getRowAtIndex(0)
        assert (row0[0] == row[0] and
                row0[1] == row[1] and
                row0[2] == row[2] and
                row0[3] == row[3])
        # Append another row to the table.
        row[0] *= 2
        row[1] *= 2
        row[2] *= 2
        row[3] *= 2
        table.appendRow(0.2, row)
        assert table.getNumRows() == 2
        assert table.getNumColumns() == 4
        row1 = table.getRow(0.2)
        assert (row1[0] == row[0] and
                row1[1] == row[1] and
                row1[2] == row[2] and
                row1[3] == row[3])
        # Append another row to the table.
        row[0] *= 2
        row[1] *= 2
        row[2] *= 2
        row[3] *= 2
        table.appendRow(0.3, row)
        assert table.getNumRows() == 3
        assert table.getNumColumns() == 4
        row2 = table.getRow(0.3)
        assert (row2[0] == row[0] and
                row2[1] == row[1] and
                row2[2] == row[2] and
                row2[3] == row[3])
        # Retrieve independent column.
        assert table.getIndependentColumn() == (0.1, 0.2, 0.3)
        # Retrieve dependent columns.
        col1 = table.getDependentColumnAtIndex(1)
        assert (col1[0] == 2 and
                col1[1] == 4 and
                col1[2] == 8)
        col3 = table.getDependentColumn('3')
        assert (col3[0] == 4 and
                col3[1] == 8 and
                col3[2] == 16)
        assert table.hasColumn(0)
        assert table.hasColumn(2)
        # Edit rows of the table.
        row0 = table.getRowAtIndex(0)
        row0[0] = 10
        row0[1] = 10
        row0[2] = 10
        row0[3] = 10
        row0 = table.getRowAtIndex(0)
        assert (row0[0] == 10 and
                row0[1] == 10 and
                row0[2] == 10 and
                row0[3] == 10)
        row2 = table.getRow(0.3)
        row2[0] = 20
        row2[1] = 20
        row2[2] = 20
        row2[3] = 20
        row2 = table.getRow(0.3)
        assert (row2[0] == 20 and
                row2[1] == 20 and
                row2[2] == 20 and
                row2[3] == 20)
        # Edit columns of the table.
        col1 = table.getDependentColumnAtIndex(1)
        col1[0] = 30
        col1[1] = 30
        col1[2] = 30
        col1 = table.getDependentColumnAtIndex(1)
        assert (col1[0] == 30 and
                col1[1] == 30 and
                col1[2] == 30)
        col3 = table.getDependentColumn('3')
        col3[0] = 40
        col3[1] = 40
        col3[2] = 40
        col3 = table.getDependentColumn('3')
        assert (col3[0] == 40 and
                col3[1] == 40 and
                col3[2] == 40)
        # Access eleemnt with index out of bounds. Exception expected.
        try:
            shouldThrow = row0[5]
            assert false
        except RuntimeError:
            pass
        try:
            shouldThrow = col1[5]
            assert false
        except RuntimeError:
            pass
        # Access row with index out of bounds. Exception expected.
        try:
            shouldThrow = table.getRowAtIndex(5)
            assert false
        except RuntimeError:
            pass
        # Access row with timestamp that does not exist. Exception expected.
        try:
            shouldThrow = table.getRow(5.5)
            assert false
        except RuntimeError:
            pass
        # Access column with index out of bounds. Exception expected.
        try:
            shouldThrow = table.getDependentColumnAtIndex(5)
            assert false
        except RuntimeError:
            pass
        # Access column with label that does not exist. Exception expected.
        try:
            shouldThrow = table.getDependentColumn('not-found')
            assert false
        except RuntimeError:
            pass

    def test_TimeSeriesTable(self):
        table = osim.TimeSeriesTable()
        table.setColumnLabels(('col1', 'col2', 'col3', 'col4'))
        assert(table.getColumnLabels() == ('col1', 'col2', 'col3', 'col4'))
        # Append a row to the table.
        row = osim.RowVector([1, 2, 3, 4])
        table.appendRow(0.1, row)
        assert table.getNumRows() == 1
        assert table.getNumColumns() == 4
        row0 = table.getRowAtIndex(0)
        assert (row0[0] == row[0] and
                row0[1] == row[1] and
                row0[2] == row[2] and
                row0[3] == row[3])
        # Append another row to the table.
        row[0] *= 2
        row[1] *= 2
        row[2] *= 2
        row[3] *= 2
        table.appendRow(0.2, row)
        assert table.getNumRows() == 2
        assert table.getNumColumns() == 4
        row1 = table.getRow(0.2)
        assert (row1[0] == row[0] and
                row1[1] == row[1] and
                row1[2] == row[2] and
                row1[3] == row[3])
        # Append another row to the table with a timestamp
        # less than the previous one. Exception expected.
        try:
            table.appendRow(0.15, row)
            assert False
        except RuntimeError:
            pass

    def test_DataTableVec3(self):
        table = osim.DataTableVec3()
        # Set columns labels.
        table.setColumnLabels(['0', '1', '2'])
        assert table.getColumnLabels() == ('0', '1', '2')
        # Append a row to the table.
        row = osim.RowVectorOfVec3([osim.Vec3(1, 2, 3),
                                    osim.Vec3(4, 5, 6),
                                    osim.Vec3(7, 8, 9)])
        table.appendRow(0.1, row)
        assert table.getNumRows() == 1
        assert table.getNumColumns() == 3
        row0 = table.getRowAtIndex(0)
        assert (str(row0[0]) == str(row[0]) and
                str(row0[1]) == str(row[1]) and
                str(row0[2]) == str(row[2]))
        # Append another row to the table.
        row = osim.RowVectorOfVec3([osim.Vec3( 2,  4,  6),
                                    osim.Vec3( 8, 10, 12),
                                    osim.Vec3(14, 16, 18)])
        table.appendRow(0.2, row)
        assert table.getNumRows() == 2
        assert table.getNumColumns() == 3
        row1 = table.getRow(0.2)
        assert (str(row1[0]) == str(row[0]) and
                str(row1[1]) == str(row[1]) and
                str(row1[2]) == str(row[2]))
        # Append another row to the table.
        row = osim.RowVectorOfVec3([osim.Vec3( 4,  8, 12),
                                    osim.Vec3(16, 20, 24),
                                    osim.Vec3(28, 32, 36)])
        table.appendRow(0.3, row)
        assert table.getNumRows() == 3
        assert table.getNumColumns() == 3
        row2 = table.getRow(0.3)
        assert (str(row2[0]) == str(row[0]) and
                str(row2[1]) == str(row[1]) and
                str(row2[2]) == str(row[2]))
        # Retrieve independent column.
        assert table.getIndependentColumn() == (0.1, 0.2, 0.3)
        # Retrieve dependent columns.
        col1 = table.getDependentColumnAtIndex(1)
        assert (str(col1[0]) == str(osim.Vec3( 4,  5,  6)) and
                str(col1[1]) == str(osim.Vec3( 8, 10, 12)) and
                str(col1[2]) == str(osim.Vec3(16, 20, 24)))
        col2 = table.getDependentColumn('2')
        assert (str(col2[0]) == str(osim.Vec3( 7,  8,  9)) and
                str(col2[1]) == str(osim.Vec3(14, 16, 18)) and
                str(col2[2]) == str(osim.Vec3(28, 32, 36)))
        # Edit rows of the table.
        row0 = table.getRowAtIndex(0)
        row0[0] = osim.Vec3(10, 10, 10)
        row0[1] = osim.Vec3(10, 10, 10)
        row0[2] = osim.Vec3(10, 10, 10)
        row0 = table.getRowAtIndex(0)
        assert (str(row0[0]) == str(osim.Vec3(10, 10, 10)) and
                str(row0[1]) == str(osim.Vec3(10, 10, 10)) and
                str(row0[2]) == str(osim.Vec3(10, 10, 10)))
        row2 = table.getRow(0.3)
        row2[0] = osim.Vec3(20, 20, 20)
        row2[1] = osim.Vec3(20, 20, 20)
        row2[2] = osim.Vec3(20, 20, 20)
        row2 = table.getRow(0.3)
        assert (str(row2[0]) == str(osim.Vec3(20, 20, 20)) and
                str(row2[1]) == str(osim.Vec3(20, 20, 20)) and
                str(row2[2]) == str(osim.Vec3(20, 20, 20)))
        # Edit columns of the table.
        col1 = table.getDependentColumnAtIndex(1)
        col1[0] = osim.Vec3(30, 30, 30)
        col1[1] = osim.Vec3(30, 30, 30)
        col1[2] = osim.Vec3(30, 30, 30)
        col1 = table.getDependentColumnAtIndex(1)
        assert (str(col1[0]) == str(osim.Vec3(30, 30, 30)) and
                str(col1[1]) == str(osim.Vec3(30, 30, 30)) and
                str(col1[2]) == str(osim.Vec3(30, 30, 30)))
        col2 = table.getDependentColumn('2')
        col2[0] = osim.Vec3(40, 40, 40)
        col2[1] = osim.Vec3(40, 40, 40)
        col2[2] = osim.Vec3(40, 40, 40)
        col2 = table.getDependentColumn('2')
        assert (str(col2[0]) == str(osim.Vec3(40, 40, 40)) and
                str(col2[1]) == str(osim.Vec3(40, 40, 40)) and
                str(col2[2]) == str(osim.Vec3(40, 40, 40)))

        tableDouble = table.flatten()
        assert tableDouble.getNumRows() == 3
        assert tableDouble.getNumColumns() == 9
        assert len(tableDouble.getColumnLabels()) == 9
        assert tableDouble.getRowAtIndex(0)[0] == 1
        assert tableDouble.getRowAtIndex(1)[0] == 2
        assert tableDouble.getRowAtIndex(2)[0] == 4
        assert tableDouble.getRowAtIndex(0)[8] == 9
        assert tableDouble.getRowAtIndex(1)[8] == 18
        assert tableDouble.getRowAtIndex(2)[8] == 36

        tableDouble = table.flatten(['_x', '_y', '_z'])
        assert tableDouble.getColumnLabels() == ('0_x', '0_y', '0_z',
                                                 '1_x', '1_y', '1_z',
                                                 '2_x', '2_y', '2_z')

    def test_TimeSeriesTableVec3(self):
        table = osim.TimeSeriesTableVec3()
        # Set columns labels.
        table.setColumnLabels(['0', '1', '2'])
        assert table.getColumnLabels() == ('0', '1', '2')
        # Append a row to the table.
        row = osim.RowVectorOfVec3([osim.Vec3(1, 2, 3),
                                    osim.Vec3(4, 5, 6),
                                    osim.Vec3(7, 8, 9)])
        table.appendRow(0.1, row)
        assert table.getNumRows() == 1
        assert table.getNumColumns() == 3
        row0 = table.getRowAtIndex(0)
        assert (str(row0[0]) == str(row[0]) and
                str(row0[1]) == str(row[1]) and
                str(row0[2]) == str(row[2]))
        # Append another row to the table.
        row = osim.RowVectorOfVec3([osim.Vec3( 2,  4,  6),
                                    osim.Vec3( 8, 10, 12),
                                    osim.Vec3(14, 16, 18)])
        table.appendRow(0.2, row)
        assert table.getNumRows() == 2
        assert table.getNumColumns() == 3
        row1 = table.getRow(0.2)
        assert (str(row1[0]) == str(row[0]) and
                str(row1[1]) == str(row[1]) and
                str(row1[2]) == str(row[2]))
        # Append another row to the table with a timestamp
        # less than the previous one. Exception expected.
        try:
            table.appendRow(0.15, row)
            assert False
        except RuntimeError:
            pass

        tableDouble = table.flatten()
        assert tableDouble.getNumRows() == 2
        assert tableDouble.getNumColumns() == 9
        assert len(tableDouble.getColumnLabels()) == 9
        assert tableDouble.getRowAtIndex(0)[0] == 1
        assert tableDouble.getRowAtIndex(1)[0] == 2
        assert tableDouble.getRowAtIndex(0)[8] == 9
        assert tableDouble.getRowAtIndex(1)[8] == 18

        tableDouble = table.flatten(['_x', '_y', '_z'])
        assert tableDouble.getColumnLabels() == ('0_x', '0_y', '0_z',
                                                 '1_x', '1_y', '1_z',
                                                 '2_x', '2_y', '2_z')

<commit_msg>Fix the test. Previous merge broke it by rearranging code.<commit_after>"""
Test DataTable interface.
"""
import os, unittest
import opensim as osim

class TestDataTable(unittest.TestCase):
    def test_DataTable(self):
        table = osim.DataTable()
        # Set column labels.
        table.setColumnLabels(['0', '1', '2', '3'])
        assert table.getColumnLabels() == ('0', '1', '2', '3')
        assert table.hasColumn('0')
        assert table.hasColumn('2')
        assert not table.hasColumn('not-found')
        table.setColumnLabel(0, 'zero')
        table.setColumnLabel(2, 'two')
        assert table.getColumnLabel(0) == 'zero'
        assert table.getColumnLabel(2) == 'two'
        assert table.getColumnIndex('zero') == 0
        assert table.getColumnIndex('two')  == 2
        # Append a row to the table.
        row = osim.RowVector([1, 2, 3, 4])
        table.appendRow(0.1, row)
        assert table.getNumRows() == 1
        assert table.getNumColumns() == 4
        row0 = table.getRowAtIndex(0)
        assert (row0[0] == row[0] and
                row0[1] == row[1] and
                row0[2] == row[2] and
                row0[3] == row[3])
        # Append another row to the table.
        row[0] *= 2
        row[1] *= 2
        row[2] *= 2
        row[3] *= 2
        table.appendRow(0.2, row)
        assert table.getNumRows() == 2
        assert table.getNumColumns() == 4
        row1 = table.getRow(0.2)
        assert (row1[0] == row[0] and
                row1[1] == row[1] and
                row1[2] == row[2] and
                row1[3] == row[3])
        # Append another row to the table.
        row[0] *= 2
        row[1] *= 2
        row[2] *= 2
        row[3] *= 2
        table.appendRow(0.3, row)
        assert table.getNumRows() == 3
        assert table.getNumColumns() == 4
        row2 = table.getRow(0.3)
        assert (row2[0] == row[0] and
                row2[1] == row[1] and
                row2[2] == row[2] and
                row2[3] == row[3])
        # Retrieve independent column.
        assert table.getIndependentColumn() == (0.1, 0.2, 0.3)
        # Retrieve dependent columns.
        col1 = table.getDependentColumnAtIndex(1)
        assert (col1[0] == 2 and
                col1[1] == 4 and
                col1[2] == 8)
        col3 = table.getDependentColumn('3')
        assert (col3[0] == 4 and
                col3[1] == 8 and
                col3[2] == 16)
        assert table.hasColumn(0)
        assert table.hasColumn(2)
        # Edit rows of the table.
        row0 = table.getRowAtIndex(0)
        row0[0] = 10
        row0[1] = 10
        row0[2] = 10
        row0[3] = 10
        row0 = table.getRowAtIndex(0)
        assert (row0[0] == 10 and
                row0[1] == 10 and
                row0[2] == 10 and
                row0[3] == 10)
        row2 = table.getRow(0.3)
        row2[0] = 20
        row2[1] = 20
        row2[2] = 20
        row2[3] = 20
        row2 = table.getRow(0.3)
        assert (row2[0] == 20 and
                row2[1] == 20 and
                row2[2] == 20 and
                row2[3] == 20)
        # Edit columns of the table.
        col1 = table.getDependentColumnAtIndex(1)
        col1[0] = 30
        col1[1] = 30
        col1[2] = 30
        col1 = table.getDependentColumnAtIndex(1)
        assert (col1[0] == 30 and
                col1[1] == 30 and
                col1[2] == 30)
        col3 = table.getDependentColumn('3')
        col3[0] = 40
        col3[1] = 40
        col3[2] = 40
        col3 = table.getDependentColumn('3')
        assert (col3[0] == 40 and
                col3[1] == 40 and
                col3[2] == 40)
        # Access eleemnt with index out of bounds. Exception expected.
        try:
            shouldThrow = row0[5]
            assert false
        except RuntimeError:
            pass
        try:
            shouldThrow = col1[5]
            assert false
        except RuntimeError:
            pass
        # Access row with index out of bounds. Exception expected.
        try:
            shouldThrow = table.getRowAtIndex(5)
            assert false
        except RuntimeError:
            pass
        # Access row with timestamp that does not exist. Exception expected.
        try:
            shouldThrow = table.getRow(5.5)
            assert false
        except RuntimeError:
            pass
        # Access column with index out of bounds. Exception expected.
        try:
            shouldThrow = table.getDependentColumnAtIndex(5)
            assert false
        except RuntimeError:
            pass
        # Access column with label that does not exist. Exception expected.
        try:
            shouldThrow = table.getDependentColumn('not-found')
            assert false
        except RuntimeError:
            pass

    def test_TimeSeriesTable(self):
        table = osim.TimeSeriesTable()
        table.setColumnLabels(('col1', 'col2', 'col3', 'col4'))
        assert(table.getColumnLabels() == ('col1', 'col2', 'col3', 'col4'))
        # Append a row to the table.
        row = osim.RowVector([1, 2, 3, 4])
        table.appendRow(0.1, row)
        assert table.getNumRows() == 1
        assert table.getNumColumns() == 4
        row0 = table.getRowAtIndex(0)
        assert (row0[0] == row[0] and
                row0[1] == row[1] and
                row0[2] == row[2] and
                row0[3] == row[3])
        # Append another row to the table.
        row[0] *= 2
        row[1] *= 2
        row[2] *= 2
        row[3] *= 2
        table.appendRow(0.2, row)
        assert table.getNumRows() == 2
        assert table.getNumColumns() == 4
        row1 = table.getRow(0.2)
        assert (row1[0] == row[0] and
                row1[1] == row[1] and
                row1[2] == row[2] and
                row1[3] == row[3])
        # Append another row to the table with a timestamp
        # less than the previous one. Exception expected.
        try:
            table.appendRow(0.15, row)
            assert False
        except RuntimeError:
            pass

    def test_DataTableVec3(self):
        table = osim.DataTableVec3()
        # Set columns labels.
        table.setColumnLabels(['0', '1', '2'])
        assert table.getColumnLabels() == ('0', '1', '2')
        # Append a row to the table.
        row = osim.RowVectorOfVec3([osim.Vec3(1, 2, 3),
                                    osim.Vec3(4, 5, 6),
                                    osim.Vec3(7, 8, 9)])
        table.appendRow(0.1, row)
        assert table.getNumRows() == 1
        assert table.getNumColumns() == 3
        row0 = table.getRowAtIndex(0)
        assert (str(row0[0]) == str(row[0]) and
                str(row0[1]) == str(row[1]) and
                str(row0[2]) == str(row[2]))
        # Append another row to the table.
        row = osim.RowVectorOfVec3([osim.Vec3( 2,  4,  6),
                                    osim.Vec3( 8, 10, 12),
                                    osim.Vec3(14, 16, 18)])
        table.appendRow(0.2, row)
        assert table.getNumRows() == 2
        assert table.getNumColumns() == 3
        row1 = table.getRow(0.2)
        assert (str(row1[0]) == str(row[0]) and
                str(row1[1]) == str(row[1]) and
                str(row1[2]) == str(row[2]))
        # Append another row to the table.
        row = osim.RowVectorOfVec3([osim.Vec3( 4,  8, 12),
                                    osim.Vec3(16, 20, 24),
                                    osim.Vec3(28, 32, 36)])
        table.appendRow(0.3, row)
        assert table.getNumRows() == 3
        assert table.getNumColumns() == 3
        row2 = table.getRow(0.3)
        assert (str(row2[0]) == str(row[0]) and
                str(row2[1]) == str(row[1]) and
                str(row2[2]) == str(row[2]))
        # Retrieve independent column.
        assert table.getIndependentColumn() == (0.1, 0.2, 0.3)
        # Retrieve dependent columns.
        col1 = table.getDependentColumnAtIndex(1)
        assert (str(col1[0]) == str(osim.Vec3( 4,  5,  6)) and
                str(col1[1]) == str(osim.Vec3( 8, 10, 12)) and
                str(col1[2]) == str(osim.Vec3(16, 20, 24)))
        col2 = table.getDependentColumn('2')
        assert (str(col2[0]) == str(osim.Vec3( 7,  8,  9)) and
                str(col2[1]) == str(osim.Vec3(14, 16, 18)) and
                str(col2[2]) == str(osim.Vec3(28, 32, 36)))
        # Flatten the table into table of doubles.
        tableDouble = table.flatten()
        assert tableDouble.getNumRows() == 3
        assert tableDouble.getNumColumns() == 9
        assert len(tableDouble.getColumnLabels()) == 9
        assert tableDouble.getRowAtIndex(0)[0] == 1
        assert tableDouble.getRowAtIndex(1)[0] == 2
        assert tableDouble.getRowAtIndex(2)[0] == 4
        assert tableDouble.getRowAtIndex(0)[8] == 9
        assert tableDouble.getRowAtIndex(1)[8] == 18
        assert tableDouble.getRowAtIndex(2)[8] == 36

        tableDouble = table.flatten(['_x', '_y', '_z'])
        assert tableDouble.getColumnLabels() == ('0_x', '0_y', '0_z',
                                                 '1_x', '1_y', '1_z',
                                                 '2_x', '2_y', '2_z')
        # Edit rows of the table.
        row0 = table.getRowAtIndex(0)
        row0[0] = osim.Vec3(10, 10, 10)
        row0[1] = osim.Vec3(10, 10, 10)
        row0[2] = osim.Vec3(10, 10, 10)
        row0 = table.getRowAtIndex(0)
        assert (str(row0[0]) == str(osim.Vec3(10, 10, 10)) and
                str(row0[1]) == str(osim.Vec3(10, 10, 10)) and
                str(row0[2]) == str(osim.Vec3(10, 10, 10)))
        row2 = table.getRow(0.3)
        row2[0] = osim.Vec3(20, 20, 20)
        row2[1] = osim.Vec3(20, 20, 20)
        row2[2] = osim.Vec3(20, 20, 20)
        row2 = table.getRow(0.3)
        assert (str(row2[0]) == str(osim.Vec3(20, 20, 20)) and
                str(row2[1]) == str(osim.Vec3(20, 20, 20)) and
                str(row2[2]) == str(osim.Vec3(20, 20, 20)))
        # Edit columns of the table.
        col1 = table.getDependentColumnAtIndex(1)
        col1[0] = osim.Vec3(30, 30, 30)
        col1[1] = osim.Vec3(30, 30, 30)
        col1[2] = osim.Vec3(30, 30, 30)
        col1 = table.getDependentColumnAtIndex(1)
        assert (str(col1[0]) == str(osim.Vec3(30, 30, 30)) and
                str(col1[1]) == str(osim.Vec3(30, 30, 30)) and
                str(col1[2]) == str(osim.Vec3(30, 30, 30)))
        col2 = table.getDependentColumn('2')
        col2[0] = osim.Vec3(40, 40, 40)
        col2[1] = osim.Vec3(40, 40, 40)
        col2[2] = osim.Vec3(40, 40, 40)
        col2 = table.getDependentColumn('2')
        assert (str(col2[0]) == str(osim.Vec3(40, 40, 40)) and
                str(col2[1]) == str(osim.Vec3(40, 40, 40)) and
                str(col2[2]) == str(osim.Vec3(40, 40, 40)))

    def test_TimeSeriesTableVec3(self):
        table = osim.TimeSeriesTableVec3()
        # Set columns labels.
        table.setColumnLabels(['0', '1', '2'])
        assert table.getColumnLabels() == ('0', '1', '2')
        # Append a row to the table.
        row = osim.RowVectorOfVec3([osim.Vec3(1, 2, 3),
                                    osim.Vec3(4, 5, 6),
                                    osim.Vec3(7, 8, 9)])
        table.appendRow(0.1, row)
        assert table.getNumRows() == 1
        assert table.getNumColumns() == 3
        row0 = table.getRowAtIndex(0)
        assert (str(row0[0]) == str(row[0]) and
                str(row0[1]) == str(row[1]) and
                str(row0[2]) == str(row[2]))
        # Append another row to the table.
        row = osim.RowVectorOfVec3([osim.Vec3( 2,  4,  6),
                                    osim.Vec3( 8, 10, 12),
                                    osim.Vec3(14, 16, 18)])
        table.appendRow(0.2, row)
        assert table.getNumRows() == 2
        assert table.getNumColumns() == 3
        row1 = table.getRow(0.2)
        assert (str(row1[0]) == str(row[0]) and
                str(row1[1]) == str(row[1]) and
                str(row1[2]) == str(row[2]))
        # Append another row to the table with a timestamp
        # less than the previous one. Exception expected.
        try:
            table.appendRow(0.15, row)
            assert False
        except RuntimeError:
            pass

        tableDouble = table.flatten()
        assert tableDouble.getNumRows() == 2
        assert tableDouble.getNumColumns() == 9
        assert len(tableDouble.getColumnLabels()) == 9
        assert tableDouble.getRowAtIndex(0)[0] == 1
        assert tableDouble.getRowAtIndex(1)[0] == 2
        assert tableDouble.getRowAtIndex(0)[8] == 9
        assert tableDouble.getRowAtIndex(1)[8] == 18

        tableDouble = table.flatten(['_x', '_y', '_z'])
        assert tableDouble.getColumnLabels() == ('0_x', '0_y', '0_z',
                                                 '1_x', '1_y', '1_z',
                                                 '2_x', '2_y', '2_z')

<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand
import random

from invoice import models

class Command(BaseCommand):
    help = 'Creates some sample data to use for testing'

    def handle(self, *args, **options):
        from django.conf import settings

        if not (settings.DEBUG or settings.STAGING):
            raise CommandError('You cannot run this command in production')

        random.seed('Some object to seed the random number generator')  # otherwise it is done by time, which could lead to inconsistant tests

        self.create_companies()
        self.create_invoices()

    def create_companies(self):
        company_names = ['Dodgey Dave\'s DJs', 'The Man In A Van', 'A Shitty Hotel']
        company_addresses = ['Somewhere in Birmingham', 'The Moon', 'My Mate\'s Garage']
        company_emails = ['test@test.com']

        for i in range(5):
            models.Company.objects.create(
                name=company_names[i%len(company_names)],
                address=company_addresses[i%len(company_addresses)],
                email=company_emails[i%len(company_emails)],
                invoice_prefix='TEST'
                )

    def create_invoices(self):
        invoice_items = ['Some lights and PA', 'Get-out in a muddy field', 'Boring show call', 'Corproate job in Manchester', 'Dealing in child actors', 'Literal waste of my time']
        amounts = [0.00, 100.00, 5.99, 600, 130.34]

        for i in range(10):  # Create 10 random invoices
            company = random.choice(models.Company.objects.all())

            invoice = models.Invoice.objects.create(
                company=company,
                invoice_number=str(company.invoice_prefix) + str(random.randint(1, 999))
            )

            for j in range(random.randint(1, 5)):  # Create a random number of items
                models.InvoiceItem.objects.create(
                    invoice=invoice,
                    description=random.choice(invoice_items),
                    amount=int(random.choice(amounts))
                )
<commit_msg>Custom management command<commit_after>from django.core.management.base import BaseCommand
import random

from invoice import models

class Command(BaseCommand):
    help = 'Creates some sample data to use for testing'

    def handle(self, *args, **options):
        from django.conf import settings

        if not (settings.DEBUG or settings.STAGING):
            raise CommandError('You cannot run this command in production')

        random.seed('Some object to seed the random number generator')  # otherwise it is done by time, which could lead to inconsistant tests

        self.create_companies()
        self.create_invoices()

    def create_companies(self):
        company_names = ['Dodgey Dave\'s DJs', 'The Man In A Van', 'A Shitty Hotel']
        company_addresses = ['Somewhere in Birmingham', 'The Moon', 'My Mate\'s Garage']
        company_emails = ['test@test.com']

        for i in range(5):
            models.Company.objects.create(
                name=company_names[i%len(company_names)],
                address=company_addresses[i%len(company_addresses)],
                email=company_emails[i%len(company_emails)]
                )

    def create_invoices(self):
        invoice_items = ['Some lights and PA', 'Get-out in a muddy field', 'Boring show call', 'Corproate job in Manchester', 'Dealing in child actors', 'Literal waste of my time']
        amounts = [0.00, 100.00, 5.99, 600, 130.34]

        for i in range(10):  # Create 10 random invoices
            company = random.choice(models.Company.objects.all())

            invoice = models.Invoice.objects.create(
                company=company,
            )

            for j in range(random.randint(1, 5)):  # Create a random number of items
                models.InvoiceItem.objects.create(
                    invoice=invoice,
                    description=random.choice(invoice_items),
                    amount=int(random.choice(amounts))
                )
<|endoftext|>


<commit_before>"""
awslimitchecker/versioncheck.py

The latest version of this package is available at:
<https://github.com/jantman/awslimitchecker>

################################################################################
Copyright 2015 Jason Antman <jason@jasonantman.com> <http://www.jasonantman.com>

    This file is part of awslimitchecker, also known as awslimitchecker.

    awslimitchecker is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    awslimitchecker is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with awslimitchecker.  If not, see <http://www.gnu.org/licenses/>.

The Copyright and Authors attributions contained herein may not be removed or
otherwise altered, except to add the Author attribution of a contributor to
this work. (Additional Terms pursuant to Section 7b of the AGPL v3)
################################################################################
While not legally required, I sincerely request that anyone who finds
bugs please submit them at <https://github.com/jantman/awslimitchecker> or
to me via email, and that you send any contributions or improvements
either as a pull request on GitHub, or to me via email.
################################################################################

AUTHORS:
Jason Antman <jason@jasonantman.com> <http://www.jasonantman.com>
################################################################################
"""

import os
import subprocess
import logging
import re
try:
    from subprocess import DEVNULL
except ImportError:
    DEVNULL = open(os.devnull, 'wb')
import locale
logger = logging.getLogger(__name__)

class AGPLVersionChecker(object):

    def find_package_version(self):
        """
        Find the installed version of the specified package, and as much
        information about it as possible (source URL, git ref or tag, etc.)

        This attempts, to the best of our ability, to find out if the package
        was installed from git, and if so, provide information on the origin
        of that git repository and status of the clone. Otherwise, it uses
        pip and pkg_resources to find the version and homepage of the installed
        distribution.

        This class is not a sure-fire method of identifying the source of
        the distribution or ensuring AGPL compliance; it simply helps with this
        process _iff_ a modified version is installed from an editable git URL
        _and_ all changes are pushed up to the publicly-visible origin.

        Returns a dict with keys 'version', 'tag', 'commit', and 'url'.
        Values are strings or None.

        :param package_name: name of the package to find information for
        :type package_name: str
        :returns: information about the installed version of the package
        :rtype: dict
        """
        res = {
            'version': None,
            'url': None,
            'tag': None,
            'commit': None
        }
        git_info = self._find_git_info()
        logger.debug("Git info: %s", git_info)
        for k, v in git_info.items():
            if v is not None:
                res[k] = v
        if res['dirty'] and res['tag'] is not None:
            res['tag'] += '*'
        if res['dirty'] and res['commit'] is not None:
            res['commit'] += '*'
        try:
            pip_info = self._find_pip_info()
        except Exception:
            # we NEVER want this to crash the program
            pip_info = {}
        logger.debug("pip info: %s", pip_info)
        if 'version' in pip_info:
            res['version'] = pip_info['version']
        if 'url' in pip_info and res['url'] is None:
            res['url'] = pip_info['url']
        try:
            pkg_info = self._find_pkg_info()
        except Exception:
            pkg_info = {}
        logger.debug("pkg_resources info: %s", pkg_info)
        if 'version' in pkg_info and res['version'] is None:
            res['version'] = pkg_info['version']
        if 'url' in pkg_info and res['url'] is None:
            res['url'] = pkg_info['url']
        logger.debug("Final package info: %s", res)
        return res

    def _find_pkg_info(self):
        """
        Find information about the installed awslimitchecker from pkg_resources.

        :return: dict of information from pkg_resources
        """
        import pkg_resources
        dist = pkg_resources.require('awslimitchecker')[0]
        ver, url = self._dist_version_url(dist)
        return {'version': ver, 'url': url}

    def _find_pip_info(self):
        """
        Try to find information about the installed awslimitchecker from pip.
        This should be wrapped in a try/except.

        :return: dict of information from pip
        """
        import pip
        res = {}
        dist = None
        for d in pip.get_installed_distributions():
            if d.project_name == 'awslimitchecker':
                dist = d
        if dist is None:
            return res
        ver, url = self._dist_version_url(dist)
        res['version'] = ver
        # this is a bit of an ugly, lazy hack...
        req = pip.FrozenRequirement.from_dist(dist, [])
        if ':' in req.req or '@' in req.req:
            res['url'] = req.req
        else:
            res['url'] = url
        return res

    def _dist_version_url(self, dist):
        """get version and homepage for a pkg_resources.Distribution"""
        ver = str(dist.parsed_version)
        url = None
        for line in dist.get_metadata_lines(dist.PKG_INFO):
            line = line.strip()
            if ':' not in line:
                continue
            (k, v) = line.split(':', 1)
            if k == 'Home-page':
                url = v.strip()
        return (ver, url)

    def _find_git_info(self):
        """
        Find information about the git repository, if this file is in a clone.

        :rtype: dict
        """
        res = {'url': None, 'tag': None, 'commit': None}
        oldcwd = os.getcwd()
        logger.debug("Current directory: %s", oldcwd)
        logger.debug("This file: %s (%s)", __file__, os.path.abspath(__file__))
        newdir = os.path.dirname(os.path.abspath(__file__))
        logger.debug("cd to %s", newdir)
        os.chdir(newdir)
        res['commit'] = _get_git_commit()
        if res['commit'] is not None:
            res['tag'] = _get_git_tag(res['commit'])
            res['url'] = _get_git_url()
        try:
            res['dirty'] = self._is_git_dirty()
        except Exception:
            pass
        logger.debug("cd back to %s", oldcwd)
        os.chdir(oldcwd)
        return res

    def _is_git_dirty(self):
        status = _check_output([
            'git',
            'status',
            '-u'
        ]).strip()
        if (('Your branch is up-to-date with' not in status and
                'HEAD detached at' not in status) or
                'nothing to commit' not in status):
            logger.debug("Git repository dirty based on status: %s", status)
            return True
        return False

def _check_output(args, stderr=None):
    """
    Python version compatibility wrapper for subprocess.check_output

    :param stderr: what to do with STDERR - None or an appropriate argument
     to subprocess.check_output / subprocess.Popen
    :return: string output
    """
    if hasattr(subprocess, 'check_output'):
        res = subprocess.check_output(args, stderr=stderr)
        # python3
        try:
            res.strip().split("\n")
        except TypeError:
            res = res.decode(locale.getdefaultlocale()[1])
    else:
        p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=stderr)
        (res, err) = p.communicate()
    return res

def _get_git_commit():
    """
    Get the current git commit of the current directory.

    :return: string short git hash
    """
    try:
        commit = _check_output([
            'git',
            'rev-parse',
            '--short',
            'HEAD'
        ], stderr=DEVNULL).strip()
        logger.debug("Found source git commit: %s", commit)
    except Exception:
        logger.debug("Unable to run git to get commit")
        commit = None
    return commit

def _get_git_tag(commit):
    """get the git tag for the specified commit, or None"""
    if commit is None:
        return None
    try:
        tag = _check_output([
            'git',
            'describe',
            '--exact-match',
            '--tags',
            commit
        ]).strip()
    except subprocess.CalledProcessError:
        tag = None
    return tag

def _get_git_url():
    try:
        url = None
        lines = _check_output([
            'git',
            'remote',
            '-v'
        ]).strip().split("\n")
        urls = {}
        for line in lines:
            parts = re.split(r'\s+', line)
            if parts[2] != '(fetch)':
                continue
            urls[parts[0]] = parts[1]
        if 'origin' in urls:
            return urls['origin']
        for k, v in urls.items():
            return v
    except subprocess.CalledProcessError:
        url = None
    return url
<commit_msg>fix empty tag bug<commit_after>"""
awslimitchecker/versioncheck.py

The latest version of this package is available at:
<https://github.com/jantman/awslimitchecker>

################################################################################
Copyright 2015 Jason Antman <jason@jasonantman.com> <http://www.jasonantman.com>

    This file is part of awslimitchecker, also known as awslimitchecker.

    awslimitchecker is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    awslimitchecker is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with awslimitchecker.  If not, see <http://www.gnu.org/licenses/>.

The Copyright and Authors attributions contained herein may not be removed or
otherwise altered, except to add the Author attribution of a contributor to
this work. (Additional Terms pursuant to Section 7b of the AGPL v3)
################################################################################
While not legally required, I sincerely request that anyone who finds
bugs please submit them at <https://github.com/jantman/awslimitchecker> or
to me via email, and that you send any contributions or improvements
either as a pull request on GitHub, or to me via email.
################################################################################

AUTHORS:
Jason Antman <jason@jasonantman.com> <http://www.jasonantman.com>
################################################################################
"""

import os
import subprocess
import logging
import re
try:
    from subprocess import DEVNULL
except ImportError:
    DEVNULL = open(os.devnull, 'wb')
import locale
logger = logging.getLogger(__name__)

class AGPLVersionChecker(object):

    def find_package_version(self):
        """
        Find the installed version of the specified package, and as much
        information about it as possible (source URL, git ref or tag, etc.)

        This attempts, to the best of our ability, to find out if the package
        was installed from git, and if so, provide information on the origin
        of that git repository and status of the clone. Otherwise, it uses
        pip and pkg_resources to find the version and homepage of the installed
        distribution.

        This class is not a sure-fire method of identifying the source of
        the distribution or ensuring AGPL compliance; it simply helps with this
        process _iff_ a modified version is installed from an editable git URL
        _and_ all changes are pushed up to the publicly-visible origin.

        Returns a dict with keys 'version', 'tag', 'commit', and 'url'.
        Values are strings or None.

        :param package_name: name of the package to find information for
        :type package_name: str
        :returns: information about the installed version of the package
        :rtype: dict
        """
        res = {
            'version': None,
            'url': None,
            'tag': None,
            'commit': None
        }
        git_info = self._find_git_info()
        logger.debug("Git info: %s", git_info)
        for k, v in git_info.items():
            if v is not None:
                res[k] = v
        if res['dirty'] and res['tag'] is not None:
            res['tag'] += '*'
        if res['dirty'] and res['commit'] is not None:
            res['commit'] += '*'
        try:
            pip_info = self._find_pip_info()
        except Exception:
            # we NEVER want this to crash the program
            pip_info = {}
        logger.debug("pip info: %s", pip_info)
        if 'version' in pip_info:
            res['version'] = pip_info['version']
        if 'url' in pip_info and res['url'] is None:
            res['url'] = pip_info['url']
        try:
            pkg_info = self._find_pkg_info()
        except Exception:
            pkg_info = {}
        logger.debug("pkg_resources info: %s", pkg_info)
        if 'version' in pkg_info and res['version'] is None:
            res['version'] = pkg_info['version']
        if 'url' in pkg_info and res['url'] is None:
            res['url'] = pkg_info['url']
        logger.debug("Final package info: %s", res)
        return res

    def _find_pkg_info(self):
        """
        Find information about the installed awslimitchecker from pkg_resources.

        :return: dict of information from pkg_resources
        """
        import pkg_resources
        dist = pkg_resources.require('awslimitchecker')[0]
        ver, url = self._dist_version_url(dist)
        return {'version': ver, 'url': url}

    def _find_pip_info(self):
        """
        Try to find information about the installed awslimitchecker from pip.
        This should be wrapped in a try/except.

        :return: dict of information from pip
        """
        import pip
        res = {}
        dist = None
        for d in pip.get_installed_distributions():
            if d.project_name == 'awslimitchecker':
                dist = d
        if dist is None:
            return res
        ver, url = self._dist_version_url(dist)
        res['version'] = ver
        # this is a bit of an ugly, lazy hack...
        req = pip.FrozenRequirement.from_dist(dist, [])
        if ':' in req.req or '@' in req.req:
            res['url'] = req.req
        else:
            res['url'] = url
        return res

    def _dist_version_url(self, dist):
        """get version and homepage for a pkg_resources.Distribution"""
        ver = str(dist.parsed_version)
        url = None
        for line in dist.get_metadata_lines(dist.PKG_INFO):
            line = line.strip()
            if ':' not in line:
                continue
            (k, v) = line.split(':', 1)
            if k == 'Home-page':
                url = v.strip()
        return (ver, url)

    def _find_git_info(self):
        """
        Find information about the git repository, if this file is in a clone.

        :rtype: dict
        """
        res = {'url': None, 'tag': None, 'commit': None}
        oldcwd = os.getcwd()
        logger.debug("Current directory: %s", oldcwd)
        logger.debug("This file: %s (%s)", __file__, os.path.abspath(__file__))
        newdir = os.path.dirname(os.path.abspath(__file__))
        logger.debug("cd to %s", newdir)
        os.chdir(newdir)
        res['commit'] = _get_git_commit()
        if res['commit'] is not None:
            res['tag'] = _get_git_tag(res['commit'])
            res['url'] = _get_git_url()
        try:
            res['dirty'] = self._is_git_dirty()
        except Exception:
            pass
        logger.debug("cd back to %s", oldcwd)
        os.chdir(oldcwd)
        return res

    def _is_git_dirty(self):
        status = _check_output([
            'git',
            'status',
            '-u'
        ]).strip()
        if (('Your branch is up-to-date with' not in status and
                'HEAD detached at' not in status) or
                'nothing to commit' not in status):
            logger.debug("Git repository dirty based on status: %s", status)
            return True
        return False

def _check_output(args, stderr=None):
    """
    Python version compatibility wrapper for subprocess.check_output

    :param stderr: what to do with STDERR - None or an appropriate argument
     to subprocess.check_output / subprocess.Popen
    :return: string output
    """
    if hasattr(subprocess, 'check_output'):
        res = subprocess.check_output(args, stderr=stderr)
        # python3
        try:
            res.strip().split("\n")
        except TypeError:
            res = res.decode(locale.getdefaultlocale()[1])
    else:
        p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=stderr)
        (res, err) = p.communicate()
    return res

def _get_git_commit():
    """
    Get the current git commit of the current directory.

    :return: string short git hash
    """
    try:
        commit = _check_output([
            'git',
            'rev-parse',
            '--short',
            'HEAD'
        ], stderr=DEVNULL).strip()
        logger.debug("Found source git commit: %s", commit)
    except Exception:
        logger.debug("Unable to run git to get commit")
        commit = None
    return commit

def _get_git_tag(commit):
    """get the git tag for the specified commit, or None"""
    if commit is None:
        return None
    try:
        tag = _check_output([
            'git',
            'describe',
            '--exact-match',
            '--tags',
            commit
        ]).strip()
    except subprocess.CalledProcessError:
        tag = None
    if tag == '':
        return None
    return tag

def _get_git_url():
    try:
        url = None
        lines = _check_output([
            'git',
            'remote',
            '-v'
        ]).strip().split("\n")
        urls = {}
        for line in lines:
            parts = re.split(r'\s+', line)
            if parts[2] != '(fetch)':
                continue
            urls[parts[0]] = parts[1]
        if 'origin' in urls:
            return urls['origin']
        for k, v in urls.items():
            return v
    except subprocess.CalledProcessError:
        url = None
    return url
<|endoftext|>


<commit_before># Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.

# Python
import datetime
import hashlib
import hmac
import uuid

# Django
from django.conf import settings
from django.db import models, connection
from django.contrib.auth.models import User
from django.utils.timezone import now as tz_now
from django.utils.translation import ugettext_lazy as _

# AWX
from awx.api.versioning import reverse
from awx.main.fields import AutoOneToOneField, ImplicitRoleField
from awx.main.models.base import * # noqa
from awx.main.models.rbac import (
    ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
    ROLE_SINGLETON_SYSTEM_AUDITOR,
)
from awx.main.models.mixins import ResourceMixin

__all__ = ['Organization', 'Team', 'Profile', 'AuthToken']

class Organization(CommonModel, NotificationFieldsModel, ResourceMixin):
    '''
    An organization is the basic unit of multi-tenancy divisions
    '''

    class Meta:
        app_label = 'main'
        ordering = ('name',)

    instance_groups = models.ManyToManyField(
        'InstanceGroup',
        blank=True,
    )
    admin_role = ImplicitRoleField(
        parent_role='singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
    )
    auditor_role = ImplicitRoleField(
        parent_role='singleton:' + ROLE_SINGLETON_SYSTEM_AUDITOR,
    )
    member_role = ImplicitRoleField(
        parent_role='admin_role',
    )
    read_role = ImplicitRoleField(
        parent_role=['member_role', 'auditor_role'],
    )

    def get_absolute_url(self, request=None):
        return reverse('api:organization_detail', kwargs={'pk': self.pk}, request=request)

    def __unicode__(self):
        return self.name

class Team(CommonModelNameNotUnique, ResourceMixin):
    '''
    A team is a group of users that work on common projects.
    '''

    class Meta:
        app_label = 'main'
        unique_together = [('organization', 'name')]
        ordering = ('organization__name', 'name')

    organization = models.ForeignKey(
        'Organization',
        blank=False,
        null=False,
        on_delete=models.CASCADE,
        related_name='teams',
    )
    admin_role = ImplicitRoleField(
        parent_role='organization.admin_role',
    )
    member_role = ImplicitRoleField(
        parent_role='admin_role',
    )
    read_role = ImplicitRoleField(
        parent_role=['organization.auditor_role', 'member_role'],
    )

    def get_absolute_url(self, request=None):
        return reverse('api:team_detail', kwargs={'pk': self.pk}, request=request)

class Profile(CreatedModifiedModel):
    '''
    Profile model related to User object. Currently stores LDAP DN for users
    loaded from LDAP.
    '''

    class Meta:
        app_label = 'main'

    user = AutoOneToOneField(
        'auth.User',
        related_name='profile',
        editable=False,
    )
    ldap_dn = models.CharField(
        max_length=1024,
        default='',
    )

"""
Since expiration and session expiration is event driven a token could be
invalidated for both reasons. Further, we only support a single reason for a
session token being invalid. For this case, mark the token as expired.

Note: Again, because the value of reason is event based. The reason may not be
set (i.e. may equal '') even though a session is expired or a limit is reached.
"""

class AuthToken(BaseModel):
    '''
    Custom authentication tokens per user with expiration and request-specific
    data.
    '''

    REASON_CHOICES = [
        ('', _('Token not invalidated')),
        ('timeout_reached', _('Token is expired')),
        ('limit_reached', _('The maximum number of allowed sessions for this user has been exceeded.')),
        # invalid_token is not a used data-base value, but is returned by the
        # api when a token is not found
        ('invalid_token', _('Invalid token')),
    ]

    class Meta:
        app_label = 'main'

    key = models.CharField(max_length=40, primary_key=True)
    user = prevent_search(models.ForeignKey('auth.User',
                          related_name='auth_tokens', on_delete=models.CASCADE))
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)
    expires = models.DateTimeField(default=tz_now)
    request_hash = prevent_search(models.CharField(max_length=40, blank=True,
                                                   default=''))
    reason = models.CharField(
        max_length=1024,
        blank=True,
        default='',
        help_text=_('Reason the auth token was invalidated.')
    )

    @staticmethod
    def reason_long(reason):
        for x in AuthToken.REASON_CHOICES:
            if x[0] == reason:
                return unicode(x[1])
        return None

    @classmethod
    def get_request_hash(cls, request):
        h = hashlib.sha1()
        h.update(settings.SECRET_KEY)
        for header in settings.REMOTE_HOST_HEADERS:
            value = request.META.get(header, '').split(',')[0].strip()
            if value:
                h.update(value)
                break
        h.update(request.META.get('HTTP_USER_AGENT', ''))
        return h.hexdigest()

    def save(self, *args, **kwargs):
        if not self.pk:
            self.refresh(save=False)
        if not self.key:
            self.key = self.generate_key()
        return super(AuthToken, self).save(*args, **kwargs)

    def refresh(self, now=None, save=True):
        if not now:
            now = tz_now()
        if not self.pk or not self.is_expired(now=now):
            self.expires = now + datetime.timedelta(seconds=settings.AUTH_TOKEN_EXPIRATION)
            if save:
                connection.on_commit(lambda: self.save(update_fields=['expires']))

    def invalidate(self, reason='timeout_reached', save=True):
        if not AuthToken.reason_long(reason):
            raise ValueError(_('Invalid reason specified'))
        self.reason = reason
        if save:
            self.save()
        return reason

    @staticmethod
    def get_tokens_over_limit(user, now=None):
        if now is None:
            now = tz_now()
        invalid_tokens = AuthToken.objects.none()
        if settings.AUTH_TOKEN_PER_USER != -1:
            invalid_tokens = AuthToken.objects.filter(
                user=user,
                expires__gt=now,
                reason='',
            ).order_by('-created')[settings.AUTH_TOKEN_PER_USER:]
        return invalid_tokens

    def generate_key(self):
        unique = uuid.uuid4()
        return hmac.new(unique.bytes, digestmod=hashlib.sha1).hexdigest()

    def is_expired(self, now=None):
        if not now:
            now = tz_now()
        return bool(self.expires < now)

    @property
    def invalidated(self):
        return bool(self.reason != '')

    """
    Token is valid if it's in the set of unexpired tokens.
    The unexpired token set is:
        * tokens not expired
        * limited to number of tokens per-user
        * sorted by created on date
    """
    def in_valid_tokens(self, now=None):
        if not now:
            now = tz_now()
        valid_n_tokens_qs = self.user.auth_tokens.filter(
            expires__gt=now,
            reason='',
        ).order_by('-created')[0:settings.AUTH_TOKEN_PER_USER]
        valid_n_tokens = valid_n_tokens_qs.values_list('key', flat=True)

        return bool(self.key in valid_n_tokens)

    def __unicode__(self):
        return self.key

# Add get_absolute_url method to User model if not present.
if not hasattr(User, 'get_absolute_url'):
    def user_get_absolute_url(user, request=None):
        return reverse('api:user_detail', kwargs={'pk': user.pk}, request=request)
    User.add_to_class('get_absolute_url', user_get_absolute_url)
<commit_msg>Fix issue when per user tokens are disabled<commit_after># Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.

# Python
import datetime
import hashlib
import hmac
import uuid

# Django
from django.conf import settings
from django.db import models, connection
from django.contrib.auth.models import User
from django.utils.timezone import now as tz_now
from django.utils.translation import ugettext_lazy as _

# AWX
from awx.api.versioning import reverse
from awx.main.fields import AutoOneToOneField, ImplicitRoleField
from awx.main.models.base import * # noqa
from awx.main.models.rbac import (
    ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
    ROLE_SINGLETON_SYSTEM_AUDITOR,
)
from awx.main.models.mixins import ResourceMixin

__all__ = ['Organization', 'Team', 'Profile', 'AuthToken']

class Organization(CommonModel, NotificationFieldsModel, ResourceMixin):
    '''
    An organization is the basic unit of multi-tenancy divisions
    '''

    class Meta:
        app_label = 'main'
        ordering = ('name',)

    instance_groups = models.ManyToManyField(
        'InstanceGroup',
        blank=True,
    )
    admin_role = ImplicitRoleField(
        parent_role='singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
    )
    auditor_role = ImplicitRoleField(
        parent_role='singleton:' + ROLE_SINGLETON_SYSTEM_AUDITOR,
    )
    member_role = ImplicitRoleField(
        parent_role='admin_role',
    )
    read_role = ImplicitRoleField(
        parent_role=['member_role', 'auditor_role'],
    )

    def get_absolute_url(self, request=None):
        return reverse('api:organization_detail', kwargs={'pk': self.pk}, request=request)

    def __unicode__(self):
        return self.name

class Team(CommonModelNameNotUnique, ResourceMixin):
    '''
    A team is a group of users that work on common projects.
    '''

    class Meta:
        app_label = 'main'
        unique_together = [('organization', 'name')]
        ordering = ('organization__name', 'name')

    organization = models.ForeignKey(
        'Organization',
        blank=False,
        null=False,
        on_delete=models.CASCADE,
        related_name='teams',
    )
    admin_role = ImplicitRoleField(
        parent_role='organization.admin_role',
    )
    member_role = ImplicitRoleField(
        parent_role='admin_role',
    )
    read_role = ImplicitRoleField(
        parent_role=['organization.auditor_role', 'member_role'],
    )

    def get_absolute_url(self, request=None):
        return reverse('api:team_detail', kwargs={'pk': self.pk}, request=request)

class Profile(CreatedModifiedModel):
    '''
    Profile model related to User object. Currently stores LDAP DN for users
    loaded from LDAP.
    '''

    class Meta:
        app_label = 'main'

    user = AutoOneToOneField(
        'auth.User',
        related_name='profile',
        editable=False,
    )
    ldap_dn = models.CharField(
        max_length=1024,
        default='',
    )

"""
Since expiration and session expiration is event driven a token could be
invalidated for both reasons. Further, we only support a single reason for a
session token being invalid. For this case, mark the token as expired.

Note: Again, because the value of reason is event based. The reason may not be
set (i.e. may equal '') even though a session is expired or a limit is reached.
"""

class AuthToken(BaseModel):
    '''
    Custom authentication tokens per user with expiration and request-specific
    data.
    '''

    REASON_CHOICES = [
        ('', _('Token not invalidated')),
        ('timeout_reached', _('Token is expired')),
        ('limit_reached', _('The maximum number of allowed sessions for this user has been exceeded.')),
        # invalid_token is not a used data-base value, but is returned by the
        # api when a token is not found
        ('invalid_token', _('Invalid token')),
    ]

    class Meta:
        app_label = 'main'

    key = models.CharField(max_length=40, primary_key=True)
    user = prevent_search(models.ForeignKey('auth.User',
                          related_name='auth_tokens', on_delete=models.CASCADE))
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)
    expires = models.DateTimeField(default=tz_now)
    request_hash = prevent_search(models.CharField(max_length=40, blank=True,
                                                   default=''))
    reason = models.CharField(
        max_length=1024,
        blank=True,
        default='',
        help_text=_('Reason the auth token was invalidated.')
    )

    @staticmethod
    def reason_long(reason):
        for x in AuthToken.REASON_CHOICES:
            if x[0] == reason:
                return unicode(x[1])
        return None

    @classmethod
    def get_request_hash(cls, request):
        h = hashlib.sha1()
        h.update(settings.SECRET_KEY)
        for header in settings.REMOTE_HOST_HEADERS:
            value = request.META.get(header, '').split(',')[0].strip()
            if value:
                h.update(value)
                break
        h.update(request.META.get('HTTP_USER_AGENT', ''))
        return h.hexdigest()

    def save(self, *args, **kwargs):
        if not self.pk:
            self.refresh(save=False)
        if not self.key:
            self.key = self.generate_key()
        return super(AuthToken, self).save(*args, **kwargs)

    def refresh(self, now=None, save=True):
        if not now:
            now = tz_now()
        if not self.pk or not self.is_expired(now=now):
            self.expires = now + datetime.timedelta(seconds=settings.AUTH_TOKEN_EXPIRATION)
            if save:
                connection.on_commit(lambda: self.save(update_fields=['expires']))

    def invalidate(self, reason='timeout_reached', save=True):
        if not AuthToken.reason_long(reason):
            raise ValueError(_('Invalid reason specified'))
        self.reason = reason
        if save:
            self.save()
        return reason

    @staticmethod
    def get_tokens_over_limit(user, now=None):
        if now is None:
            now = tz_now()
        invalid_tokens = AuthToken.objects.none()
        if settings.AUTH_TOKEN_PER_USER != -1:
            invalid_tokens = AuthToken.objects.filter(
                user=user,
                expires__gt=now,
                reason='',
            ).order_by('-created')[settings.AUTH_TOKEN_PER_USER:]
        return invalid_tokens

    def generate_key(self):
        unique = uuid.uuid4()
        return hmac.new(unique.bytes, digestmod=hashlib.sha1).hexdigest()

    def is_expired(self, now=None):
        if not now:
            now = tz_now()
        return bool(self.expires < now)

    @property
    def invalidated(self):
        return bool(self.reason != '')

    """
    Token is valid if it's in the set of unexpired tokens.
    The unexpired token set is:
        * tokens not expired
        * limited to number of tokens per-user
        * sorted by created on date
    """
    def in_valid_tokens(self, now=None):
        if not now:
            now = tz_now()
        valid_n_tokens_qs = self.user.auth_tokens.filter(
            expires__gt=now,
            reason='',
        ).order_by('-created')
        if settings.AUTH_TOKEN_PER_USER != -1:
            valid_n_tokens_qs = valid_n_tokens_qs[0:settings.AUTH_TOKEN_PER_USER]
        valid_n_tokens = valid_n_tokens_qs.values_list('key', flat=True)

        return bool(self.key in valid_n_tokens)

    def __unicode__(self):
        return self.key

# Add get_absolute_url method to User model if not present.
if not hasattr(User, 'get_absolute_url'):
    def user_get_absolute_url(user, request=None):
        return reverse('api:user_detail', kwargs={'pk': user.pk}, request=request)
    User.add_to_class('get_absolute_url', user_get_absolute_url)
<|endoftext|>