<commit_before>import datetime
import re

from django.shortcuts import render_to_response, get_object_or_404, redirect
from models import Article, Version
import models
import json
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.core.urlresolvers import reverse
import urllib
import django.db
import time
from django.template import Context, RequestContext, loader
from django.views.decorators.cache import cache_page
from django.db.models import Count, Max

OUT_FORMAT = '%B %d, %Y at %l:%M%P EDT'

SEARCH_ENGINES = """
http://www.ask.com
http://www.google
https://www.google
search.yahoo.com
http://www.bing.com
""".split()

def came_from_search_engine(request):
    return any(x in request.META.get('HTTP_REFERER', '')
               for x in SEARCH_ENGINES)

def Http400():
    t = loader.get_template('404.html')
    return HttpResponse(t.render(Context()), status=400)

def get_first_update(source):
    if source is None:
        source = ''
    updates = models.Article.objects.order_by('last_update').filter(last_update__gt=datetime.datetime(1990, 1, 1, 0, 0),
                                                                    url__contains=source)
    try:
        return updates[0].last_update
    except IndexError:
        return datetime.datetime.now()

def get_last_update(source):
    if source is None:
        source = ''
    updates = models.Article.objects.order_by('-last_update').filter(last_update__gt=datetime.datetime(1990, 1, 1, 0, 0), url__contains=source)
    try:
        return updates[0].last_update
    except IndexError:
        return datetime.datetime.now()

def get_articles(source=None, distance=0):
    articles = []
    rx = re.compile(r'^https?://(?:[^/]*\.)%s/' % source if source else '')

    pagelength = datetime.timedelta(days=1)
    end_date = datetime.datetime.now() - distance * pagelength
    start_date = end_date - pagelength

    article_qs = Article.objects.all().annotate(
        version_count=Count('version'), age=Max('version__date')
    ).filter(
        age__gt=start_date, age__lt=end_date,
        version_count__gte=2, version__boring=False
    ).prefetch_related('version_set')

    for article in article_qs:
        url = article.url
        if not rx.match(url):
            print 'REJECTING', url
            continue
        if 'blogs.nytimes.com' in url: #XXX temporary
            continue

        versions = sorted(
            [version for version in article.version_set.all() if not version.boring],
            key=lambda version: version,
            reverse=True
        )

        if len(versions) < 2:
            continue
        rowinfo = get_rowinfo(article, versions)
        articles.append((article, versions[-1], rowinfo))
    articles.sort(key = lambda x: x[-1][0][1].date, reverse=True)

    return articles

SOURCES = '''nytimes.com cnn.com politico.com washingtonpost.com
bbc.co.uk nos.nl nu.nl'''.split()

def is_valid_domain(domain):
    """Cheap method to tell whether a domain is being tracked."""
    return any(domain.endswith(source) for source in SOURCES)

@cache_page(60 * 30)  #30 minute cache
def browse(request, source=''):
    if source not in SOURCES + ['']:
        raise Http404
    pagestr=request.REQUEST.get('page', '1')
    try:
        page = int(pagestr)
    except ValueError:
        page = 1

    first_update = get_first_update(source)
    num_pages = (datetime.datetime.now() - first_update).days + 1
    page_list=range(1, 1+num_pages)

    articles = get_articles(source=source, distance=page-1)
    return render_to_response('browse.html', {
            'source': source, 'articles': articles,
            'page':page,
            'page_list': page_list,
            'first_update': first_update,
            'sources': SOURCES
            })

@cache_page(60 * 30)  #30 minute cache
def feed(request, source=''):
    if source not in SOURCES + ['']:
        raise Http404
    pagestr=request.REQUEST.get('page', '1')
    try:
        page = int(pagestr)
    except ValueError:
        page = 1

    first_update = get_first_update(source)
    last_update = get_last_update(source)
    num_pages = (datetime.datetime.now() - first_update).days + 1
    page_list=range(1, 1+num_pages)

    articles = get_articles(source=source, distance=page-1)
    return render_to_response('feed.xml', {
            'source': source, 'articles': articles,
            'page':page,
            'request':request,
            'page_list': page_list,
            'last_update': last_update,
            'sources': SOURCES
            },
            context_instance=RequestContext(request),
            mimetype='application/atom+xml')

def old_diffview(request):
    """Support for legacy diff urls"""
    url = request.REQUEST.get('url')
    v1tag = request.REQUEST.get('v1')
    v2tag = request.REQUEST.get('v2')
    if url is None or v1tag is None or v2tag is None:
        return HttpResponseRedirect(reverse(front))

    try:
        v1 = Version.objects.get(v=v1tag)
        v2 = Version.objects.get(v=v2tag)
    except Version.DoesNotExist:
        return Http400()

    try:
        article = Article.objects.get(url=url)
    except Article.DoesNotExist:
        return Http400()

    return redirect(reverse('diffview', kwargs=dict(vid1=v1.id,
                                                    vid2=v2.id,
                                                    urlarg=article.filename())),
                    permanent=True)

def diffview(request, vid1, vid2, urlarg):
    # urlarg is unused, and only for readability
    # Could be strict and enforce urlarg == article.filename()
    try:
        v1 = Version.objects.get(id=int(vid1))
        v2 = Version.objects.get(id=int(vid2))
    except Version.DoesNotExist:
        raise Http404

    article = v1.article

    if v1.article != v2.article:
        raise Http404

    title = article.latest_version().title

    versions = dict(enumerate(article.versions()))

    adjacent_versions = []
    dates = []
    texts = []

    for v in (v1, v2):
        texts.append(v.text())
        dates.append(v.date.strftime(OUT_FORMAT))

        indices = [i for i, x in versions.items() if x == v]
        if not indices:
            #One of these versions doesn't exist / is boring
            return Http400()
        index = indices[0]
        adjacent_versions.append([versions.get(index+offset)
                                  for offset in (-1, 1)])

    if any(x is None for x in texts):
        return Http400()

    links = []
    for i in range(2):
        if all(x[i] for x in adjacent_versions):
            diffl = reverse('diffview', kwargs=dict(vid1=adjacent_versions[0][i].id,
                                                    vid2=adjacent_versions[1][i].id,
                                                    urlarg=article.filename()))
            links.append(diffl)
        else:
            links.append('')

    return render_to_response('diffview.html', {
            'title': title,
            'date1':dates[0], 'date2':dates[1],
            'text1':texts[0], 'text2':texts[1],
            'prev':links[0], 'next':links[1],
            'article_shorturl': article.filename(),
            'article_url': article.url, 'v1': v1, 'v2': v2,
            'display_search_banner': came_from_search_engine(request),
            })

def get_rowinfo(article, version_lst=None):
    if version_lst is None:
        version_lst = article.versions()
    rowinfo = []
    lastv = None
    urlarg = article.filename()
    for version in version_lst:
        date = version.date
        if lastv is None:
            diffl = ''
        else:
            diffl = reverse('diffview', kwargs=dict(vid1=lastv.id,
                                                    vid2=version.id,
                                                    urlarg=urlarg))
        rowinfo.append((diffl, version))
        lastv = version
    rowinfo.reverse()
    return rowinfo

def prepend_http(url):
    """Return a version of the url that starts with the proper scheme.

    url may look like

    www.nytimes.com
    https:/www.nytimes.com    <- because double slashes get stripped
    http://www.nytimes.com
    """
    components = url.split('/', 2)
    if len(components) <= 2 or '.' in components[0]:
        components = ['http:', '']+components
    elif components[1]:
        components[1:1] = ['']
    return '/'.join(components)

def article_history(request, urlarg=''):
    url = request.REQUEST.get('url') # this is the deprecated interface.
    if url is None:
        url = urlarg
    if len(url) == 0:
        return HttpResponseRedirect(reverse(front))

    url = url.split('?')[0]  #For if user copy-pastes from news site

    url = prepend_http(url)

    # This is a hack to deal with unicode passed in the URL.
    # Otherwise gives an error, since our table character set is latin1.
    url = url.encode('ascii', 'ignore')

    # Give an error on urls with the wrong hostname without hitting the
    # database.  These queries are usually spam.
    domain = url.split('/')[2]
    if not is_valid_domain(domain):
        return render_to_response('article_history_missing.html', {'url': url})

    try:
        article = Article.objects.get(url=url)
    except Article.DoesNotExist:
        try:
            return render_to_response('article_history_missing.html', {'url': url})
        except (TypeError, ValueError):
            # bug in django + mod_rewrite can cause this. =/
            return HttpResponse('Bug!')

    if len(urlarg) == 0:
        return HttpResponseRedirect(reverse(article_history, args=[article.filename()]))

    rowinfo = get_rowinfo(article)
    return render_to_response('article_history.html', {'article':article,
                                                       'versions':rowinfo,
            'display_search_banner': came_from_search_engine(request),
                                                       })
def article_history_feed(request, url=''):
    url = prepend_http(url)
    article = get_object_or_404(Article, url=url)
    rowinfo = get_rowinfo(article)
    return render_to_response('article_history.xml',
                              { 'article': article,
                                'versions': rowinfo,
                                'request': request,
                                },
                              context_instance=RequestContext(request),
                              mimetype='application/atom+xml')

def json_view(request, vid):
    version = get_object_or_404(Version, id=int(vid))
    data = dict(
        title=version.title,
        byline = version.byline,
        date = version.date.isoformat(),
        text = version.text(),
        )
    return HttpResponse(json.dumps(data), mimetype="application/json")

def upvote(request):
    article_url = request.REQUEST.get('article_url')
    diff_v1 = request.REQUEST.get('diff_v1')
    diff_v2 = request.REQUEST.get('diff_v2')
    remote_ip = request.META.get('REMOTE_ADDR')
    article_id = Article.objects.get(url=article_url).id
    models.Upvote(article_id=article_id, diff_v1=diff_v1, diff_v2=diff_v2, creation_time=datetime.datetime.now(), upvoter_ip=remote_ip).save()
    return render_to_response('upvote.html')

def about(request):
    return render_to_response('about.html', {})

def examples(request):
    return render_to_response('examples.html', {})

def contact(request):
    return render_to_response('contact.html', {})

def front(request):
    return render_to_response('front.html', {'sources': SOURCES})

def subscribe(request):
    return render_to_response('subscribe.html', {})

def press(request):
    return render_to_response('press.html', {})

<commit_msg>Oops, I missed this up.<commit_after>import datetime
import re

from django.shortcuts import render_to_response, get_object_or_404, redirect
from models import Article, Version
import models
import json
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.core.urlresolvers import reverse
import urllib
import django.db
import time
from django.template import Context, RequestContext, loader
from django.views.decorators.cache import cache_page
from django.db.models import Count, Max

OUT_FORMAT = '%B %d, %Y at %l:%M%P EDT'

SEARCH_ENGINES = """
http://www.ask.com
http://www.google
https://www.google
search.yahoo.com
http://www.bing.com
""".split()

def came_from_search_engine(request):
    return any(x in request.META.get('HTTP_REFERER', '')
               for x in SEARCH_ENGINES)

def Http400():
    t = loader.get_template('404.html')
    return HttpResponse(t.render(Context()), status=400)

def get_first_update(source):
    if source is None:
        source = ''
    updates = models.Article.objects.order_by('last_update').filter(last_update__gt=datetime.datetime(1990, 1, 1, 0, 0),
                                                                    url__contains=source)
    try:
        return updates[0].last_update
    except IndexError:
        return datetime.datetime.now()

def get_last_update(source):
    if source is None:
        source = ''
    updates = models.Article.objects.order_by('-last_update').filter(last_update__gt=datetime.datetime(1990, 1, 1, 0, 0), url__contains=source)
    try:
        return updates[0].last_update
    except IndexError:
        return datetime.datetime.now()

def get_articles(source=None, distance=0):
    articles = []
    rx = re.compile(r'^https?://(?:[^/]*\.)%s/' % source if source else '')

    pagelength = datetime.timedelta(days=1)
    end_date = datetime.datetime.now() - distance * pagelength
    start_date = end_date - pagelength

    article_qs = Article.objects.all().annotate(
        version_count=Count('version'), age=Max('version__date')
    ).filter(
        age__gt=start_date, age__lt=end_date,
        version_count__gte=2, version__boring=False
    ).prefetch_related('version_set')

    for article in article_qs:
        url = article.url
        if not rx.match(url):
            print 'REJECTING', url
            continue
        if 'blogs.nytimes.com' in url: #XXX temporary
            continue

        versions = sorted(
            [version for version in article.version_set.all() if not version.boring],
            key=lambda version: version.date,
        )

        if len(versions) < 2:
            continue
        rowinfo = get_rowinfo(article, versions)
        articles.append((article, versions[-1], rowinfo))
    articles.sort(key = lambda x: x[-1][0][1].date, reverse=True)

    return articles

SOURCES = '''nytimes.com cnn.com politico.com washingtonpost.com
bbc.co.uk nos.nl nu.nl'''.split()

def is_valid_domain(domain):
    """Cheap method to tell whether a domain is being tracked."""
    return any(domain.endswith(source) for source in SOURCES)

#@cache_page(60 * 30)  #30 minute cache
def browse(request, source=''):
    if source not in SOURCES + ['']:
        raise Http404
    pagestr=request.REQUEST.get('page', '1')
    try:
        page = int(pagestr)
    except ValueError:
        page = 1

    first_update = get_first_update(source)
    num_pages = (datetime.datetime.now() - first_update).days + 1
    page_list=range(1, 1+num_pages)

    articles = get_articles(source=source, distance=page-1)
    return render_to_response('browse.html', {
            'source': source, 'articles': articles,
            'page':page,
            'page_list': page_list,
            'first_update': first_update,
            'sources': SOURCES
            })

@cache_page(60 * 30)  #30 minute cache
def feed(request, source=''):
    if source not in SOURCES + ['']:
        raise Http404
    pagestr=request.REQUEST.get('page', '1')
    try:
        page = int(pagestr)
    except ValueError:
        page = 1

    first_update = get_first_update(source)
    last_update = get_last_update(source)
    num_pages = (datetime.datetime.now() - first_update).days + 1
    page_list=range(1, 1+num_pages)

    articles = get_articles(source=source, distance=page-1)
    return render_to_response('feed.xml', {
            'source': source, 'articles': articles,
            'page':page,
            'request':request,
            'page_list': page_list,
            'last_update': last_update,
            'sources': SOURCES
            },
            context_instance=RequestContext(request),
            mimetype='application/atom+xml')

def old_diffview(request):
    """Support for legacy diff urls"""
    url = request.REQUEST.get('url')
    v1tag = request.REQUEST.get('v1')
    v2tag = request.REQUEST.get('v2')
    if url is None or v1tag is None or v2tag is None:
        return HttpResponseRedirect(reverse(front))

    try:
        v1 = Version.objects.get(v=v1tag)
        v2 = Version.objects.get(v=v2tag)
    except Version.DoesNotExist:
        return Http400()

    try:
        article = Article.objects.get(url=url)
    except Article.DoesNotExist:
        return Http400()

    return redirect(reverse('diffview', kwargs=dict(vid1=v1.id,
                                                    vid2=v2.id,
                                                    urlarg=article.filename())),
                    permanent=True)

def diffview(request, vid1, vid2, urlarg):
    # urlarg is unused, and only for readability
    # Could be strict and enforce urlarg == article.filename()
    try:
        v1 = Version.objects.get(id=int(vid1))
        v2 = Version.objects.get(id=int(vid2))
    except Version.DoesNotExist:
        raise Http404

    article = v1.article

    if v1.article != v2.article:
        raise Http404

    title = article.latest_version().title

    versions = dict(enumerate(article.versions()))

    adjacent_versions = []
    dates = []
    texts = []

    for v in (v1, v2):
        texts.append(v.text())
        dates.append(v.date.strftime(OUT_FORMAT))

        indices = [i for i, x in versions.items() if x == v]
        if not indices:
            #One of these versions doesn't exist / is boring
            return Http400()
        index = indices[0]
        adjacent_versions.append([versions.get(index+offset)
                                  for offset in (-1, 1)])

    if any(x is None for x in texts):
        return Http400()

    links = []
    for i in range(2):
        if all(x[i] for x in adjacent_versions):
            diffl = reverse('diffview', kwargs=dict(vid1=adjacent_versions[0][i].id,
                                                    vid2=adjacent_versions[1][i].id,
                                                    urlarg=article.filename()))
            links.append(diffl)
        else:
            links.append('')

    return render_to_response('diffview.html', {
            'title': title,
            'date1':dates[0], 'date2':dates[1],
            'text1':texts[0], 'text2':texts[1],
            'prev':links[0], 'next':links[1],
            'article_shorturl': article.filename(),
            'article_url': article.url, 'v1': v1, 'v2': v2,
            'display_search_banner': came_from_search_engine(request),
            })

def get_rowinfo(article, version_lst=None):
    if version_lst is None:
        version_lst = article.versions()
    rowinfo = []
    lastv = None
    urlarg = article.filename()
    for version in version_lst:
        date = version.date
        if lastv is None:
            diffl = ''
        else:
            diffl = reverse('diffview', kwargs=dict(vid1=lastv.id,
                                                    vid2=version.id,
                                                    urlarg=urlarg))
        rowinfo.append((diffl, version))
        lastv = version
    rowinfo.reverse()
    return rowinfo

def prepend_http(url):
    """Return a version of the url that starts with the proper scheme.

    url may look like

    www.nytimes.com
    https:/www.nytimes.com    <- because double slashes get stripped
    http://www.nytimes.com
    """
    components = url.split('/', 2)
    if len(components) <= 2 or '.' in components[0]:
        components = ['http:', '']+components
    elif components[1]:
        components[1:1] = ['']
    return '/'.join(components)

def article_history(request, urlarg=''):
    url = request.REQUEST.get('url') # this is the deprecated interface.
    if url is None:
        url = urlarg
    if len(url) == 0:
        return HttpResponseRedirect(reverse(front))

    url = url.split('?')[0]  #For if user copy-pastes from news site

    url = prepend_http(url)

    # This is a hack to deal with unicode passed in the URL.
    # Otherwise gives an error, since our table character set is latin1.
    url = url.encode('ascii', 'ignore')

    # Give an error on urls with the wrong hostname without hitting the
    # database.  These queries are usually spam.
    domain = url.split('/')[2]
    if not is_valid_domain(domain):
        return render_to_response('article_history_missing.html', {'url': url})

    try:
        article = Article.objects.get(url=url)
    except Article.DoesNotExist:
        try:
            return render_to_response('article_history_missing.html', {'url': url})
        except (TypeError, ValueError):
            # bug in django + mod_rewrite can cause this. =/
            return HttpResponse('Bug!')

    if len(urlarg) == 0:
        return HttpResponseRedirect(reverse(article_history, args=[article.filename()]))

    rowinfo = get_rowinfo(article)
    return render_to_response('article_history.html', {'article':article,
                                                       'versions':rowinfo,
            'display_search_banner': came_from_search_engine(request),
                                                       })
def article_history_feed(request, url=''):
    url = prepend_http(url)
    article = get_object_or_404(Article, url=url)
    rowinfo = get_rowinfo(article)
    return render_to_response('article_history.xml',
                              { 'article': article,
                                'versions': rowinfo,
                                'request': request,
                                },
                              context_instance=RequestContext(request),
                              mimetype='application/atom+xml')

def json_view(request, vid):
    version = get_object_or_404(Version, id=int(vid))
    data = dict(
        title=version.title,
        byline = version.byline,
        date = version.date.isoformat(),
        text = version.text(),
        )
    return HttpResponse(json.dumps(data), mimetype="application/json")

def upvote(request):
    article_url = request.REQUEST.get('article_url')
    diff_v1 = request.REQUEST.get('diff_v1')
    diff_v2 = request.REQUEST.get('diff_v2')
    remote_ip = request.META.get('REMOTE_ADDR')
    article_id = Article.objects.get(url=article_url).id
    models.Upvote(article_id=article_id, diff_v1=diff_v1, diff_v2=diff_v2, creation_time=datetime.datetime.now(), upvoter_ip=remote_ip).save()
    return render_to_response('upvote.html')

def about(request):
    return render_to_response('about.html', {})

def examples(request):
    return render_to_response('examples.html', {})

def contact(request):
    return render_to_response('contact.html', {})

def front(request):
    return render_to_response('front.html', {'sources': SOURCES})

def subscribe(request):
    return render_to_response('subscribe.html', {})

def press(request):
    return render_to_response('press.html', {})

<|endoftext|>


<commit_before>import sys
import os
# add Backend directory to python path so we can do
# cross directory imports
path = os.path.dirname(os.path.realpath(__file__))
sys.path.append(path[:-7])

import socketserver
import DB.api.handler as handler
PORT = 8081

class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
    """ Handle requests in a separate thread """

Handler = handler.ApiHandler

httpd = ThreadedTCPServer(("", PORT), Handler)

print("serving at port", PORT)
httpd.serve_forever()

<commit_msg>8081->8080<commit_after>import sys
import os
# add Backend directory to python path so we can do
# cross directory imports
path = os.path.dirname(os.path.realpath(__file__))
sys.path.append(path[:-7])

import socketserver
import DB.api.handler as handler
PORT = 8080

class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
    """ Handle requests in a separate thread """

Handler = handler.ApiHandler

httpd = ThreadedTCPServer(("", PORT), Handler)

print("serving at port", PORT)
httpd.serve_forever()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import division
__author__ = 'nickl-'

import sys
import textwrap

from .__version__ import __version_info__

from .cache import CacheProviderFactory
from .adapters import AVAILABLE_ADAPTERS

def coroutine(func):
    def start(*args,**kwargs):
        cr = func(*args,**kwargs)
        cr.next()
        return cr
    return start

class ProgressTicker():

    ref = None
    result = None
    steps = len(AVAILABLE_ADAPTERS)
    taken = 0

    def routine(self, routine):
        self.ref = routine

    def done(self):
        return (self.taken * 100) // self.steps

    def terminate(self):
        self.taken = self.steps
        try:
            self.ref.send('terminate')
        except StopIteration:
            pass

    def send(self, args):
        if isinstance(args, int):
            self.taken += args
        elif isinstance(args[0], str) and args[0] == 'step':
            self.steps += args[1]
        else:
            self.taken += args[0]
            if isinstance(args[1], dict):
                if not self.result:
                    self.result = {}
                self.result.update(args[1])
            elif isinstance(args[1], list):
                if not self.result:
                    self.result = []
                self.result.extend(args[1])
        self.ref.send(self.result)

class CommandProcessor():

    cache = CacheProviderFactory().whichProvider()
    data = None
    out = None
    ticker = ProgressTicker()
    clear = '\x1b[2J\x1b[0;0H'

    def __init__(self, data):
        self.data = data
        self.out = self.write()
        if self.data.invalidate:
            self.cache.invalidate(True)
        self.out.send(
            self.clear +
            'aero v{}.{}.{} {} {}\n\n'.format(*__version_info__)
        )

    def execute(self):
        try:
            next = self.wiring()
            self.do(self.data.packages, next)
        except BaseException as e:
            self.out.send(
                '\n\n[{}{}] Terminating...\n'.format(
                    type(e).__name__,
                    ': {}'.format(e) if len(str(e)) else ''
                ))
            self.ticker.terminate()

    # wire coroutines
    def wiring(self):
        self.ticker.routine(self.progress(self.res()))
        return self.each(self.call(self.ticker))

    @coroutine
    def res(self):
        while True: (yield)

    def cmd(self):
        return self.__class__.__name__.replace('Command', '').lower()

    def do(self, packages, adapters):
        self.ticker.steps = len(packages) * len(AVAILABLE_ADAPTERS)
        for package in packages:
            if ':' in package:
                mngr = package.partition(':')
                pkg = mngr[2]
                mngr = mngr[0]
            else:
                mngr = None
                pkg = package
            adapters.send((pkg, mngr))

    @coroutine
    def write(self):
        from StringIO import StringIO
        out = StringIO()
        while True:
            text = (yield)
            out.write(str(text))
            print out.getvalue()

    def seen(self, command, adapter, package, result=False):
        cache_key = command + "+" + adapter + ":" + package
        return self.cache.seen(cache_key, result)

    @coroutine
    def each(self, target):
        while True:
            args = (yield)
            package = args[0]
            manager = args[1]
            for adapter in AVAILABLE_ADAPTERS:
                adapter_name = adapter[0]
                if adapter_name not in self.data.disabled:
                    if not manager or manager == adapter_name:
                        length = str(max([len(a[0]) for a in AVAILABLE_ADAPTERS]))
                        self.out.send(
                            ('Doing an aero {} of package: {} using {:<' + length + '} ').format(
                                self.cmd(), package, adapter_name
                            )
                        )
                        res = self.seen(self.cmd(), adapter_name, package)
                        if self.data.invalidate or res is False:
                            target.send((adapter, package))
                        else:
                            self.out.send('Found {:>4} options [CACHED]\n'.format(len(res)))
                            self.ticker.send((1, res))
                    else:
                        self.ticker.send(1)
                else:
                    self.ticker.send(1)

    @coroutine
    def progress(self, target):
        from progbar import ProgBar
        bar = ProgBar('Progress: ', 30)
        bar.start()
        while True:
            result = (yield)
            self.out.send('')
            bar.percent = self.ticker.done()
            if self.ticker.done() == 100:
                if bar.isAlive():
                    bar.stop()
                bar.join()
                self.out.send('')
                if not isinstance(result, str): # terminate
                    target.send(result)

    @coroutine
    def call(self, target):
        while True:
            adapter_name = ''
            payload = (yield)
            adapter_name = payload[0][0]
            adapter = payload[0][1]
            package = payload[1]
            try:
                adapter.passthruArgs(self.data.passthru)
                aero = getattr(adapter, self.cmd())(package)
                if self.cmd() == 'search':
                    self.out.send('Found {:>4} options\n'.format(len(aero)))
                else:
                    self.out.send('\n')
                target.send((1,
                    self.seen(
                        self.cmd(),
                        adapter_name,
                        package,
                        aero
                    ))
                )
            except NotImplementedError:
                target.send((1,
                    ['Aborted: {} has no implementation for command: {}\n'.format(
                        adapter_name, self.cmd()
                    )]
                ))

class DebugCommandProcessor(CommandProcessor):

    def seen(self, command, adapter, package, result=False):
        if not isinstance(result, bool):
            print '\n{:=^100}'.format('')
            print '{:=^100}'.format(' '+command+' '+adapter+' '+package+' ')
            print '{:=^100}'.format('')
            print 'Result type: ', type(result).__name__
            itr = result
            if isinstance(result, dict):
                itr = result.items()
            for r in itr:
                print '{} | {:^5} | {}'.format(type(r).__name__, len(r), r)
            print '\n'
        return result

    @coroutine
    def write(self):
        import sys
        out = sys.stdout
        while True:
            text = (yield)
            out.write(str(text))

    @coroutine
    def progress(self, target):
        while True:
            result = (yield)
            if self.ticker.done() == 100:
                target.send(result)

    def execute(self):
        next = self.wiring()
        self.do(self.data.packages, next)

class SearchCommand(CommandProcessor):

    @coroutine
    def res(self):
        while True:
            res = (yield)
            if res:
                if isinstance(res, list):
                    print '\n' + res[0]
                    continue
                res = sorted(res.items())
                from StringIO import StringIO
                pager = StringIO()
                pager.write(u"\n{:>48}   {:<52}\n".format("PACKAGE NAME", "DESCRIPTION"))
                pager.write(u"{:>48}   {:<52}\n".format("_" * 40, "_" * 50))
                for key, value in res:
                    try:
                        value = value.decode('utf-8')
                        if value.startswith(u'\ufeff'):
                            value = value[len(u'\ufeff'):]
                    except UnicodeDecodeError:
                        value = value.decode('latin1')
                    try:
                        key = key.decode('utf-8')
                        if key.startswith(u'\ufeff'):
                            key = key[len(u'\ufeff'):]
                    except UnicodeDecodeError:
                        key = key.decode('latin1')

                    key = key.encode('utf')
                    for line in value.splitlines():
                        line = line.encode('utf')
                        if key:
                            key += ' :'
                        if len(line) > 50:
                            for wrap in textwrap.wrap(line, 50):
                                pager.write("{:>50} {:<50}\n".format(key, wrap))
                                key = ''
                        else:
                            pager.write("{:>50} {:<50}\n".format(key, line))
                        key = ''
                pager.write('\n')
                from pygments import highlight
                from pygments.lexers import CppLexer
                from pygments.formatters import Terminal256Formatter
                out = pager.getvalue()
                out = highlight(out, CppLexer(), Terminal256Formatter())
                out = out.encode('utf')
                if len(out.splitlines()) > 30:
                    from subprocess import Popen, PIPE
                    Popen(self.data.pager, shell=True, stdin=PIPE).communicate(input=out)
                else:
                    print out

class InstallCommand(CommandProcessor):

    def wiring(self):
        self.out = self.write()
        self.ticker.routine(self.progress(None))
        return self.each(self.spacing(self.call(self.res())))

    def seen(self, command, adapter, package, result=False):
        return result

    @coroutine
    def write(self):
        import sys
        out = sys.stdout
        while True:
            text = (yield)
            out.write(text)

    @coroutine
    def spacing(self, target):
        while True:
            payload = (yield)
            print '\n'
            target.send(payload)

    @coroutine
    def progress(self, responder):
        while True: (yield)

class InfoCommand(CommandProcessor):

    @coroutine
    def res(self):
        while True:
            res = (yield)
            if 'Aborted:' in res[0]:
                print res[0]
                continue
            key = ''
            from cStringIO import StringIO
            pager = StringIO()
            pager.write("\n{:>48}   {:<52}\n".format(
                '',
                'INFORMATION: ' + ', '.join(map(
                    lambda x: x if ':' not in x else x.split(':')[1],
                    self.data.packages
                ))
            ))
            pager.write("{:>47}    {:<52}\n".format("_" * 40, "_" * 50))
            for line in res:
                if isinstance(line, tuple) or isinstance(line, list):
                    if len(line) >= 2:
                        key = line[0] + ': : ' if line[0] else '   '
                        line = line[1]
                    else:
                        line = line[0]
                if line:
                    for l in line.splitlines():
                        if len(l) > 50:
                            for wrap in textwrap.wrap(l, 50):
                                pager.write("{:>50} {:50}\n".format(key, wrap))
                                key = ''
                        else:
                            pager.write("{:>50} {:50}\n".format(key, l))
                            key = ''
            from pygments import highlight
            from pygments.lexers import CppLexer
            from pygments.formatters import Terminal256Formatter
            print highlight(pager.getvalue(), CppLexer(), Terminal256Formatter())

<commit_msg>How do I unicode thee? let me count the ways...<commit_after># -*- coding: utf-8 -*-
from __future__ import division
__author__ = 'nickl-'

import sys
import textwrap

from .__version__ import __version_info__

from .cache import CacheProviderFactory
from .adapters import AVAILABLE_ADAPTERS

def coroutine(func):
    def start(*args,**kwargs):
        cr = func(*args,**kwargs)
        cr.next()
        return cr
    return start

class ProgressTicker():

    ref = None
    result = None
    steps = len(AVAILABLE_ADAPTERS)
    taken = 0

    def routine(self, routine):
        self.ref = routine

    def done(self):
        return (self.taken * 100) // self.steps

    def terminate(self):
        self.taken = self.steps
        try:
            self.ref.send('terminate')
        except StopIteration:
            pass

    def send(self, args):
        if isinstance(args, int):
            self.taken += args
        elif isinstance(args[0], str) and args[0] == 'step':
            self.steps += args[1]
        else:
            self.taken += args[0]
            if isinstance(args[1], dict):
                if not self.result:
                    self.result = {}
                self.result.update(args[1])
            elif isinstance(args[1], list):
                if not self.result:
                    self.result = []
                self.result.extend(args[1])
        self.ref.send(self.result)

class CommandProcessor():

    cache = CacheProviderFactory().whichProvider()
    data = None
    out = None
    ticker = ProgressTicker()
    clear = '\x1b[2J\x1b[0;0H'

    def __init__(self, data):
        self.data = data
        self.out = self.write()
        if self.data.invalidate:
            self.cache.invalidate(True)
        self.out.send(
            self.clear +
            'aero v{}.{}.{} {} {}\n\n'.format(*__version_info__)
        )

    def execute(self):
        try:
            next = self.wiring()
            self.do(self.data.packages, next)
        except BaseException as e:
            self.out.send(
                '\n\n[{}{}] Terminating...\n'.format(
                    type(e).__name__,
                    ': {}'.format(e) if len(str(e)) else ''
                ))
            self.ticker.terminate()

    # wire coroutines
    def wiring(self):
        self.ticker.routine(self.progress(self.res()))
        return self.each(self.call(self.ticker))

    @coroutine
    def res(self):
        while True: (yield)

    def cmd(self):
        return self.__class__.__name__.replace('Command', '').lower()

    def do(self, packages, adapters):
        self.ticker.steps = len(packages) * len(AVAILABLE_ADAPTERS)
        for package in packages:
            if ':' in package:
                mngr = package.partition(':')
                pkg = mngr[2]
                mngr = mngr[0]
            else:
                mngr = None
                pkg = package
            adapters.send((pkg, mngr))

    @coroutine
    def write(self):
        from StringIO import StringIO
        out = StringIO()
        while True:
            text = (yield)
            out.write(str(text))
            print out.getvalue()

    def seen(self, command, adapter, package, result=False):
        cache_key = command + "+" + adapter + ":" + package
        return self.cache.seen(cache_key, result)

    @coroutine
    def each(self, target):
        while True:
            args = (yield)
            package = args[0]
            manager = args[1]
            for adapter in AVAILABLE_ADAPTERS:
                adapter_name = adapter[0]
                if adapter_name not in self.data.disabled:
                    if not manager or manager == adapter_name:
                        length = str(max([len(a[0]) for a in AVAILABLE_ADAPTERS]))
                        self.out.send(
                            (u'Doing an aero {} of package: {} using {:<' + length + '} ').format(
                                self.cmd(), package, adapter_name
                            )
                        )
                        res = self.seen(self.cmd(), adapter_name, package)
                        if self.data.invalidate or res is False:
                            target.send((adapter, package))
                        else:
                            self.out.send(u'Found {:>4} options [CACHED]\n'.format(len(res)))
                            self.ticker.send((1, res))
                    else:
                        self.ticker.send(1)
                else:
                    self.ticker.send(1)

    @coroutine
    def progress(self, target):
        from progbar import ProgBar
        bar = ProgBar(u'Progress: ', 30)
        bar.start()
        while True:
            result = (yield)
            self.out.send('')
            bar.percent = self.ticker.done()
            if self.ticker.done() == 100:
                if bar.isAlive():
                    bar.stop()
                bar.join()
                self.out.send('')
                if not isinstance(result, str): # terminate
                    target.send(result)

    @coroutine
    def call(self, target):
        while True:
            adapter_name = ''
            payload = (yield)
            adapter_name = payload[0][0]
            adapter = payload[0][1]
            package = payload[1]
            try:
                adapter.passthruArgs(self.data.passthru)
                aero = getattr(adapter, self.cmd())(package)
                if self.cmd() == u'search':
                    self.out.send(u'Found {:>4} options\n'.format(len(aero)))
                else:
                    self.out.send(u'\n')
                target.send((1,
                    self.seen(
                        self.cmd(),
                        adapter_name,
                        package,
                        aero
                    ))
                )
            except NotImplementedError:
                target.send((1,
                    [u'Aborted: {} has no implementation for command: {}\n'.format(
                        adapter_name, self.cmd()
                    )]
                ))

class DebugCommandProcessor(CommandProcessor):

    def seen(self, command, adapter, package, result=False):
        if not isinstance(result, bool):
            print u'\n{:=^100}'.format('')
            print u'{:=^100}'.format(' '+command+' '+adapter+' '+package+' ')
            print u'{:=^100}'.format('')
            print u'Result type: ', type(result).__name__
            itr = result
            if isinstance(result, dict):
                itr = result.items()
            for r in itr:
                print u'{} | {:^5} | {}'.format(type(r).__name__, len(r), r)
            print u'\n'
        return result

    @coroutine
    def write(self):
        import sys
        out = sys.stdout
        while True:
            text = (yield)
            out.write(str(text))

    @coroutine
    def progress(self, target):
        while True:
            result = (yield)
            if self.ticker.done() == 100:
                target.send(result)

    def execute(self):
        next = self.wiring()
        self.do(self.data.packages, next)

class SearchCommand(CommandProcessor):

    @coroutine
    def res(self):
        while True:
            res = (yield)
            if res:
                if isinstance(res, list):
                    print u'\n' + res[0]
                    continue
                res = sorted(res.items())
                from StringIO import StringIO
                pager = StringIO()
                pager.write(u"\n{:>48}   {:<52}\n".format(u"PACKAGE NAME", u"DESCRIPTION"))
                pager.write(u"{:>48}   {:<52}\n".format(u"_" * 40, u"_" * 50))
                for key, value in res:
                    for line in value.splitlines():
                        if key:
                            key += u' :'
                        if len(line) > 50:
                            for wrap in textwrap.wrap(line, 50):
                                pager.write(u"{:>50} {:<50}\n".format(key, wrap))
                                key = u''
                        else:
                            pager.write(u"{:>50} {:<50}\n".format(key, line))
                        key = ''
                pager.write(u'\n')
                from pygments import highlight
                from pygments.lexers import CppLexer
                from pygments.formatters import Terminal256Formatter
                out = pager.getvalue()
                out = highlight(out, CppLexer(), Terminal256Formatter())
                out = out.encode('utf')
                if len(out.splitlines()) > 30:
                    from subprocess import Popen, PIPE
                    Popen(self.data.pager, shell=True, stdin=PIPE).communicate(input=out)
                else:
                    print out

class InstallCommand(CommandProcessor):

    def wiring(self):
        self.out = self.write()
        self.ticker.routine(self.progress(None))
        return self.each(self.spacing(self.call(self.res())))

    def seen(self, command, adapter, package, result=False):
        return result

    @coroutine
    def write(self):
        import sys
        out = sys.stdout
        while True:
            text = (yield)
            out.write(text)

    @coroutine
    def spacing(self, target):
        while True:
            payload = (yield)
            print u'\n'
            target.send(payload)

    @coroutine
    def progress(self, responder):
        while True: (yield)

class InfoCommand(CommandProcessor):

    @coroutine
    def res(self):
        while True:
            res = (yield)
            if u'Aborted:' in res[0]:
                print res[0]
                continue
            key = u''
            from cStringIO import StringIO
            pager = StringIO()
            pager.write(u"\n{:>48}   {:<52}\n".format(
                u'',
                u'INFORMATION: ' + ', '.join(map(
                    lambda x: x if u':' not in x else x.split(u':')[1],
                    self.data.packages
                ))
            ))
            pager.write(u"{:>47}    {:<52}\n".format(u"_" * 40, u"_" * 50))
            for line in res:
                if isinstance(line, tuple) or isinstance(line, list):
                    if len(line) >= 2:
                        key = line[0] + u': : ' if line[0] else u'   '
                        line = line[1]
                    else:
                        line = line[0]
                if line:
                    for l in line.splitlines():
                        if len(l) > 50:
                            for wrap in textwrap.wrap(l, 50):
                                pager.write(u"{:>50} {:50}\n".format(key, wrap))
                                key = u''
                        else:
                            pager.write(u"{:>50} {:50}\n".format(key, l))
                            key = u''
            from pygments import highlight
            from pygments.lexers import CppLexer
            from pygments.formatters import Terminal256Formatter
            print highlight(pager.getvalue(), CppLexer(), Terminal256Formatter())

<|endoftext|>


<commit_before>import asyncio
import json
from collections import namedtuple

import asyncio_redis
import websockets
from logbook import Logger

from aesop.utils import setup_logging

log = Logger('aesop.events')

EVENTS_CHANNEL = 'aesop-events'

def add_blocking(func):
    def blocking(*args, **kwargs):
        try:
            loop = asyncio.get_event_loop()
        except Exception:
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)

        coro = func(*args, **kwargs)
        return loop.run_until_complete(coro)
    func.blocking = blocking
    return func

@asyncio.coroutine
@add_blocking
def info(message):
    return notify(message, level='info')

@asyncio.coroutine
@add_blocking
def error(message):
    return notify(message, level='error')

@asyncio.coroutine
@add_blocking
def warning(message):
    return notify(message, level='warning')

@asyncio.coroutine
@add_blocking
def success(message):
    return notify(message, level='success')

@asyncio.coroutine
@add_blocking
def notify(message, **kwargs):
    return broadcast(message=message, type='notification', **kwargs)

@asyncio.coroutine
@add_blocking
def broadcast(type, **kwargs):
    connection = yield from _get_connection()

    kwargs['type'] = type
    yield from connection.publish(EVENTS_CHANNEL, json.dumps(kwargs))

    connection.close()

@asyncio.coroutine
def _get_connection():
    connection = yield from asyncio_redis.Connection.create(host='localhost', port=6379)
    return connection

def listener(*events):
    ev = EventListener(events)
    asyncio.async(ev.start())
    return ev

class EventListener:
    def __init__(self, event_types):
        self.event_types = event_types
        self.waiters = []

    @asyncio.coroutine
    def start(self):
        self.connection = yield from _get_connection()
        self._subscriber = yield from self.connection.start_subscribe()

        yield from self._subscriber.subscribe([EVENTS_CHANNEL])

        yield from self._runner()

    @asyncio.coroutine
    def _runner(self):
        while True:
            event = yield from self._subscriber.next_published()
            event = json.loads(event.value)
            attrs = sorted(set(list(event.keys()) + ['private', 'type']))
            event.setdefault('private', False)
            event = namedtuple('Event', attrs)(**event)

            if self.event_types and event.type not in self.event_types:
                continue

            if event.type == 'shutdown':
                self.connection.close()
                event = None

            waiters, self.waiters = self.waiters, []

            for waiter in waiters:
                waiter.set_result(event)

    @asyncio.coroutine
    def wait(self):
        self.waiters.append(asyncio.Future())
        return self.waiters[-1]

    def close(self):
        self.connection.close()

@asyncio.coroutine
def server(websocket, path):
    client = ':'.join(map(str, websocket.writer.get_extra_info('peername')))
    log.info("New client {}", client)

    yield from broadcast('new-client', private=True)

    _listener = listener()

    while True:
        event = yield from _listener.wait()
        if event is None:
            break
        if event.private:
            continue

        log.info("{} <- {!r}", client, event)

        try:
            yield from websocket.send(json.dumps(event._asdict()))
        except websockets.InvalidState:
            _listener.close()
            return

def main():
    setup_logging('aesop.events', 'INFO')
    loop = asyncio.get_event_loop()
    loop.run_until_complete(websockets.serve(server, '0.0.0.0', 5001))
    log.info("Client event server started on port 5001")
    loop.run_forever()

if __name__ == '__main__':
    main()
<commit_msg>Only need one waiter at a time.<commit_after>import asyncio
import json
from collections import namedtuple

import asyncio_redis
import websockets
from logbook import Logger

from aesop.utils import setup_logging

log = Logger('aesop.events')

EVENTS_CHANNEL = 'aesop-events'

def add_blocking(func):
    def blocking(*args, **kwargs):
        try:
            loop = asyncio.get_event_loop()
        except Exception:
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)

        coro = func(*args, **kwargs)
        return loop.run_until_complete(coro)
    func.blocking = blocking
    return func

@asyncio.coroutine
@add_blocking
def info(message):
    return notify(message, level='info')

@asyncio.coroutine
@add_blocking
def error(message):
    return notify(message, level='error')

@asyncio.coroutine
@add_blocking
def warning(message):
    return notify(message, level='warning')

@asyncio.coroutine
@add_blocking
def success(message):
    return notify(message, level='success')

@asyncio.coroutine
@add_blocking
def notify(message, **kwargs):
    return broadcast(message=message, type='notification', **kwargs)

@asyncio.coroutine
@add_blocking
def broadcast(type, **kwargs):
    connection = yield from _get_connection()

    kwargs['type'] = type
    yield from connection.publish(EVENTS_CHANNEL, json.dumps(kwargs))

    connection.close()

@asyncio.coroutine
def _get_connection():
    connection = yield from asyncio_redis.Connection.create(host='localhost', port=6379)
    return connection

def listener(*events):
    ev = EventListener(events)
    asyncio.async(ev.start())
    return ev

class EventListener:
    def __init__(self, event_types):
        self.event_types = event_types
        self.waiter = None

    @asyncio.coroutine
    def start(self):
        self.connection = yield from _get_connection()
        self._subscriber = yield from self.connection.start_subscribe()

        yield from self._subscriber.subscribe([EVENTS_CHANNEL])

        yield from self._runner()

    @asyncio.coroutine
    def _runner(self):
        while True:
            event = yield from self._subscriber.next_published()
            event = json.loads(event.value)
            attrs = sorted(set(list(event.keys()) + ['private', 'type']))
            event.setdefault('private', False)
            event = namedtuple('Event', attrs)(**event)

            if self.event_types and event.type not in self.event_types:
                continue

            if event.type == 'shutdown':
                self.connection.close()
                event = None

            if self.waiter is not None:
                self.waiter.set_result(event)
                self.waiter = None

    @asyncio.coroutine
    def wait(self):
        if self.waiter is not None:
            raise ValueError("Only one waiter at a time")
        self.waiter = asyncio.Future()
        return self.waiter

    def close(self):
        self.connection.close()

@asyncio.coroutine
def server(websocket, path):
    client = ':'.join(map(str, websocket.writer.get_extra_info('peername')))
    log.info("New client {}", client)

    yield from broadcast('new-client', private=True)

    _listener = listener()

    while True:
        event = yield from _listener.wait()
        if event is None:
            break
        if event.private:
            continue

        log.info("{} <- {!r}", client, event)

        try:
            yield from websocket.send(json.dumps(event._asdict()))
        except websockets.InvalidState:
            _listener.close()
            return

def main():
    setup_logging('aesop.events', 'INFO')
    loop = asyncio.get_event_loop()
    loop.run_until_complete(websockets.serve(server, '0.0.0.0', 5001))
    log.info("Client event server started on port 5001")
    loop.run_forever()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.views.generic import TemplateView
from django.conf.urls.defaults import patterns, url

from crumbs.views import CrumbsMixin

class TestView(CrumbsMixin, TemplateView):
    template_name = 'test.html'

    def get_crumbs(self, context):
        return [('Home', '/'), ('Test', '/test/')]

urlpatterns = patterns('',
    url(r'^test/', TestView.as_view(), name='test_view'),
)
<commit_msg>fix imports to work with django 1.6<commit_after># -*- coding: utf-8 -*-

from django.views.generic import TemplateView
from django.conf.urls import patterns, url

from crumbs.views import CrumbsMixin

class TestView(CrumbsMixin, TemplateView):
    template_name = 'test.html'

    def get_crumbs(self, context):
        return [('Home', '/'), ('Test', '/test/')]

urlpatterns = patterns('',
    url(r'^test/', TestView.as_view(), name='test_view'),
)
<|endoftext|>


<commit_before>from datetime import datetime, timedelta

from django.core.urlresolvers import reverse
from django.db import models
from django.db.models.signals import post_save

# this is the glue to the activity events framework, provided as a no-op here
def issue_update(kind, **kwargs):
    pass

class ForumCategory(models.Model):

    name = models.CharField(max_length=100)
    parent = models.ForeignKey("self", null=True, blank=True, related_name="subcategories")
    # @@@ total descendant forum count?
    # @@@ make group-aware

    class Meta:
        verbose_name_plural = "forum categories"

    def __unicode__(self):
        return self.name

class Forum(models.Model):

    title = models.CharField(max_length=100)
    description = models.TextField()
    creation_date = models.DateTimeField(default=datetime.now)

    # must only have one of these (or neither):
    parent = models.ForeignKey("self", null=True, blank=True, related_name="subforums")
    category = models.ForeignKey(ForumCategory, null=True, blank=True, related_name="forums")

    # @@@ make group-aware

    last_modified = models.DateTimeField(default=datetime.now)
    last_reply = models.ForeignKey("ForumReply", null=True, editable=False)

    view_count = models.IntegerField(default=0, editable=False)
    reply_count = models.IntegerField(default=0, editable=False)

    # this is what gets run normally
    def inc_views(self):
        self.view_count += 1
        self.save()

    # this can be used occasionally to get things back in sync
    def update_view_count(self):
        view_count = 0
        for thread in self.threads.all():
            view_count += thread.view_count
        self.view_count = view_count
        self.save()

    def update_reply_count(self):
        reply_count = 0
        for forum in self.subforums.all():
            forum.update_reply_count()
            reply_count += forum.reply_count
        for thread in self.threads.all():
            thread.update_reply_count()
            reply_count += thread.reply_count
        self.reply_count = reply_count
        self.save()

    def new_reply(self, reply):
        self.reply_count += 1 # if this gets out of sync run update_reply_count
        self.last_modified = reply.timestamp
        self.last_reply = reply
        self.save()
        if self.parent:
            self.parent.new_reply(reply)

    def __unicode__(self):
        return self.title

class ForumThread(models.Model):

    forum = models.ForeignKey(Forum, related_name="threads")

    title = models.CharField(max_length=100)
    creation_date = models.DateTimeField(default=datetime.now)
    author = models.ForeignKey(User, related_name="threads_started")
    last_modified = models.DateTimeField(default=datetime.now)
    last_reply = models.ForeignKey("ForumReply", null=True) # only temporarily null

    # @@@ support markup
    content = models.TextField()

    # @@@ sticky threads
    # @@@ closed threads

    view_count = models.IntegerField(default=0, editable=False)
    reply_count = models.IntegerField(default=0, editable=False)
    subscriber_count = models.IntegerField(default=0, editable=False)

    def inc_views(self):
        self.view_count += 1
        self.save()
        self.forum.inc_views()

    def update_reply_count(self):
        self.reply_count = self.replies.all().count()
        self.save()

    def update_subscriber_count(self):
        self.subscriber_count = self.subscriptions.count()
        self.save()

    def new_reply(self, reply):
        self.reply_count += 1
        self.last_modified = reply.timestamp
        self.last_reply = reply
        self.save()
        self.forum.new_reply(reply)

    def __unicode__(self):
        return self.title

class ForumReply(models.Model):

    thread = models.ForeignKey(ForumThread, related_name="replies")

    author = models.ForeignKey(User, related_name="replies")
    timestamp = models.DateTimeField(default=datetime.now)

    # @@@ support markup
    content = models.TextField()

    # allow editing for short period after posting
    def editable(self, user):
        if user == self.author:
            if datetime.now() < self.timestamp + timedelta(minutes=30): # @@@ factor out time interval
                return True
        return False

class UserPostCount(models.Model):

    user = models.ForeignKey(User, related_name="post_count")
    count = models.IntegerField(default=0)

class ThreadSubscription(models.Model):

    thread = models.ForeignKey(ForumThread, related_name="subscriptions")
    user = models.ForeignKey(User, related_name="forum_subscriptions")

    class Meta:
          unique_together = [("thread", "user")]

def forum_thread_save(sender, instance=None, created=False, **kwargs):
    if instance and created:
        issue_update("forum_thread", user=instance.author, forum_thread=instance)

def forum_reply_save(sender, instance=None, created=False, **kwargs):
    if instance and created:
        thread = instance.thread
        thread.new_reply(instance)
        # @@@ this next part could be manager method
        post_count, created = UserPostCount.objects.get_or_create(user=instance.author)
        post_count.count += 1
        post_count.save()
        issue_update("forum_reply", user=instance.author, forum_reply=instance)
        # @@@ where do subscribers get notified?

def forum_subscription_save(sender, instance=None, created=False, **kwargs):
    if instance and created:
        thread = instance.thread
        thread.update_subscriber_count()

post_save.connect(forum_thread_save, sender=ForumThread)
post_save.connect(forum_reply_save, sender=ForumReply)
post_save.connect(forum_subscription_save, sender=ThreadSubscription)

# @@@ handling unsubscribe?
<commit_msg>added note about need to implement deletion handling<commit_after>from datetime import datetime, timedelta

from django.core.urlresolvers import reverse
from django.db import models
from django.db.models.signals import post_save

# this is the glue to the activity events framework, provided as a no-op here
def issue_update(kind, **kwargs):
    pass

class ForumCategory(models.Model):

    name = models.CharField(max_length=100)
    parent = models.ForeignKey("self", null=True, blank=True, related_name="subcategories")
    # @@@ total descendant forum count?
    # @@@ make group-aware

    class Meta:
        verbose_name_plural = "forum categories"

    def __unicode__(self):
        return self.name

class Forum(models.Model):

    title = models.CharField(max_length=100)
    description = models.TextField()
    creation_date = models.DateTimeField(default=datetime.now)

    # must only have one of these (or neither):
    parent = models.ForeignKey("self", null=True, blank=True, related_name="subforums")
    category = models.ForeignKey(ForumCategory, null=True, blank=True, related_name="forums")

    # @@@ make group-aware

    last_modified = models.DateTimeField(default=datetime.now)
    last_reply = models.ForeignKey("ForumReply", null=True, editable=False)

    view_count = models.IntegerField(default=0, editable=False)
    reply_count = models.IntegerField(default=0, editable=False)

    # this is what gets run normally
    def inc_views(self):
        self.view_count += 1
        self.save()

    # this can be used occasionally to get things back in sync
    def update_view_count(self):
        view_count = 0
        for thread in self.threads.all():
            view_count += thread.view_count
        self.view_count = view_count
        self.save()

    def update_reply_count(self):
        reply_count = 0
        for forum in self.subforums.all():
            forum.update_reply_count()
            reply_count += forum.reply_count
        for thread in self.threads.all():
            thread.update_reply_count()
            reply_count += thread.reply_count
        self.reply_count = reply_count
        self.save()

    def new_reply(self, reply):
        self.reply_count += 1 # if this gets out of sync run update_reply_count
        self.last_modified = reply.timestamp
        self.last_reply = reply
        self.save()
        if self.parent:
            self.parent.new_reply(reply)

    def __unicode__(self):
        return self.title

class ForumThread(models.Model):

    forum = models.ForeignKey(Forum, related_name="threads")

    title = models.CharField(max_length=100)
    creation_date = models.DateTimeField(default=datetime.now)
    author = models.ForeignKey(User, related_name="threads_started")
    last_modified = models.DateTimeField(default=datetime.now)
    last_reply = models.ForeignKey("ForumReply", null=True) # only temporarily null

    # @@@ support markup
    content = models.TextField()

    # @@@ sticky threads
    # @@@ closed threads

    view_count = models.IntegerField(default=0, editable=False)
    reply_count = models.IntegerField(default=0, editable=False)
    subscriber_count = models.IntegerField(default=0, editable=False)

    def inc_views(self):
        self.view_count += 1
        self.save()
        self.forum.inc_views()

    def update_reply_count(self):
        self.reply_count = self.replies.all().count()
        self.save()

    def update_subscriber_count(self):
        self.subscriber_count = self.subscriptions.count()
        self.save()

    def new_reply(self, reply):
        self.reply_count += 1
        self.last_modified = reply.timestamp
        self.last_reply = reply
        self.save()
        self.forum.new_reply(reply)

    def __unicode__(self):
        return self.title

class ForumReply(models.Model):

    thread = models.ForeignKey(ForumThread, related_name="replies")

    author = models.ForeignKey(User, related_name="replies")
    timestamp = models.DateTimeField(default=datetime.now)

    # @@@ support markup
    content = models.TextField()

    # allow editing for short period after posting
    def editable(self, user):
        if user == self.author:
            if datetime.now() < self.timestamp + timedelta(minutes=30): # @@@ factor out time interval
                return True
        return False

class UserPostCount(models.Model):

    user = models.ForeignKey(User, related_name="post_count")
    count = models.IntegerField(default=0)

class ThreadSubscription(models.Model):

    thread = models.ForeignKey(ForumThread, related_name="subscriptions")
    user = models.ForeignKey(User, related_name="forum_subscriptions")

    class Meta:
          unique_together = [("thread", "user")]

def forum_thread_save(sender, instance=None, created=False, **kwargs):
    if instance and created:
        issue_update("forum_thread", user=instance.author, forum_thread=instance)

def forum_reply_save(sender, instance=None, created=False, **kwargs):
    if instance and created:
        thread = instance.thread
        thread.new_reply(instance)
        # @@@ this next part could be manager method
        post_count, created = UserPostCount.objects.get_or_create(user=instance.author)
        post_count.count += 1
        post_count.save()
        issue_update("forum_reply", user=instance.author, forum_reply=instance)
        # @@@ where do subscribers get notified?

def forum_subscription_save(sender, instance=None, created=False, **kwargs):
    if instance and created:
        thread = instance.thread
        thread.update_subscriber_count()

post_save.connect(forum_thread_save, sender=ForumThread)
post_save.connect(forum_reply_save, sender=ForumReply)
post_save.connect(forum_subscription_save, sender=ThreadSubscription)

# @@@ handling unsubscribe?

# @@@ handling deletion? (e.g. counts, last_modified, last_reply)
<|endoftext|>


<commit_before>import os
import re
import tables as tab
import numpy as np
import scipy as sp
from scipy.optimize import fmin_bfgs

class Run():
    '''
    The fundamental class for a run.

    '''
    def __init__(self, runid, datafile):
        '''Loads all parameters if available otherwise it generates them.

        Parameters
        ----------
        runid : int or string
            The run id: 5 or '00005'.
        datafile : pytable object of an hdf5 file
            This file must contain the run data table and the calibration data
            table.

        '''

        # get the data table
        datatable = datafile.root.data.datatable

        self.d = {}

        for col in datatable.colnames:
            coldata = datatable[int(runid)][col]
            try:
                if len(coldata) == 12000:
                    numsamp = datatable[int(runid)]['NINumSamples']
                    coldata = unsize_vector(coldata, numsamp)
            except:
                pass
            self.d[col] = coldata

def sync_error(tau,s1,s2,t):
    '''
    Returns the error between two signals.

    Parameters
    ----------
    tau :
    s1 :
    s2 :
    t :

    Returns
    -------
    e :

    '''
    N = t.shape[0]
    t1_interp = np.linspace(np.min(t) + np.abs(tau),
                            np.max(t) - np.abs(tau),
                            N)
    t2_interp = t1_interp - tau
    s1_interp = sp.interp(t1_interp, t, s1);
    s2_interp = sp.interp(t2_interp,
                          t[~np.isnan(s2)],
                          s2[~np.isnan(s2)]);
    e  = sum((s1_interp[:round(.2*N)]-s2_interp[:round(.2*N)])**2)
    return e

def find_timeshift(s1, s2, Fs):
    '''
    Returns the timeshift (tau) of the VectorNav (VN) data relative to the
    National Instruments (NI) data based on the first 20% of the data.

    Parameters
    ----------
    s1 : ndarray, shape(n, )
        Vertical acceleration data of the NI accelerometer
    s2 : ndarray, shape(n, )
        Vertical acceleration data of the VN accelerometer

    Returns
    -------
    tau : array
        Timeshift relative to the NI signals

    '''

    N = s1.shape[0]
    t = np.arange(0, N)/Fs

    # Error Landscape
    tau_range = np.linspace(-1, 1, 201)
    e = np.zeros(tau_range.shape)
    for ran in tau_range:
        e[i] = sync_error(ran, s1, s2, t)

    # Find initial condition from landscape and optimize!
    tau0 = tau_range[np.argmin(e)]
    tau  = fmin_bfgs(sync_error, tau0, args=(s1, s2, t))

    return tau

def unsize_vector(vector, m):
    '''Returns a vector with the nan padding removed.

    Parameters
    ----------
    vector : numpy array, shape(n, )
        A vector that may or may not have nan padding and the end of the data.
    m : int
        Number of valid values in the vector.

    Returns
    -------
    numpy array, shape(m, )
        The vector with the padding removed. m = samplenum

    '''
    # this case removes the nan padding
    if m < len(vector):
        oldvec = vector[:m]
    elif m > len(vector):
        oldvec = vector
        print('This one is actually longer, you may want to get the ' +
              'complete data, or improve this function so it does that.')
    elif m == len(vector):
        oldvec = vector
    else:
        print "Something's wrong here"
    return oldvec

def size_vector(vector, m):
    '''Returns a vector with nan's padded on to the end or a slice of the
    vector if length is less than the length of the vector.

    Parameters
    ----------
    vector : numpy array, shape(n, )
        The vector that needs sizing.
    m : int
        The desired length after the sizing.

    Returns
    -------
    newvec : numpy array, shape(m, )

    '''
    nsamp = len(vector)
    # if the desired length is larger then pad witn nan's
    if m > nsamp:
        nans = np.ones(m-nsamp)*np.nan
        newvec = np.append(vector, nans)
    elif m < nsamp:
        newvec = vector[:m]
    elif m == nsamp:
        newvec = vector
    else:
        print "This didn't work"
    return newvec

def fill_table(datafile):
    '''Adds all the data from the hdf5 files in the h5 directory to the table.

    Parameters
    ----------
    datafile : string
        path to the main hdf5 file
    '''

    # load the files from the ../BicycleDAQ/data/h5 directory
    pathtoh5 = os.path.join('..', 'BicycleDAQ', 'data', 'h5')
    files = sorted(os.listdir(pathtoh5))
    # open an hdf5 file for appending
    data = tab.openFile(datafile, mode='a')
    # get the table
    datatable = data.root.data.datatable
    # get the row
    row = datatable.row
    # fill the rows with data
    for run in files:
        print 'Adding run: %s' % run
        rundata = get_run_data(os.path.join(pathtoh5, run))
        for par, val in rundata['par'].items():
            row[par] = val
        # only take the first 12000 samples for all runs
        for i, col in enumerate(rundata['NICols']):
            try: # there are no roll pot measurements
                row[col] = size_vector(rundata['NIData'][i], 12000)
            except:
                print "There is no %s measurement" % col
        for i, col in enumerate(rundata['VNavCols']):
            row[col] = size_vector(rundata['VNavData'][i], 12000)
        row.append()
    datatable.flush()
    data.close()

def create_database():
    '''Creates an HDF5 file for data collected from the instrumented bicycle'''

    # load the latest file in the ../BicycleDAQ/data/h5 directory
    pathtoh5 = os.path.join('..', 'BicycleDAQ', 'data', 'h5')
    files = sorted(os.listdir(pathtoh5))
    filteredrun = get_run_data(os.path.join(pathtoh5, files[0]))
    unfilteredrun = get_run_data(os.path.join(pathtoh5, files[-1]))
    if filteredrun['par']['ADOT'] is not 14:
        print('Run %d is not a filtered run, choose again' %
              filteredrun['par']['RunID'])
    if unfilteredrun['par']['ADOT'] is not 253:
        print('Run %d is not a unfiltered run, choose again' %
              unfilteredrun['par']['RunID'])
    # generate the table description class
    RunTable = create_run_table_class(filteredrun, unfilteredrun)
    # open a new hdf5 file for writing
    data = tab.openFile('InstrumentedBicycleData.h5', mode='w',
                        title='Instrumented Bicycle Data')
    # create a group for the raw data
    rgroup = data.createGroup('/', 'data', 'Data')
    # add the data table to this group
    rtable = data.createTable(rgroup, 'datatable', RunTable, 'Primary Data Table')
    rtable.flush()
    data.close()

def create_run_table_class(filteredrun, unfilteredrun):
    '''Generates a class that is used for the table description for raw data
    for each run.

    Parameters
    ----------
    rundata : dict
        Contains the python dictionary of a particular run.

    Returns
    -------
    Run : class
        Table description class for pytables with columns defined.

    '''

    # combine the VNavCols from unfiltered and filtered
    VNavCols = set(filteredrun['VNavCols'] + unfilteredrun['VNavCols'])

    # set up the table description
    class RunTable(tab.IsDescription):
        # add all of the column headings from par, NICols and VNavCols
        for i, col in enumerate(unfilteredrun['NICols']):
            exec(col + " = tab.Float32Col(shape=(12000, ), pos=i)")
        for k, col in enumerate(VNavCols):
            exec(col + " = tab.Float32Col(shape=(12000, ), pos=i+1+k)")
        for i, (key, val) in enumerate(unfilteredrun['par'].items()):
            pos = k+1+i
            if isinstance(val, type(1)):
                exec(key + " = tab.Int64Col(pos=pos)")
            elif isinstance(val, type('')):
                exec(key + " = tab.StringCol(itemsize=50, pos=pos)")
            elif isinstance(val, type(1.)):
                exec(key + " = tab.Float64Col(pos=pos)")
            elif isinstance(val, type(np.ones(1))):
                exec(key + " = tab.Float64Col(shape=(" + str(len(val)) + ", ), pos=pos)")

        # add the columns for the processed data
        processedCols = ['SteerAngle', 'SteerRate', 'RollAngle', 'RollRate',
                         'RearWheelRate', 'SteerTorque', 'YawRate',
                         'PitchRate', 'FrameAccelerationX',
                         'FrameAccelerationY', 'FrameAccelerationZ', 'tau']
        for k, col in enumerate(processedCols):
            exec(col + " = tab.Float32Col(shape=(12000, ), pos=i+1+k)")

        # get rid intermediate variables so they are not stored in the class
        del(i, k, col, key, pos, val, processedCols)

    return RunTable

def parse_vnav_string(vnstr, remove=0):
    '''Gets the good info from a VNav string

    Parameters
    ----------
    vnstr : string
        A string from the VectorNav serial output.
    remove : int
        Specifies how many values to remove from the beginning of the output
        list. Useful for removing VNWRG, etc.

    Returns
    -------
    vnlist : list
        A list of each element in the VectorNav string.
        ['VNWRG', '26', ..., ..., ...]

    '''
    # get rid of the $ and the *checksum
    vnstr = re.sub('\$(.*)\*.*', r'\1', vnstr)
    # make it a list
    vnlist = vnstr.split(',')
    # return the last values with regards to remove
    return vnlist[remove:]

def get_run_data(pathtofile):
    '''Returns data from the run h5 files using pytables and formats it better
    for python.

    Parameters
    ----------
    pathtofile : string
        The path to the h5 file that contains run data.

    Returns
    -------
    rundata : dictionary
        A dictionary that looks similar to how the data was stored in Matlab.

    '''

    # open the file
    runfile = tab.openFile(pathtofile)

    # intialize a dictionary for storage
    rundata = {}

    # put the parameters into a dictionary
    rundata['par'] = {}
    for col in runfile.root.par:
        # convert them to regular python types
        try:
            if col.name == 'Speed':
                rundata['par'][col.name] = float(col.read()[0])
            else:
                rundata['par'][col.name] = int(col.read()[0])
        except:
            pstr = str(col.read()[0])
            rundata['par'][col.name] = pstr
            if pstr[0] == '$':
                parsed = parse_vnav_string(pstr, remove=2)
                if len(parsed) == 1:
                    try:
                        parsed = int(parsed[0])
                    except:
                        parsed = parsed[0]
                else:
                    parsed = np.array([float(x) for x in parsed])
                rundata['par'][col.name] = parsed

    # get the NIData and VNavData
    rundata['NIData'] = runfile.root.NIData.read()
    rundata['VNavData'] = runfile.root.VNavData.read()

    # make the array into a list of python strings
    columns = [str(x) for x in runfile.root.VNavCols.read()]
    # gets rid of unescaped control characters
    columns = [re.sub(r'[^ -~].*', '', x) for x in columns]
    # gets rid of white space
    rundata['VNavCols'] = [x.replace(' ', '') for x in columns]
    # make a list of NI columns from the InputPair structure from matlab
    rundata['NICols'] = []
    for col in runfile.root.InputPairs:
        rundata['NICols'].append((str(col.name), int(col.read()[0])))

    rundata['NICols'].sort(key=lambda x: x[1])

    rundata['NICols'] = [x[0] for x in rundata['NICols']]

    # get the VNavDataText
    rundata['VNavDataText'] = [str(x) for x in runfile.root.VNavDataText.read()]

    # close the file
    runfile.close()

    return rundata
<commit_msg>the find timeshift is in place, not sure if it is working<commit_after>import os
import re
import tables as tab
import numpy as np
import scipy as sp
from scipy.optimize import fmin_bfgs

class Run():
    '''
    The fundamental class for a run.

    '''
    def __init__(self, runid, datafile):
        '''Loads all parameters if available otherwise it generates them.

        Parameters
        ----------
        runid : int or string
            The run id: 5 or '00005'.
        datafile : pytable object of an hdf5 file
            This file must contain the run data table and the calibration data
            table.

        '''

        # get the data table
        datatable = datafile.root.data.datatable

        # these are the columns that may need to be calculated
        processedCols = ['SteerAngle', 'SteerRate', 'RollAngle', 'RollRate',
                         'RearWheelRate', 'SteerTorque', 'YawRate',
                         'PitchRate', 'FrameAccelerationX',
                         'FrameAccelerationY', 'FrameAccelerationZ', 'tau']

        # make a container for the data
        self.data = {}

        # get the row number for this particular run id
        rownum = [x.nrow for x in datatable.iterrows() if x['RunID'] ==
                  int(runid)][0]
        print rownum

        # if the data hasn't been time shifted, then do it now
        curtau = datatable[rownum]['tau']
        print 'curtau', curtau
        if curtau == 0.:
            NIacc = datatable[rownum]['FrameAccelY']
            VNacc = datatable[rownum]['AccelerationZ']
            Fs = datatable[rownum]['NISampleRate']
            tau = find_timeshift(NIacc, VNacc, Fs)
            print 'This is tau', tau

        for col in datatable.colnames:
            # grab the data for the run number and column name
            coldata = datatable[rownum][col]
            # figure out if the object is one of the data arrays so we can
            # unsize it
            if isinstance(coldata, type(np.ones(1))) and coldata.shape[0] == 12000:
                numsamp = datatable[int(runid)]['NINumSamples']
                coldata = unsize_vector(coldata, numsamp)
                # now check to see if we need to process the data
                if np.sum(coldata) == 0. and col in processedCols:
                    print col, 'needs some processing'

            self.data[col] = coldata

def sync_error(tau, s1, s2, t):
    '''
    Returns the error between two signals.

    Parameters
    ----------
    tau : float
        Guess for the time shift
    s1 : ndarray, shape(n, )
        Vertical acceleration data of the NI accelerometer
    s2 : ndarray, shape(n, )
        Vertical acceleration data of the VN accelerometer
    t : ndarray
        Time

    Returns
    -------
    e : float
        Error

    '''
    N = t.shape[0]
    t1_interp = np.linspace(np.min(t) + np.abs(tau),
                            np.max(t) - np.abs(tau),
                            N)
    t2_interp = t1_interp - tau
    s1_interp = sp.interp(t1_interp, t, s1);
    s2_interp = sp.interp(t2_interp,
                          t[~np.isnan(s2)],
                          s2[~np.isnan(s2)]);
    e  = sum((s1_interp[:round(.2*N)]-s2_interp[:round(.2*N)])**2)
    return e

def find_timeshift(NIacc, VNacc, Fs):
    '''
    Returns the timeshift (tau) of the VectorNav (VN) data relative to the
    National Instruments (NI) data based on the first 20% of the data.

    Parameters
    ----------
    NIacc : ndarray, shape(n, )
       The (mostly) vertical acceleration from the NI box.
    VNacc : ndarray, shape(n, )
        The (mostly) vertical acceleration from the VectorNav.
    Fs : int
        Sample rate of the signals. This should be the same for each signal.

    Returns
    -------
    tau : array
        Timeshift relative to the NI signals

    '''

    s1_raw = -(NIacc - 1.5) / (300. / 1000.) * 9.81
    s2_raw = VNacc
    s1 = s1_raw - np.mean(s1_raw)
    s2 = s2_raw - np.mean(s2_raw[~np.isnan(s2_raw)])

    N = s1.shape[0]
    t = np.arange(0, N)/Fs

    # Error Landscape
    tau_range = np.linspace(-1, 1, 201)
    e = np.zeros(tau_range.shape)
    for i, ran in enumerate(tau_range):
        e[i] = sync_error(ran, s1, s2, t)

    # Find initial condition from landscape and optimize!
    tau0 = tau_range[np.argmin(e)]
    tau  = fmin_bfgs(sync_error, tau0, args=(s1, s2, t))

    return tau

def unsize_vector(vector, m):
    '''Returns a vector with the nan padding removed.

    Parameters
    ----------
    vector : numpy array, shape(n, )
        A vector that may or may not have nan padding and the end of the data.
    m : int
        Number of valid values in the vector.

    Returns
    -------
    numpy array, shape(m, )
        The vector with the padding removed. m = samplenum

    '''
    # this case removes the nan padding
    if m < len(vector):
        oldvec = vector[:m]
    elif m > len(vector):
        oldvec = vector
        print('This one is actually longer, you may want to get the ' +
              'complete data, or improve this function so it does that.')
    elif m == len(vector):
        oldvec = vector
    else:
        print "Something's wrong here"
    return oldvec

def size_vector(vector, m):
    '''Returns a vector with nan's padded on to the end or a slice of the
    vector if length is less than the length of the vector.

    Parameters
    ----------
    vector : numpy array, shape(n, )
        The vector that needs sizing.
    m : int
        The desired length after the sizing.

    Returns
    -------
    newvec : numpy array, shape(m, )

    '''
    nsamp = len(vector)
    # if the desired length is larger then pad witn nan's
    if m > nsamp:
        nans = np.ones(m-nsamp)*np.nan
        newvec = np.append(vector, nans)
    elif m < nsamp:
        newvec = vector[:m]
    elif m == nsamp:
        newvec = vector
    else:
        print "This didn't work"
    return newvec

def fill_table(datafile):
    '''Adds all the data from the hdf5 files in the h5 directory to the table.

    Parameters
    ----------
    datafile : string
        path to the main hdf5 file
    '''

    # load the files from the ../BicycleDAQ/data/h5 directory
    pathtoh5 = os.path.join('..', 'BicycleDAQ', 'data', 'h5')
    files = sorted(os.listdir(pathtoh5))
    # open an hdf5 file for appending
    data = tab.openFile(datafile, mode='a')
    # get the table
    datatable = data.root.data.datatable
    # get the row
    row = datatable.row
    # fill the rows with data
    for run in files:
        print 'Adding run: %s' % run
        rundata = get_run_data(os.path.join(pathtoh5, run))
        for par, val in rundata['par'].items():
            row[par] = val
        # only take the first 12000 samples for all runs
        for i, col in enumerate(rundata['NICols']):
            try: # there are no roll pot measurements
                row[col] = size_vector(rundata['NIData'][i], 12000)
            except:
                print "There is no %s measurement" % col
        for i, col in enumerate(rundata['VNavCols']):
            row[col] = size_vector(rundata['VNavData'][i], 12000)
        row.append()
    datatable.flush()
    data.close()

def create_database():
    '''Creates an HDF5 file for data collected from the instrumented bicycle'''

    # load the latest file in the ../BicycleDAQ/data/h5 directory
    pathtoh5 = os.path.join('..', 'BicycleDAQ', 'data', 'h5')
    files = sorted(os.listdir(pathtoh5))
    filteredrun = get_run_data(os.path.join(pathtoh5, files[0]))
    unfilteredrun = get_run_data(os.path.join(pathtoh5, files[-1]))
    if filteredrun['par']['ADOT'] is not 14:
        print('Run %d is not a filtered run, choose again' %
              filteredrun['par']['RunID'])
    if unfilteredrun['par']['ADOT'] is not 253:
        print('Run %d is not a unfiltered run, choose again' %
              unfilteredrun['par']['RunID'])
    # generate the table description class
    RunTable = create_run_table_class(filteredrun, unfilteredrun)
    # open a new hdf5 file for writing
    data = tab.openFile('InstrumentedBicycleData.h5', mode='w',
                        title='Instrumented Bicycle Data')
    # create a group for the raw data
    rgroup = data.createGroup('/', 'data', 'Data')
    # add the data table to this group
    rtable = data.createTable(rgroup, 'datatable', RunTable, 'Primary Data Table')
    rtable.flush()
    data.close()

def create_run_table_class(filteredrun, unfilteredrun):
    '''Generates a class that is used for the table description for raw data
    for each run.

    Parameters
    ----------
    rundata : dict
        Contains the python dictionary of a particular run.

    Returns
    -------
    Run : class
        Table description class for pytables with columns defined.

    '''

    # combine the VNavCols from unfiltered and filtered
    VNavCols = set(filteredrun['VNavCols'] + unfilteredrun['VNavCols'])

    # set up the table description
    class RunTable(tab.IsDescription):
        # add all of the column headings from par, NICols and VNavCols
        for i, col in enumerate(unfilteredrun['NICols']):
            exec(col + " = tab.Float32Col(shape=(12000, ), pos=i)")
        for k, col in enumerate(VNavCols):
            exec(col + " = tab.Float32Col(shape=(12000, ), pos=i+1+k)")
        for i, (key, val) in enumerate(unfilteredrun['par'].items()):
            pos = k+1+i
            if isinstance(val, type(1)):
                exec(key + " = tab.Int64Col(pos=pos)")
            elif isinstance(val, type('')):
                exec(key + " = tab.StringCol(itemsize=50, pos=pos)")
            elif isinstance(val, type(1.)):
                exec(key + " = tab.Float64Col(pos=pos)")
            elif isinstance(val, type(np.ones(1))):
                exec(key + " = tab.Float64Col(shape=(" + str(len(val)) + ", ), pos=pos)")

        # add the columns for the processed data
        processedCols = ['SteerAngle', 'SteerRate', 'RollAngle', 'RollRate',
                         'RearWheelRate', 'SteerTorque', 'YawRate',
                         'PitchRate', 'FrameAccelerationX',
                         'FrameAccelerationY', 'FrameAccelerationZ', 'tau']
        for k, col in enumerate(processedCols):
            if col == 'tau':
                exec(col + " = tab.Float32Col(pos=i+1+k)")
            else:
                exec(col + " = tab.Float32Col(shape=(12000, ), pos=i+1+k)")

        # get rid intermediate variables so they are not stored in the class
        del(i, k, col, key, pos, val, processedCols)

    return RunTable

def parse_vnav_string(vnstr, remove=0):
    '''Gets the good info from a VNav string

    Parameters
    ----------
    vnstr : string
        A string from the VectorNav serial output.
    remove : int
        Specifies how many values to remove from the beginning of the output
        list. Useful for removing VNWRG, etc.

    Returns
    -------
    vnlist : list
        A list of each element in the VectorNav string.
        ['VNWRG', '26', ..., ..., ...]

    '''
    # get rid of the $ and the *checksum
    vnstr = re.sub('\$(.*)\*.*', r'\1', vnstr)
    # make it a list
    vnlist = vnstr.split(',')
    # return the last values with regards to remove
    return vnlist[remove:]

def get_run_data(pathtofile):
    '''Returns data from the run h5 files using pytables and formats it better
    for python.

    Parameters
    ----------
    pathtofile : string
        The path to the h5 file that contains run data.

    Returns
    -------
    rundata : dictionary
        A dictionary that looks similar to how the data was stored in Matlab.

    '''

    # open the file
    runfile = tab.openFile(pathtofile)

    # intialize a dictionary for storage
    rundata = {}

    # put the parameters into a dictionary
    rundata['par'] = {}
    for col in runfile.root.par:
        # convert them to regular python types
        try:
            if col.name == 'Speed':
                rundata['par'][col.name] = float(col.read()[0])
            else:
                rundata['par'][col.name] = int(col.read()[0])
        except:
            pstr = str(col.read()[0])
            rundata['par'][col.name] = pstr
            if pstr[0] == '$':
                parsed = parse_vnav_string(pstr, remove=2)
                if len(parsed) == 1:
                    try:
                        parsed = int(parsed[0])
                    except:
                        parsed = parsed[0]
                else:
                    parsed = np.array([float(x) for x in parsed])
                rundata['par'][col.name] = parsed

    # get the NIData and VNavData
    rundata['NIData'] = runfile.root.NIData.read()
    rundata['VNavData'] = runfile.root.VNavData.read()

    # make the array into a list of python strings
    columns = [str(x) for x in runfile.root.VNavCols.read()]
    # gets rid of unescaped control characters
    columns = [re.sub(r'[^ -~].*', '', x) for x in columns]
    # gets rid of white space
    rundata['VNavCols'] = [x.replace(' ', '') for x in columns]
    # make a list of NI columns from the InputPair structure from matlab
    rundata['NICols'] = []
    for col in runfile.root.InputPairs:
        rundata['NICols'].append((str(col.name), int(col.read()[0])))

    rundata['NICols'].sort(key=lambda x: x[1])

    rundata['NICols'] = [x[0] for x in rundata['NICols']]

    # get the VNavDataText
    rundata['VNavDataText'] = [str(x) for x in runfile.root.VNavDataText.read()]

    # close the file
    runfile.close()

    return rundata
<|endoftext|>