<commit_before>from tasks.util import (Shp2TempTableTask, TempTableTask, TableTask, TagsTask, ColumnsTask,
                        DownloadUnzipTask, CSV2TempTableTask,
                        underscore_slugify, shell, classpath)
from tasks.meta import current_session, DENOMINATOR, GEOM_REF, UNIVERSE
from collections import OrderedDict
from luigi import IntParameter, Parameter, WrapperTask, Task, LocalTarget, ListParameter
import os
from tasks.meta import OBSTable, OBSColumn, OBSTag
from tasks.tags import SectionTags, SubsectionTags, UnitTags
from tasks.fr.communes import OutputCommuneColumns
import csv
import pandas as pd
import itertools
#
# dl_code_list = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&downfile=dic%2Fen%2F{code}.dic".format(code=code)
# flag_explanation = "http://ec.europa.eu/eurostat/data/database/information"
# database = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?dir=data&sort=1&sort=2&start={}".format(first_letter)
# dl_data = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&downfile=data%2F{}.tsv.gz".format(table_code)
# dl_data = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&file=data%2Fdemo_r_pjangrp3.tsv.gz

class DownloadEurostat(Task):

    table_code =  Parameter()
    URL = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&downfile=data%2F{}.tsv.gz".format(table_code)

    def download(self):
        shell('wget -O {output}.gz "{url}"'.format(
        output=self.output().path,
        url=self.URL))

    def run(self):
        self.output().makedirs()
        self.download()
        shell('gunzip {output}.gz'.format(output=self.output().path))

    def output(self):
        return LocalTarget(os.path.join('tmp', classpath(self), self.task_id))

class EUTempTable(CSV2TempTableTask):
    delimiter = Parameter(default='\t', significant=False)
    def version(self):
        return 2

    def requires(self):
        return DownloadEurostat()

    def input_csv(self):
        shell("cat {} | tr ',' '\t' > {}.csv".format(self.input().path,self.input().path))
        return self.input().path + '.csv'

class EUFormatTable(TempTableTask):

    def requires(self):
        return EUTempTable()

    def run(self):
        pass

class FlexEurostatColumns(ColumnsTask):

    subsection = Parameter()
    tablename = Parameter()  # Ex. "DEMO_R_PJANAGGR3"

    # From tablename, determine basis of name for columns from table_dic.dic
    # Then, look at metabase.txt to find relevant dimensions (exclude "geo" and "time", utilize "unit")
    # Finally, look up definitions for dimensions from their .dic files, and use that to complete the metadata definition

    def requires(self):
        return {
            'units': UnitTags(),
            'subsection': SubsectionTags(),
            'section': SectionTags()
        }

    def columns(self):
        columns = OrderedDict()

        input_ = self.input()

        subsectiontags = input_['subsection']
        unittags = input_['units'] #???
        eu = input_['section']['eu']

        tags=[eu,subsectiontags[self.subsection]]

        current_code = None
        codes = {}
        current_list = []
        with open(os.path.join(os.path.dirname(__file__),'metabase.txt')) as metabase:
            reader = csv.reader(metabase,delimiter='\t')
            for possible_tablenames, code, code_value in reader:
                if self.tablename.lower() == possible_tablenames.lower():
                    if code != "geo" and code != "unit" and code != "time":
                        if current_code == code:
                            current_list.append(code_value)
                        elif current_code:
                            codes[current_code] = current_list
                            current_list = []
                            current_list.append(code_value)
                        else:
                            current_list.append(code_value)
                        current_code = code
                    codes[current_code] = current_list

        product = [x for x in apply(itertools.product, codes.values())]
        cross_prod = [dict(zip(codes.keys(), p)) for p in product]

        with open(os.path.join(os.path.dirname(__file__),'table_dic.dic')) as tabledicfile:
            tablereader = csv.reader(tabledicfile,delimiter='\t')
            for possible_tablenames, table_description in tablereader:
                if self.tablename.lower() == possible_tablenames.lower():
                    table_desc = table_description
                    variable_name = table_description.split('by')[0].strip()
                    break

        if cross_prod:
            for i in cross_prod:
                var_code = underscore_slugify(variable_name+"_".join(i.values()))
                dimdefs = []
                for dimname, value in i.iteritems():
                    with open(os.path.join(os.path.dirname(__file__),'dic_lists/{dimension}.dic'.format(dimension=dimname))) as dimfile:
                        reader = csv.reader(dimfile, delimiter='\t')
                        for possible_dimvalue, dimdef in reader:
                            if value == possible_dimvalue:
                                dimdefs.append(dimdef)
                description = "{} ".format(variable_name)+", ".join([str(x) for x in dimdefs])

                columns[var_code] = OBSColumn(
                    id=var_code,
                    name=description,
                    type='Numeric',
                    description=table_desc,
                    weight=1,
                    aggregate=None, #???
                    targets={}, #???
                    tags=tags,
                )
        else:
            var_code = underscore_slugify(variable_name)
            description = variable_name
            columns[var_code] = OBSColumn(
                id=var_code,
                name=variable_name,
                type='Numeric',
                description= table_desc,
                weight=1,
                aggregate=None, #???
                targets={}, #???
                tags=tags,
            )

        return columns

class FranceCensus(TableTask):

    table_name = Parameter()
    subsection = Parameter()

    def version(self):
        return 8

    def timespan(self):
        return '2012'

    def requires(self):
        requirements = {
            'data': RawFRData(table_name=self.table_name),
=            'meta': FlexEurostatColumns(table_name=self.table_theme, subsection = self.subsection),
            'geometa': OutputAreaColumns(),
        }
        return requirements

    def columns(self):
        cols = OrderedDict()
        cols['IRIS'] = self.input()['geometa']['dcomiris']
        cols.update(self.input()['meta'])
        return cols

    def populate(self):
        session = current_session()

        column_targets = self.columns()
        colnames = ', '.join(column_targets.keys())
        colnames_typed = ','.join(['{}::{}'.format(colname, ct.get(session).type)
                              for colname, ct in column_targets.iteritems()])
        session.execute('INSERT INTO {output} ({ids}) '
                        'SELECT {ids_typed} '
                        'FROM {input} '.format(
                            ids=colnames,
                            ids_typed=colnames_typed,
                            output=self.output().table,
                            input=self.input()['iris_data'].table
                        ))
        session.execute('INSERT INTO {output} ({ids}) '
                        'SELECT {ids_typed} '
                        'FROM {input} '.format(
                            ids=colnames,
                            ids_typed=colnames_typed,
                            output=self.output().table,
                            input=self.input()['overseas_data'].table
                        ))
<commit_msg>TableTask not functioning<commit_after>from tasks.util import (Shp2TempTableTask, TempTableTask, TableTask, TagsTask, ColumnsTask,
                        DownloadUnzipTask, CSV2TempTableTask,
                        underscore_slugify, shell, classpath)
from tasks.meta import current_session, DENOMINATOR, GEOM_REF, UNIVERSE
from collections import OrderedDict
from luigi import IntParameter, Parameter, WrapperTask, Task, LocalTarget, ListParameter
import os
from tasks.meta import OBSTable, OBSColumn, OBSTag
from tasks.tags import SectionTags, SubsectionTags, UnitTags
from tasks.fr.communes import OutputCommuneColumns
import csv
import pandas as pd
import itertools
#
# dl_code_list = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&downfile=dic%2Fen%2F{code}.dic".format(code=code)
# flag_explanation = "http://ec.europa.eu/eurostat/data/database/information"
# database = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?dir=data&sort=1&sort=2&start={}".format(first_letter)
# dl_data = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&downfile=data%2F{}.tsv.gz".format(table_code)
# dl_data = "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&file=data%2Fdemo_r_pjangrp3.tsv.gz

class DownloadEurostat(Task):

    table_code =  Parameter()
    URL="http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&file=data%2F{code}.tsv.gz"

    def download(self):
        URL = self.URL.format(code=self.table_code.lower())
        shell('wget -O {output}.gz "{url}"'.format(
        output=self.output().path,
        url=URL))

    def run(self):
        self.output().makedirs()
        self.download()
        shell('gunzip {output}.gz'.format(output=self.output().path))

    def output(self):
        return LocalTarget(os.path.join('tmp', classpath(self), self.task_id))

class EUTempTable(CSV2TempTableTask):

    delimiter = Parameter(default='\t', significant=False)
    table_name = Parameter() # Ex. "DEMO_R_PJANAGGR3"

    def version(self):
        return 3

    def requires(self):
        return DownloadEurostat(table_code=self.table_name)

    def input_csv(self):
        shell("cat {} | tr ',' '\t' > {}.csv".format(self.input().path,self.input().path))
        return self.input().path + '.csv'

class EUFormatTable(TempTableTask):

    def requires(self):
        return EUTempTable()

class FlexEurostatColumns(ColumnsTask):

    subsection = Parameter()
    table_name = Parameter()  # Ex. "DEMO_R_PJANAGGR3"

    # From tablename, determine basis of name for columns from table_dic.dic
    # Then, look at metabase.txt to find relevant dimensions (exclude "geo" and "time", utilize "unit")
    # Finally, look up definitions for dimensions from their .dic files, and use that to complete the metadata definition

    def requires(self):
        return {
            'units': UnitTags(),
            'subsection': SubsectionTags(),
            'section': SectionTags()
        }

    def version(self):
        return 3

    def columns(self):
        columns = OrderedDict()

        input_ = self.input()

        subsectiontags = input_['subsection']
        unittags = input_['units'] #???
        eu = input_['section']['eu']

        tags=[eu,subsectiontags[self.subsection]]

        current_code = None
        codes = {}
        current_list = []
        with open(os.path.join(os.path.dirname(__file__),'metabase.txt')) as metabase:
            reader = csv.reader(metabase,delimiter='\t')
            for possible_tablenames, code, code_value in reader:
                if self.table_name.lower() == possible_tablenames.lower():
                    if code != "geo" and code != "unit" and code != "time":
                        if current_code == code:
                            current_list.append(code_value)
                        elif current_code:
                            codes[current_code] = current_list
                            current_list = []
                            current_list.append(code_value)
                        else:
                            current_list.append(code_value)
                        current_code = code
                    codes[current_code] = current_list

        product = [x for x in apply(itertools.product, codes.values())]
        cross_prod = [dict(zip(codes.keys(), p)) for p in product]

        with open(os.path.join(os.path.dirname(__file__),'table_dic.dic')) as tabledicfile:
            tablereader = csv.reader(tabledicfile,delimiter='\t')
            for possible_tablenames, table_description in tablereader:
                if self.table_name.lower() == possible_tablenames.lower():
                    table_desc = table_description
                    variable_name = table_description.split('by')[0].strip()
                    break

        if cross_prod:
            for i in cross_prod:
                var_code = underscore_slugify(variable_name+"_".join(i.values()))
                dimdefs = []
                for dimname, value in i.iteritems():
                    with open(os.path.join(os.path.dirname(__file__),'dic_lists/{dimension}.dic'.format(dimension=dimname))) as dimfile:
                        reader = csv.reader(dimfile, delimiter='\t')
                        for possible_dimvalue, dimdef in reader:
                            if value == possible_dimvalue:
                                dimdefs.append(dimdef)
                description = "{} ".format(variable_name)+", ".join([str(x) for x in dimdefs])
                columns[var_code] = OBSColumn(
                    id=var_code,
                    name=description,
                    type='Text',
                    description=table_desc,
                    weight=1,
                    aggregate=None, #???
                    targets={}, #???
                    tags=tags,
                    extra=i
                )
        else:
            var_code = underscore_slugify(variable_name)
            description = variable_name
            columns[var_code] = OBSColumn(
                id=var_code,
                name=variable_name,
                type='Text',
                description= table_desc,
                weight=1,
                aggregate=None, #???
                targets={}, #???
                tags=tags,
                extra=None
            )

        return columns

class NUTSMeta(ColumnsTask):
    def columns(self):
        columns = OrderedDict()
        columns['geo'] = OBSColumn(
            id='geo',
            type='text',
            weight=0,
        )
        return columns

class TableEU(TableTask):

    table_name = Parameter()
    subsection = Parameter()

    def version(self):
        return

    def timespan(self):
        return '2015'

    def requires(self):
        requirements = {
            'data': EUTempTable(table_name=self.table_name),
            'meta': FlexEurostatColumns(table_name=self.table_name, subsection = self.subsection),
            'geometa': NUTSMeta(), #still needed
        }
        return requirements

    def columns(self):
        cols = OrderedDict()
        cols['geo'] = self.input()['geometa']['geo'] #still needed
        cols.update(self.input()['meta'])
        return cols

    def populate(self):
        session = current_session()
        session.execute('ALTER TABLE {output} ADD PRIMARY KEY (geo)'.format(
            output=self.output().table))
        session.flush()
        column_targets = self.columns()
        for k,v in column_targets.iteritems():
            if k != 'geo':
                col = v.get(session)
                extra = col.extra
                keys = extra.keys()
                vals = [extra[k_] for k_ in keys]
                session.execute('INSERT INTO {output} (geo, {id}) '
                                'SELECT "{geo_id}", Nullif("{year} ", \': \')::Text '
                                'FROM {input} '
                                'WHERE ({input_dims}) = (\'{output_dims}\')'
                                'ON CONFLICT (geo)'
                                '   DO UPDATE SET {id} = EXCLUDED.{id}'.format(
                                    geo_id=r"geo\time",
                                    id=k,
                                    year=self.timespan(),
                                    input_dims = ', '.join(keys),
                                    output_dims = "', '".join(vals),
                                    output=self.output().table,
                                    input=self.input()['data'].table
                                ))
<|endoftext|>


<commit_before># Django modules
from django.contrib import admin

# Our own modules
from events.models import Event

"""
events.admin module

This is the admin module for our events application.
Here we can define admin classes for our models and customize how they appear in the admin interface.
"""

class EventAdmin(admin.ModelAdmin):
    """
    Event admin class.

    We define that the model that we handle is Event and that we want to change the form to use django_wysiwyg.
    """
    model = Event
    change_form_template = 'events/admin/change_form.html'

# Register the models in our admin view
admin.site.register(Event, EventAdmin)
<commit_msg>Add filtering on event date on admin view.<commit_after># Django modules
from django.contrib import admin

# Our own modules
from events.models import Event

"""
events.admin module

This is the admin module for our events application.
Here we can define admin classes for our models and customize how they appear in the admin interface.
"""

class EventAdmin(admin.ModelAdmin):
    """
    Event admin class.

    We define that the model that we handle is Event and that we want to change the form to use django_wysiwyg.
    """
    model = Event
    change_form_template = 'events/admin/change_form.html'
    list_filter = ['date']

# Register the models in our admin view
admin.site.register(Event, EventAdmin)
<|endoftext|>


<commit_before>from collections import defaultdict
from datetime import date,datetime,timedelta,time
from time import strftime

from django.utils.datastructures import MultiValueDictKeyError
from django.http import HttpResponse
from django.template import RequestContext
from django.shortcuts import *
from django.contrib.sites.models import Site
from django.contrib.auth.decorators import login_required
from django import forms

import vobject
import re

from compsoc.events.models import *
from compsoc.settings import DATE_FORMAT_STRING,WEEK_FORMAT_STRING
from compsoc.memberinfo.models import warwick_week_for,Term
from compsoc.shortcuts import begin_week,path_processor

from compsoc.events.similarity import closest_person

def calendar_index(request): return calendar(request,0)

def month(dat): return strftime("%b",(0,dat,0,0,0,0,0,0,0))

a_week = timedelta(days=7)

def safe_week_for(date):
    dt = datetime(date.year,date.month,date.day)
    try:
        return str(warwick_week_for(dt))
    except Term.DoesNotExist:
        return ""

def get_listable_events(offset,span):
    begin = begin_week(datetime.today())+timedelta(days=7*offset)
    end = begin + timedelta(days=7*span)
    events = Event.objects.order_by('start').filter(finish__gte=begin).exclude(displayFrom__gte=datetime.now())
    return (begin.date(),end.date(),events)

class Week:
    '''
    Abstract a week, so the template doesn't require any lookup logic
    '''
    def __init__(self,begin):
        self.begin = begin
        self.end = begin + timedelta(days=6)
        self.week_number = safe_week_for(begin)
    def __str__(self):
        return self.begin.strftime(WEEK_FORMAT_STRING)+" - "+self.end.strftime(WEEK_FORMAT_STRING)

def events_list(request):
    begin,end,events = get_listable_events(0,10)
    lookup = defaultdict(lambda: [])
    for event in events:
       lookup[begin_week(event.start)].append(event)

    lookup = map(lambda (begin,events): (Week(begin),events),sorted(lookup.items()))
    return render_to_response('events/list.html', {
        'breadcrumbs': [('/','home'),('/events/','events')],
        'events':lookup,
        'future':future_events()
    },context_instance=RequestContext(request,{},[path_processor]))

def calendar(request,delta):
    '''
    Main Calendar view for website
    presents a Table of events for the current week, and 4 successors
    '''
    offset = int(delta)
    begin,end,events = get_listable_events(offset,4)
    # lookup :: BEGIN_WEEK -> DAY -> EVENT
    lookup = defaultdict(lambda: defaultdict(lambda:[]))
    for event in events:
        for event_date in event.days():
            lookup[begin_week(event_date)][event_date.weekday()].append(event)

    # tidy information
    iter,events = begin,[]
    while iter <= end:
        week = lookup[iter]
        week_vals = []
        for i in range(0,7):
            iter_date = iter+timedelta(i)
            if (iter==begin and i==0) or iter_date.day==1:
                val = month(iter_date.month)
            else:
                val = ""
            week_vals.append((val+" "+str(iter_date.day),lookup[iter][i]))
        events.append((safe_week_for(iter),week_vals))
        iter += a_week

    return render_to_response('events/calendar.html',{
        'breadcrumbs': [('/','home'),('/events/','events'),('/events/calendar/','calendar')],
        'events':events,
        'current':date.today(),
        'prev':offset-1,
        'next':offset+1,
        'future':future_events(),
    },context_instance=RequestContext(request,{},[path_processor]))

def valid_signup(user,event):
    now = datetime.now()
    s = event.eventsignup
    if event.has_signups() and user.is_authenticated():
        can_signup = now < s.close and not event.cancelled
        if user.member.guest:
            can_signup &= s.guest_open < now
        elif user.member.is_fresher():
            can_signup &= s.fresher_open < now
        else:
            can_signup &= s.open < now
    else:
        can_signup = False
    return can_signup

def details(request,event_id):
    event = get_object_or_404(Event, id=event_id)
    signups = event.signup_set.order_by('time')
    max = event.signup_total()
    # 0 = inifnite signups
    if max:
        reserved = signups[max:]
        signups = signups[:max]
    else:
        reserved = []

    u = request.user

    dict = {
        'event':event,
        'signups':signups,
        'reserved':reserved,
        'can_edit':request.user.is_staff if request.user else False,
        'future':future_events(),
        'is_displayed':event.is_displayed(),
    }

    try:
        s = event.eventsignup
        signed_up = u.is_authenticated() and event.signup_set.filter(user=request.user)
        dict.update({
            'open':s.open.strftime(DATE_FORMAT_STRING),
            'close':s.close.strftime(DATE_FORMAT_STRING),
            'fresher':s.fresher_open.strftime(DATE_FORMAT_STRING),
            'guest':s.guest_open.strftime(DATE_FORMAT_STRING),
            'limit':s.signupsLimit,
            'can_signup':valid_signup(u,event) and not signed_up,
            'signed_up':signed_up,
            'has_seating':s.seating,
        })
    except EventSignup.DoesNotExist, e:
        dict.update({ 'can_signup':False })

    return render_to_response('events/details.html',dict,
        context_instance=RequestContext(request,{},[path_processor]))

p = re.compile(r"col([0-5])\((.*),\)")

# assumes initial revision
def seating(request, event_id, revision_no=None):
    e = get_object_or_404(Event, id=event_id)
    dict = {
            'event':e,
            'future':future_events(),
           }
    try:
        signup = e.eventsignup
        if signup.has_seating_plan():
            room = signup.seating
            closed = signup.close < datetime.now()

            if request.method == 'POST' and request.user.is_authenticated():
                if closed:
                    return render_to_response('events/plan_closed.html')
                order = request.POST['order']
                previous = SeatingRevision.objects.filter(event=e).order_by('-number')
                last_no = previous[0].number if previous else 0
                revision = e.seatingrevision_set.create(
                    creator=request.user,
                    comment=request.POST['comment'],
                    number=last_no+1
                )
                for col in order.split(';'):
                    m = p.match(col)
                    if m:
                        column = int(m.group(1))
                        for row,id_string in enumerate(m.group(2).split(',')):
                            if row >= room.max_rows or column >= room.max_cols:
                                revision.delete()
                                return render_to_response('events/seating_size_failure.html',{
                                    'event':e,
                                    'room':room,
                                },context_instance=RequestContext(request,{},[path_processor]))
                            else:
                                try:
                                    u = User.objects.get(id=int(id_string))
                                    revision.seating_set.create(user=u,col=column,row=row)
                                except ValueError,User.DoesNotExist: pass
                revisions = SeatingRevision.objects.for_event(e)
            else:
                revisions = SeatingRevision.objects.for_event(e)
                if revisions:
                    revision = revisions[0] if revision_no==None else SeatingRevision.objects.get(number=revision_no,event=e)

            # create a seat lookup dict
            seat_dict = defaultdict(lambda: defaultdict(lambda: (False,False)))
            unass = set([s.user for s in e.signup_set.all()])
            if revisions:
                added = val_users(revision.added())
                moved = map(lambda (curr,prev):curr.user.pk,revision.moved())
                removed = val_users(revision.removed())
                for seat in revision.seating_set.all():
                    uid = seat.user.pk
                    if uid in added:
                        type = 'added'
                    elif uid in moved:
                        type = 'moved'
                    else:
                        type = 'static'
                    seat_dict[seat.col][seat.row] = (seat.user,type)
                    unass.discard(seat.user)

                unass = map(lambda u: (u,'removed' if u.pk in removed else 'static'),unass)
            else:
                unass = map(lambda u: (u,'static'),unass)

            # create nested lists for rows and columns
            cols = range(0,room.max_cols)
            max_rows = [ max([-1]+seat_dict[y].keys())+1 for y in cols]
            s = [[seat_dict[y][x] for x in range(0,max_rows[y])] for y in cols]

            dict.update({
                'room':room,
                'seating':s,
                'seating_revisions':revisions,
                'new_revision_no':revisions[0].number+1 if revisions else 0,
                'unassigned':unass,
                'notclosed':not closed,
                })
    except EventSignup.DoesNotExist: pass

    return render_to_response('events/seating.html', dict
        ,context_instance=RequestContext(request,{},[path_processor]))

class CommentForm(forms.Form):
    comment = forms.CharField(max_length=255, required=False)

# TODo: add to admin section
#                if event.signup_set.filter(user=request.user):
#                    return render_to_response('events/alreadysignedup.html',{'event':event})
@login_required
def do_signup(request,event_id):
    '''
    Post:
        If a user is signed up then change their comment
        otherwise add signup entry if its valid to
    Get:
        return a form, if they are signed up, it should have their contacts in
    '''
    event = get_object_or_404(Event, id=event_id)
    if request.method == 'POST':
        form = CommentForm(request.POST)
        if form.is_valid():
            c = form.cleaned_data['comment']
            try:
                e = event.signup_set.get(user=request.user)
                e.comment = c
                e.save()
            except Signup.DoesNotExist:
                if valid_signup(request.user,event):
                    event.signup_set.create(time=datetime.now(),user=request.user,comment=c)
                else:
                    return render_to_response('events/cantsignup.html',{'event':event},
                        context_instance=RequestContext(request,{},[path_processor]))
        else:
            return render_to_response('events/comment_fail.html',{
                'event':event,
                'errors':form.errors,
            },context_instance=RequestContext(request,{},[path_processor]))

        return render_to_response('events/signup.html',{'event_id':event_id,},
            context_instance=RequestContext(request,{},[path_processor]))
    else:
        form = CommentForm()
        try:
            e = event.signup_set.get(user=request.user)
        except Signup.DoesNotExist: pass

    return render_to_response('events/edit_signup.html', {
        'form': form,
        'event_id': event_id,
    },context_instance=RequestContext(request,{},[path_processor]))

@login_required
def do_unsignup(request,event_id):
    try:
        event = get_object_or_404(Event, id=event_id)
        signup = event.signup_set.get(user=request.user)
        signup.delete()
    except Signup.DoesNotExist:
        return render_to_response('events/nonsignup.html',{'event':event,},
            context_instance=RequestContext(request,{},[path_processor]))
    return render_to_response('events/unsignup.html',{'event_id':event_id,},
        context_instance=RequestContext(request,{},[path_processor]))

def ical_feed(request):
    '''
    Generates an ical sync of all events in the future
    '''
    cal = vobject.iCalendar()
    # IE/Outlook needs this:
    cal.add('method').value = 'PUBLISH'
    # Only publish events in the future
    for event in filter(lambda e: e.is_in_future(),Event.objects.order_by('start').exclude(displayFrom__gte=datetime.now())):
        total = str(event.signup_total()) if event.signup_total() else u'\u221E'
        signups = u' [%i/%s]' % (event.signup_count(),total) if event.has_signups() else u''
        vevent = cal.add('vevent')
        vevent.add('summary').value = event.type.name + (' ":" ' + event.shortDescription if event.shortDescription else event.type.name) + signups
        vevent.add('location').value = str(event.location)
        vevent.add('dtstart').value = event.start
        vevent.add('dtend').value = event.finish
        vevent.add('dtstamp').value = event.start # again, for Outlook
        vevent.add('description').value = event.longDescription
        vevent.add('categories').value = [event.type.get_target_display()]
        url = "http://%s/events/details/%i/" % (Site.objects.get_current() , event.id)
        vevent.add('uid').value = url
        vevent.add('url').value = url
    response = HttpResponse(cal.serialize(), mimetype='text/calendar')
    response['Filename'] = 'filename.ics'  # IE needs this
    response['Content-Disposition'] = 'attachment; filename=filename.ics'
    return response

def location(request,object_id):
    '''
    Location details page controller
    '''
    loc = Location.objects.get(pk=object_id)
    return render_to_response("events/location_detail.html", {
        'object':loc,
        'map_room':loc.map_loc,
    },context_instance=RequestContext(request,{},[path_processor]))

def lan_friends(request):
    return render_to_response('events/lan_friends.html', {
        'friends':closest_person(),
    },context_instance=RequestContext(request,{},[path_processor]))

@login_required
def activity(request):
    members = []
    for u in User.objects.all():
        signup_count = u.signup_set.count()
        if signup_count:
            members.append((u.member.name(),signup_count))
    members.sort(key=lambda(x,y):y,reverse=True)

    return render_to_response('events/activity.html', {
        'breadcrumbs': [('/','home'),('/events/','events'),('/events/activity/','activity')],
        'members': members,
        'future':future_events()
    },context_instance=RequestContext(request,{},[path_processor]))

<commit_msg>Changed to - as seperator, fixing #90<commit_after>from collections import defaultdict
from datetime import date,datetime,timedelta,time
from time import strftime

from django.utils.datastructures import MultiValueDictKeyError
from django.http import HttpResponse
from django.template import RequestContext
from django.shortcuts import *
from django.contrib.sites.models import Site
from django.contrib.auth.decorators import login_required
from django import forms

import vobject
import re

from compsoc.events.models import *
from compsoc.settings import DATE_FORMAT_STRING,WEEK_FORMAT_STRING
from compsoc.memberinfo.models import warwick_week_for,Term
from compsoc.shortcuts import begin_week,path_processor

from compsoc.events.similarity import closest_person

def calendar_index(request): return calendar(request,0)

def month(dat): return strftime("%b",(0,dat,0,0,0,0,0,0,0))

a_week = timedelta(days=7)

def safe_week_for(date):
    dt = datetime(date.year,date.month,date.day)
    try:
        return str(warwick_week_for(dt))
    except Term.DoesNotExist:
        return ""

def get_listable_events(offset,span):
    begin = begin_week(datetime.today())+timedelta(days=7*offset)
    end = begin + timedelta(days=7*span)
    events = Event.objects.order_by('start').filter(finish__gte=begin).exclude(displayFrom__gte=datetime.now())
    return (begin.date(),end.date(),events)

class Week:
    '''
    Abstract a week, so the template doesn't require any lookup logic
    '''
    def __init__(self,begin):
        self.begin = begin
        self.end = begin + timedelta(days=6)
        self.week_number = safe_week_for(begin)
    def __str__(self):
        return self.begin.strftime(WEEK_FORMAT_STRING)+" - "+self.end.strftime(WEEK_FORMAT_STRING)

def events_list(request):
    begin,end,events = get_listable_events(0,10)
    lookup = defaultdict(lambda: [])
    for event in events:
       lookup[begin_week(event.start)].append(event)

    lookup = map(lambda (begin,events): (Week(begin),events),sorted(lookup.items()))
    return render_to_response('events/list.html', {
        'breadcrumbs': [('/','home'),('/events/','events')],
        'events':lookup,
        'future':future_events()
    },context_instance=RequestContext(request,{},[path_processor]))

def calendar(request,delta):
    '''
    Main Calendar view for website
    presents a Table of events for the current week, and 4 successors
    '''
    offset = int(delta)
    begin,end,events = get_listable_events(offset,4)
    # lookup :: BEGIN_WEEK -> DAY -> EVENT
    lookup = defaultdict(lambda: defaultdict(lambda:[]))
    for event in events:
        for event_date in event.days():
            lookup[begin_week(event_date)][event_date.weekday()].append(event)

    # tidy information
    iter,events = begin,[]
    while iter <= end:
        week = lookup[iter]
        week_vals = []
        for i in range(0,7):
            iter_date = iter+timedelta(i)
            if (iter==begin and i==0) or iter_date.day==1:
                val = month(iter_date.month)
            else:
                val = ""
            week_vals.append((val+" "+str(iter_date.day),lookup[iter][i]))
        events.append((safe_week_for(iter),week_vals))
        iter += a_week

    return render_to_response('events/calendar.html',{
        'breadcrumbs': [('/','home'),('/events/','events'),('/events/calendar/','calendar')],
        'events':events,
        'current':date.today(),
        'prev':offset-1,
        'next':offset+1,
        'future':future_events(),
    },context_instance=RequestContext(request,{},[path_processor]))

def valid_signup(user,event):
    now = datetime.now()
    s = event.eventsignup
    if event.has_signups() and user.is_authenticated():
        can_signup = now < s.close and not event.cancelled
        if user.member.guest:
            can_signup &= s.guest_open < now
        elif user.member.is_fresher():
            can_signup &= s.fresher_open < now
        else:
            can_signup &= s.open < now
    else:
        can_signup = False
    return can_signup

def details(request,event_id):
    event = get_object_or_404(Event, id=event_id)
    signups = event.signup_set.order_by('time')
    max = event.signup_total()
    # 0 = inifnite signups
    if max:
        reserved = signups[max:]
        signups = signups[:max]
    else:
        reserved = []

    u = request.user

    dict = {
        'event':event,
        'signups':signups,
        'reserved':reserved,
        'can_edit':request.user.is_staff if request.user else False,
        'future':future_events(),
        'is_displayed':event.is_displayed(),
    }

    try:
        s = event.eventsignup
        signed_up = u.is_authenticated() and event.signup_set.filter(user=request.user)
        dict.update({
            'open':s.open.strftime(DATE_FORMAT_STRING),
            'close':s.close.strftime(DATE_FORMAT_STRING),
            'fresher':s.fresher_open.strftime(DATE_FORMAT_STRING),
            'guest':s.guest_open.strftime(DATE_FORMAT_STRING),
            'limit':s.signupsLimit,
            'can_signup':valid_signup(u,event) and not signed_up,
            'signed_up':signed_up,
            'has_seating':s.seating,
        })
    except EventSignup.DoesNotExist, e:
        dict.update({ 'can_signup':False })

    return render_to_response('events/details.html',dict,
        context_instance=RequestContext(request,{},[path_processor]))

p = re.compile(r"col([0-5])\((.*),\)")

# assumes initial revision
def seating(request, event_id, revision_no=None):
    e = get_object_or_404(Event, id=event_id)
    dict = {
            'event':e,
            'future':future_events(),
           }
    try:
        signup = e.eventsignup
        if signup.has_seating_plan():
            room = signup.seating
            closed = signup.close < datetime.now()

            if request.method == 'POST' and request.user.is_authenticated():
                if closed:
                    return render_to_response('events/plan_closed.html')
                order = request.POST['order']
                previous = SeatingRevision.objects.filter(event=e).order_by('-number')
                last_no = previous[0].number if previous else 0
                revision = e.seatingrevision_set.create(
                    creator=request.user,
                    comment=request.POST['comment'],
                    number=last_no+1
                )
                for col in order.split(';'):
                    m = p.match(col)
                    if m:
                        column = int(m.group(1))
                        for row,id_string in enumerate(m.group(2).split(',')):
                            if row >= room.max_rows or column >= room.max_cols:
                                revision.delete()
                                return render_to_response('events/seating_size_failure.html',{
                                    'event':e,
                                    'room':room,
                                },context_instance=RequestContext(request,{},[path_processor]))
                            else:
                                try:
                                    u = User.objects.get(id=int(id_string))
                                    revision.seating_set.create(user=u,col=column,row=row)
                                except ValueError,User.DoesNotExist: pass
                revisions = SeatingRevision.objects.for_event(e)
            else:
                revisions = SeatingRevision.objects.for_event(e)
                if revisions:
                    revision = revisions[0] if revision_no==None else SeatingRevision.objects.get(number=revision_no,event=e)

            # create a seat lookup dict
            seat_dict = defaultdict(lambda: defaultdict(lambda: (False,False)))
            unass = set([s.user for s in e.signup_set.all()])
            if revisions:
                added = val_users(revision.added())
                moved = map(lambda (curr,prev):curr.user.pk,revision.moved())
                removed = val_users(revision.removed())
                for seat in revision.seating_set.all():
                    uid = seat.user.pk
                    if uid in added:
                        type = 'added'
                    elif uid in moved:
                        type = 'moved'
                    else:
                        type = 'static'
                    seat_dict[seat.col][seat.row] = (seat.user,type)
                    unass.discard(seat.user)

                unass = map(lambda u: (u,'removed' if u.pk in removed else 'static'),unass)
            else:
                unass = map(lambda u: (u,'static'),unass)

            # create nested lists for rows and columns
            cols = range(0,room.max_cols)
            max_rows = [ max([-1]+seat_dict[y].keys())+1 for y in cols]
            s = [[seat_dict[y][x] for x in range(0,max_rows[y])] for y in cols]

            dict.update({
                'room':room,
                'seating':s,
                'seating_revisions':revisions,
                'new_revision_no':revisions[0].number+1 if revisions else 0,
                'unassigned':unass,
                'notclosed':not closed,
                })
    except EventSignup.DoesNotExist: pass

    return render_to_response('events/seating.html', dict
        ,context_instance=RequestContext(request,{},[path_processor]))

class CommentForm(forms.Form):
    comment = forms.CharField(max_length=255, required=False)

# TODo: add to admin section
#                if event.signup_set.filter(user=request.user):
#                    return render_to_response('events/alreadysignedup.html',{'event':event})
@login_required
def do_signup(request,event_id):
    '''
    Post:
        If a user is signed up then change their comment
        otherwise add signup entry if its valid to
    Get:
        return a form, if they are signed up, it should have their contacts in
    '''
    event = get_object_or_404(Event, id=event_id)
    if request.method == 'POST':
        form = CommentForm(request.POST)
        if form.is_valid():
            c = form.cleaned_data['comment']
            try:
                e = event.signup_set.get(user=request.user)
                e.comment = c
                e.save()
            except Signup.DoesNotExist:
                if valid_signup(request.user,event):
                    event.signup_set.create(time=datetime.now(),user=request.user,comment=c)
                else:
                    return render_to_response('events/cantsignup.html',{'event':event},
                        context_instance=RequestContext(request,{},[path_processor]))
        else:
            return render_to_response('events/comment_fail.html',{
                'event':event,
                'errors':form.errors,
            },context_instance=RequestContext(request,{},[path_processor]))

        return render_to_response('events/signup.html',{'event_id':event_id,},
            context_instance=RequestContext(request,{},[path_processor]))
    else:
        form = CommentForm()
        try:
            e = event.signup_set.get(user=request.user)
        except Signup.DoesNotExist: pass

    return render_to_response('events/edit_signup.html', {
        'form': form,
        'event_id': event_id,
    },context_instance=RequestContext(request,{},[path_processor]))

@login_required
def do_unsignup(request,event_id):
    try:
        event = get_object_or_404(Event, id=event_id)
        signup = event.signup_set.get(user=request.user)
        signup.delete()
    except Signup.DoesNotExist:
        return render_to_response('events/nonsignup.html',{'event':event,},
            context_instance=RequestContext(request,{},[path_processor]))
    return render_to_response('events/unsignup.html',{'event_id':event_id,},
        context_instance=RequestContext(request,{},[path_processor]))

def ical_feed(request):
    '''
    Generates an ical sync of all events in the future
    '''
    cal = vobject.iCalendar()
    # IE/Outlook needs this:
    cal.add('method').value = 'PUBLISH'
    # Only publish events in the future
    for event in filter(lambda e: e.is_in_future(),Event.objects.order_by('start').exclude(displayFrom__gte=datetime.now())):
        total = str(event.signup_total()) if event.signup_total() else u'\u221E'
        signups = u' [%i/%s]' % (event.signup_count(),total) if event.has_signups() else u''
        vevent = cal.add('vevent')
        vevent.add('summary').value = event.type.name + (' - ' + event.shortDescription if event.shortDescription else event.type.name) + signups
        vevent.add('location').value = str(event.location)
        vevent.add('dtstart').value = event.start
        vevent.add('dtend').value = event.finish
        vevent.add('dtstamp').value = event.start # again, for Outlook
        vevent.add('description').value = event.longDescription
        vevent.add('categories').value = [event.type.get_target_display()]
        url = "http://%s/events/details/%i/" % (Site.objects.get_current() , event.id)
        vevent.add('uid').value = url
        vevent.add('url').value = url
    response = HttpResponse(cal.serialize(), mimetype='text/calendar')
    response['Filename'] = 'filename.ics'  # IE needs this
    response['Content-Disposition'] = 'attachment; filename=filename.ics'
    return response

def location(request,object_id):
    '''
    Location details page controller
    '''
    loc = Location.objects.get(pk=object_id)
    return render_to_response("events/location_detail.html", {
        'object':loc,
        'map_room':loc.map_loc,
    },context_instance=RequestContext(request,{},[path_processor]))

def lan_friends(request):
    return render_to_response('events/lan_friends.html', {
        'friends':closest_person(),
    },context_instance=RequestContext(request,{},[path_processor]))

@login_required
def activity(request):
    members = []
    for u in User.objects.all():
        signup_count = u.signup_set.count()
        if signup_count:
            members.append((u.member.name(),signup_count))
    members.sort(key=lambda(x,y):y,reverse=True)

    return render_to_response('events/activity.html', {
        'breadcrumbs': [('/','home'),('/events/','events'),('/events/activity/','activity')],
        'members': members,
        'future':future_events()
    },context_instance=RequestContext(request,{},[path_processor]))

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os
import sys
import subprocess
import argparse
import urllib2
import ConfigParser
import json
from datetime import datetime

CONFIG_PATH   = os.path.join(os.getenv('HOME'),'.dot')
REPOS_PATH    = os.path.join(CONFIG_PATH,'data')
METADATA_PATH = os.path.join(CONFIG_PATH,'data.json')
MANIFEST_PATH = os.path.join(CONFIG_PATH,'manifest.ini')
LOG_PATH      = os.path.join(CONFIG_PATH,'log')

class Dot:

  def __init__(self,args):
    self.dots = []
    self._parse_args(args)

  def _parse_args(self,args):
    parser = argparse.ArgumentParser()

    subparsers = parser.add_subparsers()

    parser_install = subparsers.add_parser('i',
        help="install stuff")
    parser_install.set_defaults(action='install')
    parser_install.add_argument('manifest', type=str,
        help='path or url of manifest file')

    parser_status = subparsers.add_parser('s',
        help="get git status")
    parser_status.set_defaults(action='status')

    parser_pull = subparsers.add_parser('u',
        help="update configs")
    parser_pull.set_defaults(action='update')

    parser_push = subparsers.add_parser('p',
        help="upload changes")
    parser_push.set_defaults(action='upload')

    parser_cd = subparsers.add_parser('cd',
        help="cd into config directory")
    parser_cd.set_defaults(action='chdir')
    parser_cd.add_argument('dot', type=str)

    parsed = parser.parse_args(args)

    self.action   = parsed.action
    self.manifest = lambda: parsed.manifest

  def do(self):
    if self.action   == 'install':
      self.install(self.manifest())
    elif self.action == 'status':
      self.status()
    elif self.action == 'update':
      self.update()
    elif self.action == 'upload':
      self.upload()
    elif self.action == 'chdir':
      self.chdir()

  def install(self,manifest_url):
    os.mkdir(CONFIG_PATH)
    os.mkdir(LOG_PATH)

    if os.access(MANIFEST_PATH,os.R_OK):
      self._error("sorry, manifest file already exists")
      exit(1)
    else:
      self._info('downloading manifest from "{}"...'.format(manifest_url))
      manifest = urllib2.urlopen(manifest_url).read()
      with open(MANIFEST_PATH,'w') as f:
        f.write(manifest)

    self.clone()

  def clone(self):
    self._info("started cloning...")
    job = lambda repo,url: Git().clone(url,repo)
    for name,cmd,logfile in self._in_repos(job):
      if cmd.exitcode == 0:
        self._assign_metadata(name,'cloned',datetime.utcnow().isoformat())
        self._print_result('ok')
      else:
        self._print_result('ERROR!',logfile)

  def status(self):
    self._info('repos status:')
    job = lambda repo,url: Git(repo).status()
    for _,cmd,_ in self._in_repos(job):
      if len(cmd.stdout) > 0:
        self._print_result('DIRTY')
      else:
        self._print_result("clean")

  def update(self):
    self._info('pulling from remotes...')
    job = lambda repo,url: Git(repo).pull()
    for _,cmd,logfile in self._in_repos(job):
      if cmd.exitcode == 0:
        self._print_result('ok')
      else:
        self._print_result('ERROR!',logfile)

  def upload(self):
    self._info('pushing to remotes...')
    job = lambda repo,url: Git(repo).push()
    for _,cmd,logfile in self._in_repos(job):
      if cmd.exitcode == 0:
        self._print_result('ok')
      else:
        self._print_result('ERROR!',logfile)

  def chdir(self):
    for dot in self.dots:
      name, url = dot
      repo = os.path.join(CONFIG_PATH,name)
      if self.config.dot == name:
        #os.chdir(repo)
        print repo

  def _info(self,text):
    sys.stdout.write(". "+text+"\n")

  def _error(self,text):
    sys.stderr.write(". "+text+"\n")

  def _parse_manifest(self):
    manifest_data = ConfigParser.ConfigParser()
    manifest_data.read(MANIFEST_PATH)
    self.dots = manifest_data.items('all')

  def _load_metadata(self):
    if os.access(METADATA_PATH,os.R_OK):
      with open(METADATA_PATH,'r') as m:
        self.metadata = json.loads(m.read())
    else:
        self.metadata = {}

    for repo,_ in self.dots:
      if not repo in self.metadata: self.metadata[repo]={}

  def _assign_metadata(self,repo,key,value):
    self.metadata[repo][key]=value

  def _dump_metadata(self):
    with open(METADATA_PATH,'w') as m:
      m.write(json.dumps(self.metadata,sort_keys=True,indent=2))

  def _in_repos(self,job):
    self._parse_manifest()
    self._load_metadata()

    async = Async()
    for dot in self.dots:
      name, url = dot
      path = os.path.join(REPOS_PATH,name)
      async.add(name,job(path,url))

    for name,executor in async.run():
      logfile = os.path.join(LOG_PATH,name+'.log')
      with open(logfile,'w') as log:
        cmd = ' '.join(str(i) for i in executor.cmd)
        msg = "command: {}\nreturn code: {}\n{}".format(cmd,executor.exitcode,executor.stderr)
        log.write(msg)

      sys.stdout.write(name+': ')
      yield name,executor,logfile

    self._dump_metadata()

  def _print_result(self,text,logfile=None):
    if logfile:
      text = text + ' check out logfile: {}'.format(logfile)
    sys.stdout.write(text+"\n")

class Executor():
  def __init__(self,cmd):
    self.cmd      = cmd
    self._process = None
    self.stdout   = None
    self.stderr   = None
    self.exitcode = None
  def start(self):
    self._process = subprocess.Popen(self.cmd,
        stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    return self
  def ok(self):
    return self.exitcode == 0
  def join(self):
    self.stdout,self.stderr = self._process.communicate()
    self.exitcode = self._process.returncode

class Async():
  def __init__(self):
    self.cmds={}
  def add(self,name,cmd):
    self.cmds[name]=cmd
  def run(self):
    subprocesses = {}
    for name,cmd in self.cmds.iteritems():
      subprocesses[name]=Executor(cmd).start()
    for t in subprocesses.itervalues(): t.join()
    for name,executor in subprocesses.iteritems():
      yield name,executor

class Git():

  def __init__(self,path=None):
    self.repo_path = path

  def clone(self,url,repo):
    return ['git', 'clone', '-q', '--recursive', '--', url, repo]

  def pull(self):
    return self._git() + ['pull']

  def push(self):
    return self._git() + ['push']

  def status(self):
    return self._git() + ['status', '--porcelain']

  def _git(self):
    return ['git'] + self._git_path_opts()

  def _git_path_opts(self):
    tmpl = ['--git-dir={}/.git', '--work-tree={}']
    return map(lambda s: s.format(self.repo_path),tmpl)

if __name__ == "__main__":
  Dot(sys.argv[1:]).do()
<commit_msg>basic ability to skip command based on metadata<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os
import sys
import subprocess
import argparse
import urllib2
import ConfigParser
import json
from datetime import datetime

CONFIG_PATH   = os.path.join(os.getenv('HOME'),'.dot')
REPOS_PATH    = os.path.join(CONFIG_PATH,'data')
METADATA_PATH = os.path.join(CONFIG_PATH,'data.json')
MANIFEST_PATH = os.path.join(CONFIG_PATH,'manifest.ini')
LOG_PATH      = os.path.join(CONFIG_PATH,'log')

class Dot:

  def __init__(self,args):
    self.dots = []
    self._parse_args(args)

  def _parse_args(self,args):
    parser = argparse.ArgumentParser()

    subparsers = parser.add_subparsers()

    parser_install = subparsers.add_parser('i',
        help="install stuff")
    parser_install.set_defaults(action='install')
    parser_install.add_argument('manifest', type=str,
        help='path or url of manifest file')

    parser_status = subparsers.add_parser('s',
        help="get git status")
    parser_status.set_defaults(action='status')

    parser_pull = subparsers.add_parser('u',
        help="update configs")
    parser_pull.set_defaults(action='update')

    parser_push = subparsers.add_parser('p',
        help="upload changes")
    parser_push.set_defaults(action='upload')

    parser_cd = subparsers.add_parser('cd',
        help="cd into config directory")
    parser_cd.set_defaults(action='chdir')
    parser_cd.add_argument('dot', type=str)

    parsed = parser.parse_args(args)

    self.action   = parsed.action
    self.manifest = lambda: parsed.manifest

  def do(self):
    if self.action   == 'install':
      self.install(self.manifest())
    elif self.action == 'status':
      self.status()
    elif self.action == 'update':
      self.update()
    elif self.action == 'upload':
      self.upload()
    elif self.action == 'chdir':
      self.chdir()

  def install(self,manifest_url):
    os.mkdir(CONFIG_PATH)
    os.mkdir(LOG_PATH)

    if os.access(MANIFEST_PATH,os.R_OK):
      self._error("sorry, manifest file already exists")
      exit(1)
    else:
      self._info('downloading manifest from "{}"...'.format(manifest_url))
      manifest = urllib2.urlopen(manifest_url).read()
      with open(MANIFEST_PATH,'w') as f:
        f.write(manifest)

    self.clone()

  def clone(self):
    self._info("started cloning...")
    def job(name,repo,url):
      if self._has_metadata(name,'cloned'):
        return False
      else:
        return Git().clone(url,repo)
    for name,cmd,logfile in self._in_repos(job):
      if cmd.skipped():
        self._print_result('already cloned')
      else:
        if cmd.exitcode == 0:
          self._assign_metadata(name,'cloned',datetime.utcnow().isoformat())
          self._print_result('ok')
        else:
          self._print_result('ERROR!',logfile)

  def status(self):
    self._info('repos status:')
    job = lambda name,repo,url: Git(repo).status()
    for _,cmd,_ in self._in_repos(job):
      if len(cmd.stdout) > 0:
        self._print_result('DIRTY')
      else:
        self._print_result("clean")

  def update(self):
    self._info('pulling from remotes...')
    job = lambda name,repo,url: Git(repo).pull()
    for _,cmd,logfile in self._in_repos(job):
      if cmd.exitcode == 0:
        self._print_result('ok')
      else:
        self._print_result('ERROR!',logfile)

  def upload(self):
    self._info('pushing to remotes...')
    job = lambda name,repo,url: Git(repo).push()
    for _,cmd,logfile in self._in_repos(job):
      if cmd.exitcode == 0:
        self._print_result('ok')
      else:
        self._print_result('ERROR!',logfile)

  def chdir(self):
    for dot in self.dots:
      name, url = dot
      repo = os.path.join(CONFIG_PATH,name)
      if self.config.dot == name:
        #os.chdir(repo)
        print repo

  def _info(self,text):
    sys.stdout.write(". "+text+"\n")

  def _error(self,text):
    sys.stderr.write(". "+text+"\n")

  def _parse_manifest(self):
    manifest_data = ConfigParser.ConfigParser()
    manifest_data.read(MANIFEST_PATH)
    self.dots = manifest_data.items('all')

  def _load_metadata(self):
    if os.access(METADATA_PATH,os.R_OK):
      with open(METADATA_PATH,'r') as m:
        self.metadata = json.loads(m.read())
    else:
        self.metadata = {}

    for repo,_ in self.dots:
      if not repo in self.metadata: self.metadata[repo]={}

  def _assign_metadata(self,repo,key,value):
    self.metadata[repo][key]=value

  def _has_metadata(self,repo,key):
    return key in self.metadata[repo]

  def _dump_metadata(self):
    with open(METADATA_PATH,'w') as m:
      m.write(json.dumps(self.metadata,sort_keys=True,indent=2))

  def _in_repos(self,job):
    self._parse_manifest()
    self._load_metadata()

    async = Async()
    for dot in self.dots:
      name, url = dot
      path = os.path.join(REPOS_PATH,name)
      async.add(name,job(name,path,url))

    for name,executor in async.run():
      logfile = os.path.join(LOG_PATH,name+'.log')
      with open(logfile,'w') as log:
        cmd = ' '.join(str(i) for i in executor.cmd)
        msg = "command: {}\nreturn code: {}\n{}".format(cmd,executor.exitcode,executor.stderr)
        log.write(msg)

      sys.stdout.write(name+': ')
      yield name,executor,logfile

    self._dump_metadata()

  def _print_result(self,text,logfile=None):
    if logfile:
      text = text + ' check out logfile: {}'.format(logfile)
    sys.stdout.write(text+"\n")

class Executor():
  def __init__(self,cmd):
    self.cmd      = cmd
    self._process = None
    self.stdout   = None
    self.stderr   = None
    self.exitcode = None
  def start(self):
    if self.cmd:
      self._process = subprocess.Popen(self.cmd,
          stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    return self
  def skipped(self):
    return self.cmd == False
  def join(self):
    if self._process:
      self.stdout,self.stderr = self._process.communicate()
      self.exitcode = self._process.returncode

class Async():
  def __init__(self):
    self.cmds={}
  def add(self,name,cmd):
    self.cmds[name]=cmd
  def run(self):
    subprocesses = {}
    for name,cmd in self.cmds.iteritems():
      subprocesses[name]=Executor(cmd).start()
    for t in subprocesses.itervalues(): t.join()
    for name,executor in subprocesses.iteritems():
      yield name,executor

class Git():

  def __init__(self,path=None):
    self.repo_path = path

  def clone(self,url,repo):
    return ['git', 'clone', '-q', '--recursive', '--', url, repo]

  def pull(self):
    return self._git() + ['pull']

  def push(self):
    return self._git() + ['push']

  def status(self):
    return self._git() + ['status', '--porcelain']

  def _git(self):
    return ['git'] + self._git_path_opts()

  def _git_path_opts(self):
    tmpl = ['--git-dir={}/.git', '--work-tree={}']
    return map(lambda s: s.format(self.repo_path),tmpl)

if __name__ == "__main__":
  Dot(sys.argv[1:]).do()
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-

import re
import unicodedata

def strip_accents(text):
	if text:
		return str(unicodedata.normalize('NFKD', text).encode('ASCII', 'ignore'),'utf-8')
	return ''

def trim(text):
	if text:
		return re.sub(r'\s+',' ',text.strip())
	return ''

def remove_punctuation(text, replace=''):
	return re.sub(r'[^\w\s]', replace, text)

def remove_double_letters(text, replace=''):
	if text:
		return replace.join([text[i] for i in range(len(text)-1) if text[i+1]!= text[i]]+[text[-1]])
	return ''

def remove_space_between_numbers(text, replace=''):
	if text:
		return re.sub(r'(?<=\d)[\s\\\-/]+(?=\d)', replace, text)
	return ''

def remove_urls(text, replace=''):
	if text:
		return re.sub(r'(((https?://)|(www))([-\w\.]+[-\w])+(:\d+)?(/([\w/_\.#-]*(\?\S+)?[^\.\s])?)?)', replace, text)
	return ''

def remove_email_addresses(text, replace=''):
	if text:
		return re.sub(r'[^@\s]*@[^@\s]*\.[^@\s]*', replace, text)
	return ''

def remove_html_tags(text, replace=''):
	if text:
		return re.sub(r'(<!--.*?-->|<[^>]*>)', replace, text)
	return ''

def find_hashtags(text):
	if text:
		return ['#{0}'.format(hashtag) for hashtag in re.compile(r'\B#([\w0-9_\-\']+)\b').findall(text)]
	return []

def find_mentions(text):
	if text:
		return ['@{0}'.format(mention) for mention in re.compile(r'\B@([\w0-9_\-\'\.]+)\b').findall(text)]
	return []

def find_urls(text):
	if text:
		return re.compile(r'\b(https?:\/\/(?:www\.|(?!www))[^\s\.]+\.[^\s]{2,}|www\.[^\s]+\.[^\s]{2,})').findall(text)
	return []

def vowel_harmony(word, vegyes=True):
	if word:
		mely	= re.compile('[aáoóuú]', re.IGNORECASE)
		magas	= re.compile('[eéiíöőüű]', re.IGNORECASE)
		mely_m	= len(mely.findall(word))
		magas_m	= len(magas.findall(word))
		if magas_m and mely_m:
			if vegyes:
				return 'vegyes'
			return 'magas'
		if magas_m>mely_m:
			return 'magas'
		return 'mely'
	return 'hiba'

def vowel_ending(word):
	if word:
		return (word[-1].lower() in ('a','á','e','é','i','í','o','ó','ö','ő','u','ú','ü','ű'))
	return False

#TODO: more contexts
def strip_context(text, context="search", including=None):
	if text:
		if context=='search':
			exclude		= re.compile(r'\b((a(z|rra)?)|(azok(ra)?)|(milyen)|(mennyi)|(mikor)|(hol)|(merre)|(hova)|([mk]i(vel|nek))|(mi?[eé]rt)|(r[aá])|(egy)|(mi(t|k(et)?)?)|(meg)|(be)|(nekem)|(hogy(an)?)|((sz[oó])?cikk\w*)|(oldal\w*)|([ií]r\w*)|(kapcsolat(os(an)?|ban))|(sz[oó]l[oó]?)|(keres\w*)|(n[eé]z[zd])|(mutas(s[aá][dl]|[sd]))|(alapj[aá]n)|(mond[dj]?)|(t[oö]ltse?d?)|(hoz([zd]|z[aá][dl]))|(nyis([ds]|s[aá][dl]))|(megnyit\w*)|((el)?olvas\w*)|(szeretn[eé]\w*)|(k[eé]r(ni|l?e[km]))|(megn[eé]z\w*)|(k[oö]z[oö]tt))\b', re.IGNORECASE)
			text		= exclude.sub('',text)
		elif context=='request':
			exclude		= re.compile(r'\b((a(z|rra)?)|(azok(ra)?)|([io]lya[a-z]*)|(am(elyik(ek)?|i)?ben?)|(a?mi(kor)?)|(a?hol)|(hogy)|(van(nak)?)|([mk]i(vel|nek))|(mi?[eé]rt)|(r[aá])|(egy)|(mi(t|k(et)?)?)|(meg)|(be)|(az(oka)?t)|(kell(ene)?)|(k[eé]ne)|(szeretn[eé][km])|(k[eé]rn?(([eé][km])|i)?)|(ad[dj]([aá][dl])?)|(nekem)|(van))\b', re.IGNORECASE)
			text		= exclude.sub('',text)
		if including:
			exclude		= re.compile(r''+including, re.IGNORECASE)
			text		= exclude.sub('',text)
	return trim(remove_punctuation(text))

# based on http://snowball.tartarus.org/algorithms/hungarian/stop.txt
# prepared by Anna Tordai
def remove_stopwords(text):
	if text:
		stopwords	= ['a', 'ahogy', 'ahol', 'aki', 'akik', 'akkor', 'alatt', 'által', 'altal', 'általában', 'altalaban', 'amely', 'amelyek', 'amelyekben', 'amelyeket', 'amelyet', 'amelynek', 'ami', 'amit', 'amolyan', 'amíg', 'amig', 'amikor', 'át', 'at', 'abban', 'ahhoz', 'annak', 'arra', 'arról', 'arrol', 'az', 'azok', 'azon', 'azt', 'azzal', 'azért', 'azert', 'aztán', 'aztan', 'azután', 'azutan', 'azonban', 'bár', 'bar', 'be', 'belül', 'belul', 'benne', 'cikk', 'cikkek', 'cikkeket', 'csak', 'de', 'e', 'eddig', 'egész', 'egesz', 'egy', 'egyes', 'egyetlen', 'egyéb', 'egyeb', 'egyik', 'egyre', 'ekkor', 'el', 'elég', 'eleg', 'ellen', 'elő', 'elo', 'először', 'eloszor', 'előtt', 'elott', 'első', 'elso', 'én', 'en', 'éppen', 'eppen', 'ebben', 'ehhez', 'emilyen', 'ennek', 'erre', 'ez', 'ezt', 'ezek', 'ezen', 'ezzel', 'ezért', 'ezert', 'és', 'es', 'fel', 'felé', 'fele', 'hanem', 'hiszen', 'hogy', 'hogyan', 'igen', 'így', 'igy', 'illetve', 'ill.', 'ill', 'ilyen', 'ilyenkor', 'ison', 'ismét', 'ismet', 'itt', 'jó', 'jo', 'jól', 'jol', 'jobban', 'kell', 'kellett', 'keresztül', 'keresztul', 'keressünk', 'keressunk', 'ki', 'kívül', 'kivul', 'között', 'kozott', 'közül', 'kozul', 'legalább', 'legalabb', 'lehet', 'lehetett', 'legyen', 'lenne', 'lenni', 'lesz', 'lett', 'maga', 'magát', 'magat', 'majd', 'majd', 'már', 'mar', 'más', 'mas', 'másik', 'masik', 'meg', 'még', 'meg', 'mellett', 'mert', 'mely', 'melyek', 'mi', 'mit', 'míg', 'mig', 'miért', 'miert', 'milyen', 'mikor', 'minden', 'mindent', 'mindenki', 'mindig', 'mint', 'mintha', 'mivel', 'most', 'nagy', 'nagyobb', 'nagyon', 'ne', 'néha', 'neha', 'nekem', 'neki', 'nem', 'néhány', 'nehany', 'nélkül', 'nelkul', 'nincs', 'olyan', 'ott', 'össze', 'ossze', 'ő', 'o', 'ők', 'ok', 'őket', 'oket', 'pedig', 'persze', 'rá', 'ra', 's', 'saját', 'sajat', 'sem', 'semmi', 'sok', 'sokat', 'sokkal', 'számára', 'szamara', 'szemben', 'szerint', 'szinte', 'talán', 'talan', 'tehát', 'tehat', 'teljes', 'tovább', 'tovabb', 'továbbá', 'tovabba', 'több', 'tobb', 'úgy', 'ugy', 'ugyanis', 'új', 'uj', 'újabb', 'ujabb', 'újra', 'ujra', 'után', 'utan', 'utána', 'utana', 'utolsó', 'utolso', 'vagy', 'vagyis', 'valaki', 'valami', 'valamint', 'való', 'valo', 'vagyok', 'van', 'vannak', 'volt', 'voltam', 'voltak', 'voltunk', 'vissza', 'vele', 'viszont', 'volna']
		for stopword in stopwords:
			text	= re.compile(r'\b'+stopword+'\b', re.IGNORECASE).sub('', text)
		return text
	return ''

def extract_message(text):
	extraction	= {
		"command"	: None,
		"arguments"	: [],
		"hashtags"	: [],
		"mentions"	: [],
		"urls"		: [],
	}
	if isinstance(text, str) and trim(text):
		if text[0] == '/':
			commands				= (str(text[1:]).strip()).split(" ")
			extraction['command']	= commands[0]
			if len(commands)>1:
				extraction['arguments']	= commands[1:]
		extraction['hashtags']	= find_hashtags(text)
		extraction['mentions']	= find_mentions(text)
		extraction['urls']		= find_urls(text)
	return extraction
<commit_msg>- improved stripping contexts<commit_after># -*- coding: UTF-8 -*-

import re
import unicodedata

def strip_accents(text):
	if text:
		return str(unicodedata.normalize('NFKD', text).encode('ASCII', 'ignore'),'utf-8')
	return ''

def trim(text):
	if text:
		return re.sub(r'\s+',' ',text.strip())
	return ''

def remove_punctuation(text, replace=''):
	return re.sub(r'[^\w\s]', replace, text)

def remove_double_letters(text, replace=''):
	if text:
		return replace.join([text[i] for i in range(len(text)-1) if text[i+1]!= text[i]]+[text[-1]])
	return ''

def remove_space_between_numbers(text, replace=''):
	if text:
		return re.sub(r'(?<=\d)[\s\\\-/]+(?=\d)', replace, text)
	return ''

def remove_urls(text, replace=''):
	if text:
		return re.sub(r'(((https?://)|(www))([-\w\.]+[-\w])+(:\d+)?(/([\w/_\.#-]*(\?\S+)?[^\.\s])?)?)', replace, text)
	return ''

def remove_email_addresses(text, replace=''):
	if text:
		return re.sub(r'[^@\s]*@[^@\s]*\.[^@\s]*', replace, text)
	return ''

def remove_html_tags(text, replace=''):
	if text:
		return re.sub(r'(<!--.*?-->|<[^>]*>)', replace, text)
	return ''

def find_hashtags(text):
	if text:
		return ['#{0}'.format(hashtag) for hashtag in re.compile(r'\B#([\w0-9_\-\']+)\b').findall(text)]
	return []

def find_mentions(text):
	if text:
		return ['@{0}'.format(mention) for mention in re.compile(r'\B@([\w0-9_\-\'\.]+)\b').findall(text)]
	return []

def find_urls(text):
	if text:
		return re.compile(r'\b(https?:\/\/(?:www\.|(?!www))[^\s\.]+\.[^\s]{2,}|www\.[^\s]+\.[^\s]{2,})').findall(text)
	return []

def vowel_harmony(word, vegyes=True):
	if word:
		mely	= re.compile('[aáoóuú]', re.IGNORECASE)
		magas	= re.compile('[eéiíöőüű]', re.IGNORECASE)
		mely_m	= len(mely.findall(word))
		magas_m	= len(magas.findall(word))
		if magas_m and mely_m:
			if vegyes:
				return 'vegyes'
			return 'magas'
		if magas_m>mely_m:
			return 'magas'
		return 'mely'
	return 'hiba'

def vowel_ending(word):
	if word:
		return (word[-1].lower() in ('a','á','e','é','i','í','o','ó','ö','ő','u','ú','ü','ű'))
	return False

#TODO: more contexts
def strip_context(text, context="search", including=None):
	if text:
		if context=='search':
			exclude		= re.compile(r'\b((a(z|rra)?)|(azok(ra)?)|(milyen)|(mennyi)|(mikor)|(hol)|(merre)|(hova)|([mk]i(vel|nek))|(mi?[eé]rt)|(r[aá])|(egy)|(mi(t|k(et)?)?)|(meg)|(be)|(nekem)|(hogy(an)?)|((sz[oó])?cikk\w*)|(oldal\w*)|([ií]r\w*)|(kapcsolat(os(an)?|ban))|(sz[oó]l[oó]?)|(keres\w*)|(n[eé]z[zd])|(mutas(s[aá][dl]|[sd]))|(alapj[aá]n)|(mond[dj]?)|(t[oö]ltse?d?)|(hoz([zd]|z[aá][dl]))|(nyis([ds]|s[aá][dl]))|(megnyit\w*)|((el)?olvas\w*)|(szeretn[eé]\w*)|(k[eé]r(ni|l?e[km]))|(megn[eé]z\w*)|(k[oö]z[oö]tt))\b', re.IGNORECASE)
			text		= exclude.sub('',text)
		elif context=='request':
			exclude		= re.compile(r'\b((a(z|rra)?)|(azok(ra)?)|([io]lya[a-z]*)|(am(elyik(ek)?|i)?ben?)|(a?mi(kor)?)|(a?hol)|(hogy)|(van(nak)?)|([mk]i(vel|nek))|(mi?[eé]rt)|(r[aá])|(egy)|(mi(t|k(et)?)?)|(meg)|(be)|(az(oka)?t)|(kell(ene)?)|(k[eé]ne)|(szeretn[eé][km])|(k[eé]rn?(([eé][km])|i)?)|(ad[dj]([aá][dl])?)|(nekem)|(van)|(k[uü]ld[dj]?[eé]?[dl]?))\b', re.IGNORECASE)
			text		= exclude.sub('',text)
		if including:
			exclude		= re.compile(r''+including, re.IGNORECASE)
			text		= exclude.sub('',text)
	return trim(remove_punctuation(text))

# based on http://snowball.tartarus.org/algorithms/hungarian/stop.txt
# prepared by Anna Tordai
def remove_stopwords(text):
	if text:
		stopwords	= ['a', 'ahogy', 'ahol', 'aki', 'akik', 'akkor', 'alatt', 'által', 'altal', 'általában', 'altalaban', 'amely', 'amelyek', 'amelyekben', 'amelyeket', 'amelyet', 'amelynek', 'ami', 'amit', 'amolyan', 'amíg', 'amig', 'amikor', 'át', 'at', 'abban', 'ahhoz', 'annak', 'arra', 'arról', 'arrol', 'az', 'azok', 'azon', 'azt', 'azzal', 'azért', 'azert', 'aztán', 'aztan', 'azután', 'azutan', 'azonban', 'bár', 'bar', 'be', 'belül', 'belul', 'benne', 'cikk', 'cikkek', 'cikkeket', 'csak', 'de', 'e', 'eddig', 'egész', 'egesz', 'egy', 'egyes', 'egyetlen', 'egyéb', 'egyeb', 'egyik', 'egyre', 'ekkor', 'el', 'elég', 'eleg', 'ellen', 'elő', 'elo', 'először', 'eloszor', 'előtt', 'elott', 'első', 'elso', 'én', 'en', 'éppen', 'eppen', 'ebben', 'ehhez', 'emilyen', 'ennek', 'erre', 'ez', 'ezt', 'ezek', 'ezen', 'ezzel', 'ezért', 'ezert', 'és', 'es', 'fel', 'felé', 'fele', 'hanem', 'hiszen', 'hogy', 'hogyan', 'igen', 'így', 'igy', 'illetve', 'ill.', 'ill', 'ilyen', 'ilyenkor', 'ison', 'ismét', 'ismet', 'itt', 'jó', 'jo', 'jól', 'jol', 'jobban', 'kell', 'kellett', 'keresztül', 'keresztul', 'keressünk', 'keressunk', 'ki', 'kívül', 'kivul', 'között', 'kozott', 'közül', 'kozul', 'legalább', 'legalabb', 'lehet', 'lehetett', 'legyen', 'lenne', 'lenni', 'lesz', 'lett', 'maga', 'magát', 'magat', 'majd', 'majd', 'már', 'mar', 'más', 'mas', 'másik', 'masik', 'meg', 'még', 'meg', 'mellett', 'mert', 'mely', 'melyek', 'mi', 'mit', 'míg', 'mig', 'miért', 'miert', 'milyen', 'mikor', 'minden', 'mindent', 'mindenki', 'mindig', 'mint', 'mintha', 'mivel', 'most', 'nagy', 'nagyobb', 'nagyon', 'ne', 'néha', 'neha', 'nekem', 'neki', 'nem', 'néhány', 'nehany', 'nélkül', 'nelkul', 'nincs', 'olyan', 'ott', 'össze', 'ossze', 'ő', 'o', 'ők', 'ok', 'őket', 'oket', 'pedig', 'persze', 'rá', 'ra', 's', 'saját', 'sajat', 'sem', 'semmi', 'sok', 'sokat', 'sokkal', 'számára', 'szamara', 'szemben', 'szerint', 'szinte', 'talán', 'talan', 'tehát', 'tehat', 'teljes', 'tovább', 'tovabb', 'továbbá', 'tovabba', 'több', 'tobb', 'úgy', 'ugy', 'ugyanis', 'új', 'uj', 'újabb', 'ujabb', 'újra', 'ujra', 'után', 'utan', 'utána', 'utana', 'utolsó', 'utolso', 'vagy', 'vagyis', 'valaki', 'valami', 'valamint', 'való', 'valo', 'vagyok', 'van', 'vannak', 'volt', 'voltam', 'voltak', 'voltunk', 'vissza', 'vele', 'viszont', 'volna']
		for stopword in stopwords:
			text	= re.compile(r'\b'+stopword+'\b', re.IGNORECASE).sub('', text)
		return text
	return ''

def extract_message(text):
	extraction	= {
		"command"	: None,
		"arguments"	: [],
		"hashtags"	: [],
		"mentions"	: [],
		"urls"		: [],
	}
	if isinstance(text, str) and trim(text):
		if text[0] == '/':
			commands				= (str(text[1:]).strip()).split(" ")
			extraction['command']	= commands[0]
			if len(commands)>1:
				extraction['arguments']	= commands[1:]
		extraction['hashtags']	= find_hashtags(text)
		extraction['mentions']	= find_mentions(text)
		extraction['urls']		= find_urls(text)
	return extraction
<|endoftext|>


<commit_before>from teafacto.index.wikipedia import WikipediaIndex
from teafacto.qa.multichoice.kaggleutils import *
from teafacto.core.utils import ticktock

def run(irwiki, load=False):
    tmpsdfp = "sdf.tmp.csv"
    tmpcdfp = "preds.csv"
    df = read()
    if not load:
        tdf = transform(df)
        irscorer = elemscorer(irwiki)
        sdf = scoredf(tdf, irscorer)
        print sdf
        sdf.to_csv(tmpsdfp)
    else:
        sdf = pd.DataFrame.from_csv(tmpsdfp)
    cdf = choose(sdf)
    cdf.to_csv(tmpcdfp)
    evalu(cdf, df)
    print sdf

def transform(df):
    df["question"] = df["question"].apply(lambda x: " ".join(x))
    df["answerA"] = df["answerA"].apply(lambda x: " ".join(x))
    df["answerB"] = df["answerB"].apply(lambda x: " ".join(x))
    df["answerC"] = df["answerC"].apply(lambda x: " ".join(x))
    df["answerD"] = df["answerD"].apply(lambda x: " ".join(x))
    df["x"] = df.apply(lambda row: " ".join(row), axis=1)
    tdf = pd.DataFrame()
    tdf["qA"] = df.apply(lambda row: row["question"] + " " + row["answerA"], axis=1)
    tdf["qB"] = df.apply(lambda row: row["question"] + " " + row["answerB"], axis=1)
    tdf["qC"] = df.apply(lambda row: row["question"] + " " + row["answerC"], axis=1)
    tdf["qD"] = df.apply(lambda row: row["question"] + " " + row["answerD"], axis=1)
    tdf.index = df.index
    return tdf

def scoredf(tdf, scorer):
    sdf = pd.DataFrame()
    c = 0
    tt = ticktock("Scorer")
    tt.tick()
    for i, row in tdf.iterrows():
        c += 1
        if c % 10 == 0:
            tt.tock("%d/%d" % (c, tdf.shape[0])).tick()
            break
        sdf = sdf.append(scorerow(row, scorer))
    #sdf.index = tdf.index
    #sdf = tdf.apply(lambda row: scorerow(row, scorer), axis=1)
    return sdf

def scorerow(row, scorer):
    return row.apply(lambda x: scorer(x))

def elemscorer(idx):
    idx = idx
    def scorer(el):
        res = idx.search(el)
        if len(res) > 0:
            res = res[0]
            return res["score"]
        else:
            return 0.0
    return scorer

def choose(sdf): # sdf only has qA, qB, qC, qD and the right index (question id's)
    ret = pd.DataFrame()
    ret["correctAnswer"] = sdf.idxmax(axis=1)
    ret.index = sdf.index
    return ret

def evalu(pred, orig):
    return (pred["correctAnswer"] == orig["correctAnswer"]).sum(axis=0)*1. / orig.shape[0]

if __name__ == "__main__":
    idx = WikipediaIndex(dir="../../../data/wikipedia/pagesidx/")
    run(idx)<commit_msg>ir baseline working<commit_after>from teafacto.index.wikipedia import WikipediaIndex
from teafacto.qa.multichoice.kaggleutils import *
from teafacto.core.utils import ticktock

def run(irwiki, load=False):
    tmpsdfp = "sdf.tmp.csv"
    tmpcdfp = "preds.csv"
    df = read()
    if not load:
        tdf = transform(df)
        irscorer = elemscorer(irwiki)
        sdf = scoredf(tdf, irscorer)
        print sdf
        sdf.to_csv(tmpsdfp)
    else:
        sdf = pd.DataFrame.from_csv(tmpsdfp)
    cdf = choose(sdf)
    print cdf
    cdf.to_csv(tmpcdfp)
    res = evalu(cdf, df)
    print res

def transform(df):
    df["question"] = df["question"].apply(lambda x: " ".join(x))
    df["answerA"] = df["answerA"].apply(lambda x: " ".join(x))
    df["answerB"] = df["answerB"].apply(lambda x: " ".join(x))
    df["answerC"] = df["answerC"].apply(lambda x: " ".join(x))
    df["answerD"] = df["answerD"].apply(lambda x: " ".join(x))
    tdf = pd.DataFrame()
    tdf["qA"] = df.apply(lambda row: row["question"] + " " + row["answerA"], axis=1)
    tdf["qB"] = df.apply(lambda row: row["question"] + " " + row["answerB"], axis=1)
    tdf["qC"] = df.apply(lambda row: row["question"] + " " + row["answerC"], axis=1)
    tdf["qD"] = df.apply(lambda row: row["question"] + " " + row["answerD"], axis=1)
    tdf.index = df.index
    return tdf

def scoredf(tdf, scorer):
    sdf = pd.DataFrame()
    c = 0
    tt = ticktock("Scorer")
    tt.tick()
    for i, row in tdf.iterrows():
        c += 1
        if c % 10 == 0:
            tt.tock("%d/%d" % (c, tdf.shape[0])).tick()
            break
        sdf = sdf.append(scorerow(row, scorer))
    #sdf.index = tdf.index
    #sdf = tdf.apply(lambda row: scorerow(row, scorer), axis=1)
    return sdf

def scorerow(row, scorer):
    return row.apply(lambda x: scorer(x))

def elemscorer(idx):
    idx = idx
    def scorer(el):
        res = idx.search(el)
        if len(res) > 0:
            res = res[0]
            return res["score"]
        else:
            return 0.0
    return scorer

def choose(sdf): # sdf only has qA, qB, qC, qD and the right index (question id's)
    ret = pd.DataFrame()
    ret["correctAnswer"] = sdf.idxmax(axis=1).apply(lambda x: x[1])
    ret.index = sdf.index
    return ret

def evalu(pred, orig):
    return (pred["correctAnswer"] == orig["correctAnswer"]).sum(axis=0)*1. / orig.shape[0]

if __name__ == "__main__":
    idx = WikipediaIndex(dir="../../../data/wikipedia/pagesidx/")
    run(idx, load=True)<|endoftext|>


<commit_before>"""
Task:
Descripion of script here.
"""

# Import Built-Ins
import logging

# Import Third-Party
import numpy as np

# Import Homebrew

# Init Logging Facilities
log = logging.getLogger(__name__)

def euc_distance(a, b):
    return np.abs(a - b) ** 2

def quantize(a,b):
    """
    S[k][i] - min(S[k]) / max(S[k]) - min(S[k])
    """

    ret_a = [(a[i] - min(a)) / (max(a) - min(a)) for i in range(len(a))]
    ret_b = [(b[i] - min(b)) / (max(b) - min(b)) for i in range(len(b))]

    return ret_a, ret_b

def get_lower_neighbors(xy, SM, index=False):
    x, y = xy
    lower_neighbors = []
    indices = []

    lower_neighbour_coords = [(x - 1, y), (x, y - 1), (x - 1, y - 1)]
    for coord in lower_neighbour_coords:
        if coord[0] >= 0 and coord[1] >= 0:
            try:
                if SM[coord] != 0:
                    lower_neighbors.append(SM[coord])
                    indices.append(coord)
            except IndexError:
                continue
    if index:
        return [(neighbor, i) for neighbor, i in zip(lower_neighbors, indices)]
    return lower_neighbors

def get_upper_neighbors(xy, SM):
    x, y = xy
    n, m = SM.shape
    upper_neighbors = []
    upper_neighbor_coords = [(x + 1, y), (x, y + 1), (x + 1, y + 1)]
    for coord in upper_neighbor_coords:
        if coord[0] < n and coord[1] < m:
            try:
                if SM[coord] != 0:
                    upper_neighbors.append(SM[coord])
            except IndexError:
                continue
    return upper_neighbors

def unblock_upper_neighbors(xy, SM, s, q):
    # unblock neighbors that are within SM
    n, m = SM.shape

    return SM

def populate_warp(s, q, res):
    n = len(s)
    m = len(q)
    S, Q = quantize(s, q)
    SM = np.zeros((n, m))

    lower_bound = 0
    upper_bound = res

    while 0 <= lower_bound <= 1 - res/2:
        # Gather indices that fulfill criteria
        idxS = [i for i in range(n) if (lower_bound <= S[i] <= upper_bound)]
        idxQ = [i for i in range(m) if (lower_bound <= Q[i] <= upper_bound)]

        # update bounds
        lower_bound += + res/2
        upper_bound = lower_bound + res

        # For indices, calculate euclidic distances and add to warp matrix SM
        for i in idxS:
            for j in idxQ:
                euc_d = euc_distance(s[i], q[j])
                if euc_d == 0:
                    SM[i, j] = -1
                else:
                    SM[i, j] = euc_d
    return SM

def calculat_warp_costs(s, q, SM):
    n, m = SM.shape
    for i, element in np.ndenumerate(SM):
        if SM[i] != 0:
            x, y = i

            # check lower neighbors
            lower_neighbors = get_lower_neighbors(i, SM)
            if lower_neighbors:
                min_cost = min(lower_neighbors) if not min(lower_neighbors) == -1 else 0
            else:
                min_cost = 0
            SM[i] = SM[i] + min_cost

            # Check upper neighbors
            upper_neighbors = get_upper_neighbors(i, SM)

            # Check for unblocked upper neighbors
            if len(upper_neighbors) == 0:
                upper_neighbor_coords = [(x + 1, y), (x, y + 1), (x + 1, y + 1)]
                for coord in upper_neighbor_coords:
                    if coord[0] < n and coord[1] < m:
                        try:
                            SM[coord] = euc_distance(s[coord[0]], q[coord[1]])
                        except IndexError:
                            continue
    return SM

def sparse_dtw(s, q, res=.5):
    n = len(s)
    m = len(q)

    #  Create Warp Matrix
    SM = populate_warp(s, q, res)

    #  Calculate Warp costs
    SM = calculat_warp_costs(s, q, SM)

    # Calculate warp path
    warping_path = []
    hop = (n-1, m-1)

    warping_path.append(hop)
    while hop != (0, 0):
        lower_neighbors = get_lower_neighbors(hop, SM, index=True)
        min_cost, next_hop = min(lower_neighbors, key=lambda x: x[0])
        hop = next_hop
        warping_path.append(hop)

    return warping_path

if __name__ == '__main__':
    S = [3, 4, 5, 3, 3]
    Q = [1, 2, 2, 1, 0]
    print(sparse_dtw(S, Q))<commit_msg>reverse output list<commit_after>"""
Task:
Descripion of script here.
"""

# Import Built-Ins
import logging

# Import Third-Party
import numpy as np

# Import Homebrew

# Init Logging Facilities
log = logging.getLogger(__name__)

def euc_distance(a, b):
    return np.abs(a - b) ** 2

def quantize(a,b):
    """
    S[k][i] - min(S[k]) / max(S[k]) - min(S[k])
    """

    ret_a = [(a[i] - min(a)) / (max(a) - min(a)) for i in range(len(a))]
    ret_b = [(b[i] - min(b)) / (max(b) - min(b)) for i in range(len(b))]

    return ret_a, ret_b

def get_lower_neighbors(xy, SM, index=False):
    x, y = xy
    lower_neighbors = []
    indices = []

    lower_neighbour_coords = [(x - 1, y), (x, y - 1), (x - 1, y - 1)]
    for coord in lower_neighbour_coords:
        if coord[0] >= 0 and coord[1] >= 0:
            try:
                if SM[coord] != 0:
                    lower_neighbors.append(SM[coord])
                    indices.append(coord)
            except IndexError:
                continue
    if index:
        return [(neighbor, i) for neighbor, i in zip(lower_neighbors, indices)]
    return lower_neighbors

def get_upper_neighbors(xy, SM):
    x, y = xy
    n, m = SM.shape
    upper_neighbors = []
    upper_neighbor_coords = [(x + 1, y), (x, y + 1), (x + 1, y + 1)]
    for coord in upper_neighbor_coords:
        if coord[0] < n and coord[1] < m:
            try:
                if SM[coord] != 0:
                    upper_neighbors.append(SM[coord])
            except IndexError:
                continue
    return upper_neighbors

def unblock_upper_neighbors(xy, SM, s, q):
    # unblock neighbors that are within SM
    n, m = SM.shape

    return SM

def populate_warp(s, q, res):
    n = len(s)
    m = len(q)
    S, Q = quantize(s, q)
    SM = np.zeros((n, m))

    lower_bound = 0
    upper_bound = res

    while 0 <= lower_bound <= 1 - res/2:
        # Gather indices that fulfill criteria
        idxS = [i for i in range(n) if (lower_bound <= S[i] <= upper_bound)]
        idxQ = [i for i in range(m) if (lower_bound <= Q[i] <= upper_bound)]

        # update bounds
        lower_bound += + res/2
        upper_bound = lower_bound + res

        # For indices, calculate euclidic distances and add to warp matrix SM
        for i in idxS:
            for j in idxQ:
                euc_d = euc_distance(s[i], q[j])
                if euc_d == 0:
                    SM[i, j] = -1
                else:
                    SM[i, j] = euc_d
    return SM

def calculat_warp_costs(s, q, SM):
    n, m = SM.shape
    for i, element in np.ndenumerate(SM):
        if SM[i] != 0:
            x, y = i

            # check lower neighbors
            lower_neighbors = get_lower_neighbors(i, SM)
            if lower_neighbors:
                min_cost = min(lower_neighbors) if not min(lower_neighbors) == -1 else 0
            else:
                min_cost = 0
            SM[i] = SM[i] + min_cost

            # Check upper neighbors
            upper_neighbors = get_upper_neighbors(i, SM)

            # Check for unblocked upper neighbors
            if len(upper_neighbors) == 0:
                upper_neighbor_coords = [(x + 1, y), (x, y + 1), (x + 1, y + 1)]
                for coord in upper_neighbor_coords:
                    if coord[0] < n and coord[1] < m:
                        try:
                            SM[coord] = euc_distance(s[coord[0]], q[coord[1]])
                        except IndexError:
                            continue
    return SM

def sparse_dtw(s, q, res=.5):
    n = len(s)
    m = len(q)

    #  Create Warp Matrix
    SM = populate_warp(s, q, res)

    #  Calculate Warp costs
    SM = calculat_warp_costs(s, q, SM)

    # Calculate warp path
    warping_path = []
    hop = (n-1, m-1)

    warping_path.append(hop)
    while hop != (0, 0):
        lower_neighbors = get_lower_neighbors(hop, SM, index=True)
        min_cost, next_hop = min(lower_neighbors, key=lambda x: x[0])
        hop = next_hop
        warping_path.append(hop)

    return warping_path.reverse()

if __name__ == '__main__':
    S = [3, 4, 5, 3, 3]
    Q = [1, 2, 2, 1, 0]
    print(sparse_dtw(S, Q))<|endoftext|>