<commit_before>from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.db.models import Q

from mozdns.address_record.models import AddressRecord
from mozdns.cname.models import CNAME
from mozdns.domain.models import Domain
from mozdns.mx.models import MX
from mozdns.nameserver.models import Nameserver
from mozdns.ptr.models import PTR
from mozdns.srv.models import SRV
from mozdns.sshfp.models import SSHFP
from mozdns.txt.models import TXT

from core.interface.static_intr.models import StaticInterface
from core.site.models import Site
from core.vlan.models import Vlan
from core.utils import two_to_four, IPFilter

from systems.models import System

from lexer import Lexer
import re
import pdb
import ipaddr

PR_UOP = 0
PR_LPAREN = 2
PR_RPAREN = 2
PR_AND = 1
PR_OR = 2
PR_TERM = -1

def build_filter(filter_, fields, filter_type = "icontains"):
    # rtucker++
    final_filter = Q()
    if filter_ and filter_[0] == '/':
        filter_ = filter_[1:]
        filter_type = "regex"
    for t in fields:
        final_filter = final_filter | Q(**{"{0}__{1}".format(t,
            filter_type): filter_})

    return final_filter

def build_text_qsets(f):
    # Alphabetical order
    q_sets = [
        build_filter(f, AddressRecord.search_fields),
        build_filter(f, CNAME.search_fields),
        build_filter(f, Domain.search_fields),
        build_filter(f, MX.search_fields),
        build_filter(f, Nameserver.search_fields),
        build_filter(f, PTR.search_fields),
        build_filter(f, SRV.search_fields),
        build_filter(f, SSHFP.search_fields),
        build_filter(f, StaticInterface.search_fields),
        build_filter(f, System.search_fields),
        build_filter(f, TXT.search_fields)
        ]
    return q_sets

def build_ipf_qsets(qset):
    # Alphabetical order
    q_sets = [
        qset,  # AddressRecord
        None,
        None,
        None,
        None,
        qset,  # PTR
        None,
        None,
        qset,  # StaticInterface
        None,
        None
        ]
    return q_sets

def build_rdtype_qsets(rdtype):
    """This function needs to filter out all records of a certain rdtype (like
    A or CNAME). Any filter produced here has to be able to be negated. We use
    the fact that every object has a pk > -1. When a qset is negated the query
    becomes pk <= -1.
    """
    rdtype = rdtype.upper()  # Let's get consistent
    select = Q(pk__gt=-1)
    no_select = Q(pk__lte=-1)
    q_sets = [
        select if rdtype == 'A' else no_select,
        select if rdtype == 'CNAME' else no_select,
        select if rdtype == 'DOMAIN' else no_select,
        select if rdtype == 'MX' else no_select,
        select if rdtype == 'NS' else no_select,
        select if rdtype == 'PTR' else no_select,
        select if rdtype == 'SRV' else no_select,
        select if rdtype == 'SSHFP' else no_select,
        select if rdtype == 'INTR' else no_select,
        select if rdtype == 'SYSTEM' else no_select,
        select if rdtype == 'TXT' else no_select
        ]
    return q_sets

class Token(object):
    def __init__(self, type_, value, precedence, col):
        self.value = value.strip()
        self.type_ = type_.title()
        self.precedence = precedence
        self.col = col
        self.types = [
                ('directive', re.compile("^(.*)=:(.*?)$")), # type, vlan, etc
                # It can't be ':' because ipv6 addresses use it
                ('text', re.compile("^(.*)$")), # Default to term
            ]

    def __str__(self):
        return "{0} {1}".format(self.type_, self.value)

    def __repr__(self):
        return str(self)

    def compile_Q(self):
        # Depending on the matched type, build the correct q set
        return build_text_qsets(self.value)

class BadDirective(Exception):
    pass

class DirectiveToken(Token):
    def __init__(self, type_, directive, value, precedence, col):
        self.directive = directive
        super(DirectiveToken, self).__init__(type_,
                value, precedence, col)

    def __str__(self):
        return "{0} {1}=:{2}".format(self.type_, self.directive, self.value)

    def compile_Q(self):
        if self.directive == 'network':
            if self.value.find(':') > -1:
                Klass = ipaddr.IPv6Network
                ip_type = '6'
            if self.value.find('.') > -1:
                Klass = ipaddr.IPv4Network
                ip_type = '4'
            try:
                network = Klass(self.value)
                ip_info = two_to_four(int(network.network),
                                      int(network.broadcast))
                ipf = IPFilter(None, ip_type, *ip_info)
            except (ipaddr.AddressValueError, ipaddr.NetmaskValueError), e:
                raise BadDirective("{0} isn't a valid "
                        "network.".format(self.value))
            return build_ipf_qsets(ipf.compile_Q())
        elif self.directive == 'type':
            return build_rdtype_qsets(self.value)
        elif self.directive == 'site':
            try:
                site = Site.objects.get(name=self.value)
            except ObjectDoesNotExist, e:
                raise BadDirective("{0} isn't a valid "
                        "site.".format(self.value))
            return build_ipf_qsets(site.compile_Q('4'))
        elif self.directive == 'vlan':
            try:
                if self.value.isdigit():
                    vlan = Vlan.objects.get(number=self.value)
                else:
                    vlan = Vlan.objects.get(name=self.value)
            except ObjectDoesNotExist, e:
                raise BadDirective("{0} isn't a valid "
                        "vlan identifier.".format(self.value))
            except MultipleObjectsReturned, e:
                raise BadDirective("{0} doesn't uniquely identify"
                        "a vlan.".format(self.value))
            return build_ipf_qsets(vlan.compile_Q('4'))
        else:
            raise BadDirective("Unknown directive "
                "'{0}'".format(self.directive))

class Tokenizer(object):
    def __init__(self, stmt):
        self.ll = Lexer(stmt)
        self.tokens = []
        self._get_tokens()
        self.idx = 0

    def pop(self):
        try:
            token = self.tokens[self.idx]
            self.idx += 1
            return token
        except IndexError:
            return None

    def top(self):
        try:
            token = self.tokens[self.idx]
            return token
        except IndexError:
            return None

    def peek(self):
        try:
            token = self.tokens[self.idx+1]
            return token
        except IndexError:
            return None

    # List of types
    # Operators And precedence rules
    patterns = [
        ('uop', re.compile("^(-)$"), PR_UOP), # NOT Unary Operator
        ('lparen', re.compile("^(\()$"), PR_LPAREN),  # Left paren
        ('rparen', re.compile("^(\))$"), PR_RPAREN), # Right paren
        ('bop', re.compile("(and)$", re.IGNORECASE), PR_AND), # AND Binary Operator
        ('bop', re.compile("(or)$", re.IGNORECASE), PR_OR), # OR Binary Operator
        ('directive', re.compile("^(.*)=:(.*?)$"), PR_TERM),
        ('term', re.compile("^(.*)$"), PR_TERM) # Everything else is a term
    ]

    def n(self):
        term = self.ll.lex()
        col = self.ll.pos
        if not term:
            return None
        for type_, pattern, precedence in self.patterns:
            match = pattern.match(term)
            if not match:
                continue
            if type_ == 'directive':
                # we need to make sure a trailing space doesn't through
                # us off.
                directive = match.groups(1)[0]
                value = match.groups(1)[1]
                if value:
                    return DirectiveToken(type_, directive, value, precedence,
                                          col)
                self.ll._lex_ws()
                nxt_token = self.ll.peek_token()
                for itype_, ipattern, iprecedence in self.patterns:
                    match = ipattern.match(nxt_token)
                    if not match:
                        continue
                    if itype_ == 'term':
                        value = self.ll.lex()  # we are eating this term
                        break
                    else:
                        break

                return DirectiveToken(type_, directive, value, precedence,
                                      col)
            if type_ == 'uop' or type_ == 'bop':
                opr = match.groups(1)[0]
                return Token(type_, opr.upper(), precedence, col)
            if type_ in ('lparen','rparen'):
                paren = match.groups(1)[0]
                return Token(type_, paren, precedence, col)
            if type_ == 'term':
                term = match.groups(1)[0]
                return Token(type_, term, precedence, col)

    def _get_tokens(self):
        first = True
        while True:
            token = self.n()
            if not token:
                self.tokens.append(None)
                break
            else:
                self.tokens.append(token)

        def add_ands(tokens):
            # When no operator exists we need to put an AND.
            new_tokens = []
            seen_term = False
            i = 0
            while i < (len(tokens) - 1):
                cur = tokens[i]
                nxt = tokens[i+1]
                if not nxt:
                    new_tokens.append(cur)
                    break

                if ((cur.type_ == 'Term' and nxt.type_ == 'Term')
                    or (cur.type_ == 'Term' and nxt.type_ == 'Directive')
                    or (cur.type_ == 'Directive' and nxt.type_ == 'Term')
                    or (cur.type_ == 'Directive' and nxt.type_ == 'Directive')
                    or (cur.type_ in ('Term', 'Directive')
                        and nxt.type_ == 'Uop')
                    or (cur.type_ in ('Term', 'Directive')
                        and nxt.type_ == 'Lparen')
                    or (nxt.type_ in ('Term', 'Directive')
                        and cur.type_ == 'Rparen')):
                    new_tokens.append(cur)
                    new_tokens.append(Token('Bop', 'AND', PR_AND, cur.col))
                    i += 1
                else:
                    new_tokens.append(cur)
                    i += 1
            return new_tokens

        def add_parens(tokens):
            new_tokens = []
            for token in self.tokens:
                if token.precedence > -1:
                    if token.type_ == 'Lparen':
                        for i in xrange(token.precedence):
                            new_tokens.append(Token('Lparen', '(',
                                token.precedence, None))
                        new_tokens.append(token)
                        continue
                    elif token.type_ == 'Rparen':
                        for i in xrange(token.precedence):
                            new_tokens.append(Token('Rparen', ')',
                                token.precedence, None))
                        new_tokens.append(token)
                    else:
                        for i in xrange(token.precedence):
                            new_tokens.append(Token('Rparen', ')',
                                token.precedence, None))
                        new_tokens.append(token)
                        for i in xrange(token.precedence):
                            new_tokens.append(Token('Lparen', '(',
                                token.precedence, None))
                else:
                    new_tokens.append(token)
            return new_tokens

        self.tokens = add_ands(self.tokens)
        self.tokens = add_parens(self.tokens)

def print_tokens(ss):
    tokenizer = Tokenizer(ss)
    print ss
    print tokenizer.tokens

if __name__ == "__main__":
    """

    ss = "webnode OR vlan:db,dmz site:scl4"
    print print_tokens(ss)

    print '---'
    ss = "webnode -vlan:db,dmz -site:scl4"
    print print_tokens(ss)

    print
    print '---'
    ss = "webnode vlan: site:scl4"
    print print_tokens(ss)

    print
    print '---'
    ss = "-webnode vlan: site:scl4"
    print print_tokens(ss)

    print
    print '---'
    ss = "-webnode vlan: site:"
    print print_tokens(ss)

    print
    print '---'
    ss = "webnode vlan:site:scl4"
    print print_tokens(ss)

    print
    print '---'
    ss = "webnode vlan:site:scl4"
    print print_tokens(ss)

    print
    print '---'
    ss = "v:s,y z:a, er asdf -dfddf"
    print print_tokens(ss)

    print
    print '---'
    ss = "(v:s,y z:a,) er asdf -dfddf"
    print print_tokens(ss)

    print
    print '---'

    ss = '"fooo"'
    ss = "foo: bar: baz"
    print print_tokens(ss)
    print
    print '---'

    ss = '"fooo"'
    ss = "() ()(bar ()(foo))baz"
    print print_tokens(ss)
    print
    print '---'

    ss = "(a AND (b AND c) AND a)"
    print print_tokens(ss)
    print
    print '---'
    ss = "(a (b c) a)"
    print print_tokens(ss)
    print
    print '---'
    ss = "()"
    print print_tokens(ss)
    print
    print '---'

    ss = "(foo)"
    print print_tokens(ss)
    ss = "(webnode vlan=:db vlan=: foo site:scl4)"
    print print_tokens(ss)
    ss = "(site=: vlan=: db)"
    print print_tokens(ss)

    ss = "(site=:db)"
    print print_tokens(ss)

    """
    ss = "(-site=: db)"
    print print_tokens(ss)
<commit_msg>implemented the 'view' directive.<commit_after>from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from django.db.models import Q

from mozdns.address_record.models import AddressRecord
from mozdns.cname.models import CNAME
from mozdns.domain.models import Domain
from mozdns.mx.models import MX
from mozdns.nameserver.models import Nameserver
from mozdns.ptr.models import PTR
from mozdns.srv.models import SRV
from mozdns.sshfp.models import SSHFP
from mozdns.txt.models import TXT
from mozdns.view.models import View

from core.interface.static_intr.models import StaticInterface
from core.site.models import Site
from core.vlan.models import Vlan
from core.utils import two_to_four, IPFilter

from systems.models import System

from lexer import Lexer
import re
import pdb
import ipaddr

PR_UOP = 0
PR_LPAREN = 2
PR_RPAREN = 2
PR_AND = 1
PR_OR = 2
PR_TERM = -1

def build_filter(filter_, fields, filter_type = "icontains"):
    # rtucker++
    final_filter = Q()
    if filter_ and filter_[0] == '/':
        filter_ = filter_[1:]
        filter_type = "regex"
    for t in fields:
        final_filter = final_filter | Q(**{"{0}__{1}".format(t,
            filter_type): filter_})

    return final_filter

def build_text_qsets(f):
    # Alphabetical order
    q_sets = [
        build_filter(f, AddressRecord.search_fields),
        build_filter(f, CNAME.search_fields),
        build_filter(f, Domain.search_fields),
        build_filter(f, MX.search_fields),
        build_filter(f, Nameserver.search_fields),
        build_filter(f, PTR.search_fields),
        build_filter(f, SRV.search_fields),
        build_filter(f, SSHFP.search_fields),
        build_filter(f, StaticInterface.search_fields),
        build_filter(f, System.search_fields),
        build_filter(f, TXT.search_fields)
        ]
    return q_sets

def build_ipf_qsets(qset):
    # Alphabetical order
    q_sets = [
        qset,  # AddressRecord
        None,
        None,
        None,
        None,
        qset,  # PTR
        None,
        None,
        qset,  # StaticInterface
        None,
        None
        ]
    return q_sets

def build_view_qsets(view_name):
    """
    """
    view_name = view_name.lower()  # Let's get consistent
    try:
        view = View.objects.get(name=view_name)
    except ObjectDoesNotExist:
        raise BadDirective("'{0}' isn't a valid view.".format(view_name))
    view_filter = Q(views=view) # This will slow queries down due to joins
    q_sets = [
        view_filter,
        view_filter,
        None, # Domain
        view_filter,
        view_filter,
        view_filter,
        view_filter,
        view_filter,
        view_filter, #INTR
        None, # System
        view_filter, #TXT
        ]
    return q_sets

class Token(object):
    def __init__(self, type_, value, precedence, col):
        self.value = value.strip()
        self.type_ = type_.title()
        self.precedence = precedence
        self.col = col
        self.types = [
                ('directive', re.compile("^(.*)=:(.*?)$")), # type, vlan, etc
                # It can't be ':' because ipv6 addresses use it
                ('text', re.compile("^(.*)$")), # Default to term
            ]

    def __str__(self):
        return "{0} {1}".format(self.type_, self.value)

    def __repr__(self):
        return str(self)

    def compile_Q(self):
        # Depending on the matched type, build the correct q set
        return build_text_qsets(self.value)

class BadDirective(Exception):
    pass

class DirectiveToken(Token):
    def __init__(self, type_, directive, value, precedence, col):
        self.directive = directive
        super(DirectiveToken, self).__init__(type_,
                value, precedence, col)

    def __str__(self):
        return "{0} {1}=:{2}".format(self.type_, self.directive, self.value)

    def compile_Q(self):
        if self.directive == 'network':
            # Todo move these directive processors into functions.
            if self.value.find(':') > -1:
                Klass = ipaddr.IPv6Network
                ip_type = '6'
            if self.value.find('.') > -1:
                Klass = ipaddr.IPv4Network
                ip_type = '4'
            try:
                network = Klass(self.value)
                ip_info = two_to_four(int(network.network),
                                      int(network.broadcast))
                ipf = IPFilter(None, ip_type, *ip_info)
            except (ipaddr.AddressValueError, ipaddr.NetmaskValueError), e:
                raise BadDirective("{0} isn't a valid "
                        "network.".format(self.value))
            return build_ipf_qsets(ipf.compile_Q())
        elif self.directive == 'view':
            return build_view_qsets(self.value)
        elif self.directive == 'type':
            return build_rdtype_qsets(self.value)
        elif self.directive == 'site':
            try:
                site = Site.objects.get(name=self.value)
            except ObjectDoesNotExist, e:
                raise BadDirective("{0} isn't a valid "
                        "site.".format(self.value))
            return build_ipf_qsets(site.compile_Q('4'))
        elif self.directive == 'vlan':
            try:
                if self.value.isdigit():
                    vlan = Vlan.objects.get(number=self.value)
                else:
                    vlan = Vlan.objects.get(name=self.value)
            except ObjectDoesNotExist, e:
                raise BadDirective("{0} isn't a valid "
                        "vlan identifier.".format(self.value))
            except MultipleObjectsReturned, e:
                raise BadDirective("{0} doesn't uniquely identify"
                        "a vlan.".format(self.value))
            return build_ipf_qsets(vlan.compile_Q('4'))
        else:
            raise BadDirective("Unknown directive "
                "'{0}'".format(self.directive))

class Tokenizer(object):
    def __init__(self, stmt):
        self.ll = Lexer(stmt)
        self.tokens = []
        self._get_tokens()
        self.idx = 0

    def pop(self):
        try:
            token = self.tokens[self.idx]
            self.idx += 1
            return token
        except IndexError:
            return None

    def top(self):
        try:
            token = self.tokens[self.idx]
            return token
        except IndexError:
            return None

    def peek(self):
        try:
            token = self.tokens[self.idx+1]
            return token
        except IndexError:
            return None

    # List of types
    # Operators And precedence rules
    patterns = [
        ('uop', re.compile("^(-)$"), PR_UOP), # NOT Unary Operator
        ('lparen', re.compile("^(\()$"), PR_LPAREN),  # Left paren
        ('rparen', re.compile("^(\))$"), PR_RPAREN), # Right paren
        ('bop', re.compile("(and)$", re.IGNORECASE), PR_AND), # AND Binary Operator
        ('bop', re.compile("(or)$", re.IGNORECASE), PR_OR), # OR Binary Operator
        ('directive', re.compile("^(.*)=:(.*?)$"), PR_TERM),
        ('term', re.compile("^(.*)$"), PR_TERM) # Everything else is a term
    ]

    def n(self):
        term = self.ll.lex()
        col = self.ll.pos
        if not term:
            return None
        for type_, pattern, precedence in self.patterns:
            match = pattern.match(term)
            if not match:
                continue
            if type_ == 'directive':
                # we need to make sure a trailing space doesn't through
                # us off.
                directive = match.groups(1)[0]
                value = match.groups(1)[1]
                if value:
                    return DirectiveToken(type_, directive, value, precedence,
                                          col)
                self.ll._lex_ws()
                nxt_token = self.ll.peek_token()
                for itype_, ipattern, iprecedence in self.patterns:
                    match = ipattern.match(nxt_token)
                    if not match:
                        continue
                    if itype_ == 'term':
                        value = self.ll.lex()  # we are eating this term
                        break
                    else:
                        break

                return DirectiveToken(type_, directive, value, precedence,
                                      col)
            if type_ == 'uop' or type_ == 'bop':
                opr = match.groups(1)[0]
                return Token(type_, opr.upper(), precedence, col)
            if type_ in ('lparen','rparen'):
                paren = match.groups(1)[0]
                return Token(type_, paren, precedence, col)
            if type_ == 'term':
                term = match.groups(1)[0]
                return Token(type_, term, precedence, col)

    def _get_tokens(self):
        first = True
        while True:
            token = self.n()
            if not token:
                self.tokens.append(None)
                break
            else:
                self.tokens.append(token)

        def add_ands(tokens):
            # When no operator exists we need to put an AND.
            new_tokens = []
            seen_term = False
            i = 0
            while i < (len(tokens) - 1):
                cur = tokens[i]
                nxt = tokens[i+1]
                if not nxt:
                    new_tokens.append(cur)
                    break

                if ((cur.type_ == 'Term' and nxt.type_ == 'Term')
                    or (cur.type_ == 'Term' and nxt.type_ == 'Directive')
                    or (cur.type_ == 'Directive' and nxt.type_ == 'Term')
                    or (cur.type_ == 'Directive' and nxt.type_ == 'Directive')
                    or (cur.type_ in ('Term', 'Directive')
                        and nxt.type_ == 'Uop')
                    or (cur.type_ in ('Term', 'Directive')
                        and nxt.type_ == 'Lparen')
                    or (nxt.type_ in ('Term', 'Directive')
                        and cur.type_ == 'Rparen')):
                    new_tokens.append(cur)
                    new_tokens.append(Token('Bop', 'AND', PR_AND, cur.col))
                    i += 1
                else:
                    new_tokens.append(cur)
                    i += 1
            return new_tokens

        def add_parens(tokens):
            new_tokens = []
            for token in self.tokens:
                if token.precedence > -1:
                    if token.type_ == 'Lparen':
                        for i in xrange(token.precedence):
                            new_tokens.append(Token('Lparen', '(',
                                token.precedence, None))
                        new_tokens.append(token)
                        continue
                    elif token.type_ == 'Rparen':
                        for i in xrange(token.precedence):
                            new_tokens.append(Token('Rparen', ')',
                                token.precedence, None))
                        new_tokens.append(token)
                    else:
                        for i in xrange(token.precedence):
                            new_tokens.append(Token('Rparen', ')',
                                token.precedence, None))
                        new_tokens.append(token)
                        for i in xrange(token.precedence):
                            new_tokens.append(Token('Lparen', '(',
                                token.precedence, None))
                else:
                    new_tokens.append(token)
            return new_tokens

        self.tokens = add_ands(self.tokens)
        self.tokens = add_parens(self.tokens)

def print_tokens(ss):
    tokenizer = Tokenizer(ss)
    print ss
    print tokenizer.tokens

if __name__ == "__main__":
    """

    ss = "webnode OR vlan:db,dmz site:scl4"
    print print_tokens(ss)

    print '---'
    ss = "webnode -vlan:db,dmz -site:scl4"
    print print_tokens(ss)

    print
    print '---'
    ss = "webnode vlan: site:scl4"
    print print_tokens(ss)

    print
    print '---'
    ss = "-webnode vlan: site:scl4"
    print print_tokens(ss)

    print
    print '---'
    ss = "-webnode vlan: site:"
    print print_tokens(ss)

    print
    print '---'
    ss = "webnode vlan:site:scl4"
    print print_tokens(ss)

    print
    print '---'
    ss = "webnode vlan:site:scl4"
    print print_tokens(ss)

    print
    print '---'
    ss = "v:s,y z:a, er asdf -dfddf"
    print print_tokens(ss)

    print
    print '---'
    ss = "(v:s,y z:a,) er asdf -dfddf"
    print print_tokens(ss)

    print
    print '---'

    ss = '"fooo"'
    ss = "foo: bar: baz"
    print print_tokens(ss)
    print
    print '---'

    ss = '"fooo"'
    ss = "() ()(bar ()(foo))baz"
    print print_tokens(ss)
    print
    print '---'

    ss = "(a AND (b AND c) AND a)"
    print print_tokens(ss)
    print
    print '---'
    ss = "(a (b c) a)"
    print print_tokens(ss)
    print
    print '---'
    ss = "()"
    print print_tokens(ss)
    print
    print '---'

    ss = "(foo)"
    print print_tokens(ss)
    ss = "(webnode vlan=:db vlan=: foo site:scl4)"
    print print_tokens(ss)
    ss = "(site=: vlan=: db)"
    print print_tokens(ss)

    ss = "(site=:db)"
    print print_tokens(ss)

    """
    ss = "(-site=: db)"
    print print_tokens(ss)
<|endoftext|>


<commit_before># -*- python -*-

# stdlib imports ---
import os
import os.path as osp

# waf imports ---
import waflib.Utils
import waflib.Logs as msg
from waflib.Configure import conf

#
_heptooldir = osp.dirname(osp.abspath(__file__))

def options(ctx):
    ctx.load('hep-waftools-base', tooldir=_heptooldir)
    return

def configure(ctx):
    ctx.load('hep-waftools-base', tooldir=_heptooldir)
    return

@conf
def find_posixlibs(ctx, **kwargs):

    if not ctx.env.CC or not ctx.env.CXX:
        msg.fatal('load a C or C++ compiler first')
        pass

    ctx.load('hep-waftools-base', tooldir=_heptooldir)

    # find libm
    ctx.check_with(
        ctx.check,
        "m",
        features='cxx cxxprogram',
        lib='m',
        uselib_store='m',
        **kwargs
        )

    # find dl
    ctx.check_with(
        ctx.check,
        "dl",
        features='cxx cxxprogram',
        lib='dl',
        uselib_store='dl',
        **kwargs
        )

    # find pthread
    ctx.check_with(
        ctx.check,
        "pthread",
        features='cxx cxxprogram',
        lib='pthread',
        uselib_store='pthread',
        **kwargs
        )

    # find libz
    ctx.check_with(
        ctx.check,
        "zlib",
        features='cxx cxxprogram',
        header_name="zlib.h",
        lib='z',
        uselib_store='z',
        **kwargs
        )

    # find bfd
    ctx.check_with(
        ctx.check,
        "bfd",
        features='cxx cxxprogram',
        header_name="bfd.h",
        lib='bfd',
        uselib_store='bfd',
        **kwargs
        )

    # find rt
    if not ctx.is_darwin():
        ctx.check_with(
            ctx.check,
            "rt",
            features='cxx cxxprogram',
            lib='rt',
            uselib_store='rt',
            **kwargs
            )
        pass

    # test bfd
    ctx.check_cxx(
        msg="Checking bfd_init",
        okmsg="ok",
        fragment='''\
        #include "bfd.h"
        #include <iostream>

        int main(int argc, char* argv[]) {
          bfd_init();
          bfd_error_type err = bfd_get_error();
          return (int)err;
        }
        ''',
        use="bfd",
        execute  = True,
        mandatory= True,
        )

    ctx.env.HEPWAF_FOUND_POSIXLIBS = 1
    return

## EOF ##
<commit_msg>bfd: define PACKAGE and PACKAGE_VERSION which are needed on some linux-binutils and macports/xcode installations<commit_after># -*- python -*-

# stdlib imports ---
import os
import os.path as osp

# waf imports ---
import waflib.Utils
import waflib.Logs as msg
from waflib.Configure import conf

#
_heptooldir = osp.dirname(osp.abspath(__file__))

def options(ctx):
    ctx.load('hep-waftools-base', tooldir=_heptooldir)
    return

def configure(ctx):
    ctx.load('hep-waftools-base', tooldir=_heptooldir)
    return

@conf
def find_posixlibs(ctx, **kwargs):

    if not ctx.env.CC or not ctx.env.CXX:
        msg.fatal('load a C or C++ compiler first')
        pass

    ctx.load('hep-waftools-base', tooldir=_heptooldir)

    # find libm
    ctx.check_with(
        ctx.check,
        "m",
        features='cxx cxxprogram',
        lib='m',
        uselib_store='m',
        **kwargs
        )

    # find dl
    ctx.check_with(
        ctx.check,
        "dl",
        features='cxx cxxprogram',
        lib='dl',
        uselib_store='dl',
        **kwargs
        )

    # find pthread
    ctx.check_with(
        ctx.check,
        "pthread",
        features='cxx cxxprogram',
        lib='pthread',
        uselib_store='pthread',
        **kwargs
        )

    # find libz
    ctx.check_with(
        ctx.check,
        "zlib",
        features='cxx cxxprogram',
        header_name="zlib.h",
        lib='z',
        uselib_store='z',
        **kwargs
        )

    # find bfd
    ctx.check_with(
        ctx.check,
        "bfd",
        features='cxx cxxprogram',
        defines=['PACKAGE=1','PACKAGE_VERSION=1',],
        header_name="bfd.h",
        lib='bfd',
        uselib_store='bfd',
        use='dl',
        **kwargs
        )

    # find rt
    if not ctx.is_darwin():
        ctx.check_with(
            ctx.check,
            "rt",
            features='cxx cxxprogram',
            lib='rt',
            uselib_store='rt',
            **kwargs
            )
        pass

    # test bfd
    ctx.check_cxx(
        msg="Checking bfd_init",
        okmsg="ok",
        fragment='''\
        #include "bfd.h"
        #include <iostream>

        int main(int argc, char* argv[]) {
          bfd_init();
          bfd_error_type err = bfd_get_error();
          return (int)err;
        }
        ''',
        use="bfd dl",
        #defines=['PACKAGE=1','PACKAGE_VERSION=1',],
        execute  = True,
        mandatory= True,
        )

    ctx.env.HEPWAF_FOUND_POSIXLIBS = 1
    return

## EOF ##
<|endoftext|>


<commit_before>#from ete2 import NCBITaxa
import argparse
import sys, getopt
import glob, os
import pandas as p
import numpy as np
import re
import operator
import gzip

from collections import defaultdict
from collections import Counter
import logging

# These are identities normalized with query coverage:
MIN_IDENTITY_TAXA = (0.40,0.50,0.60,0.70,0.80,0.90,0.95)

# This is local to aligned segment identity:
MIN_IDENTITY = 0.40

# No limit in nr of matches from file to be used
MAX_MATCHES = 1e100

def read_blast_input(blastinputfile,lengths, accession_mode=False):
    #k191_83_2       gi|973180054|gb|KUL19018.1|     71.2    73      21      0       9       81      337     409     6.6e-24 118.2

    #queryId, subjectId, percIdentity, alnLength, mismatchCount, gapOpenCount, queryStart, queryEnd, subjectStart, subjectEnd, eVal, bitScore

    matches = defaultdict(list)
    gids = Counter()
    nmatches = Counter();

    for line in open(blastinputfile):
        line = line.rstrip()

        (queryId, subjectId, percIdentity, alnLength, mismatchCount,
        gapOpenCount, queryStart, queryEnd, subjectStart, subjectEnd, eVal, bitScore) = line.split("\t")

        if accession_mode:
            gid = subjectId
        else:
            m = re.search(r"gi\|(.*?)\|.*", subjectId)
            gid = m.group(1)
        qLength = lengths[queryId]

        alnLength_in_query = abs(int(queryEnd) - int(queryStart)) + 1
        fHit = float(alnLength_in_query)/qLength
        fHit *= float(percIdentity)/100.0
        fHit = min(1.0,fHit)
        #hits[queryId] = hits[queryId] + 1
        if float(percIdentity) > MIN_IDENTITY and nmatches[queryId] < MAX_MATCHES:
            matches[queryId].append((gid,fHit))
            nmatches[queryId] += 1
            gids[gid] +=1

    return (matches, gids.keys())

def read_lineage_file(lineage_file):

    mapping = {}
    mapBack = defaultdict(lambda: defaultdict(list))
    for line in open(lineage_file):
        line = line.rstrip()

        tokens = line.split("\t")
        (taxaid, domainid, phylumid, classid, orderid, familyid, genusid, speciesid) = tokens

        mapping[int(taxaid)]=[domainid, phylumid, classid, orderid, familyid, genusid, speciesid];
        tokens.pop(0)
        for depth in range(6,0,-1):
            if tokens[depth] not in mapBack[depth] and tokens[depth] != 'None':
                for depth2 in range(depth - 1,-1,-1):
                    mapBack[depth][tokens[depth]].append(tokens[depth2])
    return (mapping,mapBack)

def read_query_length_file(query_length_file):

    lengths = {}

    for line in open(query_length_file):
        line = line.rstrip()

        (queryid, length) = line.split("\t")

        lengths[queryid] = float(length)

    return lengths

def map_gids_binary(gids, mapping_file):

    t = open(mapping_file, 'r')
    c = 0
    size = long(os.path.getsize(mapping_file))
    mapping = {}

    for gis in gids: #binary search #########################################
        c=c+1
        #print gis
        found=False
        offset=0
        chunk=size
        pos=chunk/2
        while found == False and chunk>0:
            chunk = chunk/2
            t.seek(pos)
            t.readline()
            entry = t.readline().split("\t")
            if entry[0]:
                filegi = int(entry[0])
                filetax = int(entry[1].rstrip("\n"))

                if filegi == int(gis):
                    answer = filetax
                    found = True
                elif filegi > int(gis):
                    pos = offset +(chunk/2)
                elif filegi < int(gis):
                    offset = offset+chunk
                    pos = pos + (chunk/2)

        if found == False:
            answer = -1

        mapping[gis] = answer

    return mapping

def map_accessions(accs, mapping_file):
    first = True
    mappings = dict([(acc, -1) for acc in accs])
    with open(mapping_file) as mapping_fh:
        for line in mapping_fh:
            if first:
                first = False
                continue

            _, acc_ver, taxid, _ = line.split("\t")
            # Only add taxids for the given acc
            if acc_ver in mappings:
                mappings[acc_ver] = int(taxid)

    return mappings

def main(argv):

    parser = argparse.ArgumentParser()

    parser.add_argument("blast_input_file", help="directory with blast 6 matches to taxaid database *.b6")

    parser.add_argument("query_length_file", help="tab delimited file of query lengths")

    parser.add_argument('-g','--gid_taxaid_mapping_file', help="mapping from gid to taxaid gzipped")

    parser.add_argument('-a','--acc_taxaid_mapping_file', help="mapping from accession to taxaid gzipped")

    parser.add_argument('-l','--lineage_file', help="text taxaid to lineage mapping")

    parser.add_argument('-f','--min_fraction', default=0.5, type=float, help="Minimum fraction of weights needed to assign to a particular level, default 0.5. 0.9 would be more strict.")

    parser.add_argument('-o','--output_dir', type=str, default="output",
        help=("string specifying output directory and file stubs"))

    args = parser.parse_args()

    if args.gid_taxaid_mapping_file and args.acc_taxaid_mapping_file:
        raise Exception("Both gid_taxaid_mapping_file and acc_taxaid_mapping_file are given, but only one at a time is allowed")
    elif args.gid_taxaid_mapping_file:
        accession_mode = False
    else:
        accession_mode = True

    lengths = read_query_length_file(args.query_length_file)
    logging.info("Finished reading lengths file")

    (matches,gids) = read_blast_input(args.blast_input_file,lengths,accession_mode)
    logging.info("Finished reading in blast results file")

    (lineages,mapBack) = read_lineage_file(args.lineage_file)
    logging.info("Finished reading in lineage file")

    if accession_mode:
        mapping = map_accessions(gids, args.acc_taxaid_mapping_file)
    else:
        mapping = map_gids_binary(gids, args.gid_taxaid_mapping_file)
    logging.info("Finished loading taxaid map file")

    geneAssign = defaultdict(dict)
    contigLengths = Counter()
    contigAssignDepth = list()
    for depth in range(7):
            contigAssignDepth.append(defaultdict(lambda: Counter()))

    contigGenes = defaultdict(list)
    for gene, matchs in matches.items():
        #print str(gene)
        m = re.search(r"(.*)_\d+", gene)
        contig = m.group(1)
        contigLengths[contig] += lengths[gene]
        contigGenes[contig].append(gene)

        collate_hits = list()
        for depth in range(7):
            collate_hits.append(Counter())

        added_matches = set()
        for (match,fHit) in sorted(matchs, key=lambda x: x[1], reverse=True):

            if mapping[match] > -1:
                tax_id = mapping[match]
                if tax_id not in added_matches:     # Only add the best hit per species
                    added_matches.add(tax_id)
                    if tax_id not in lineages:
                        logging.warning("Taxa id {} is missing from lineage file".format(tax_id))
                        continue
                    hits = lineages[tax_id]
                    for depth in range(7):
                        if hits[depth] != "None":
                            weight = (fHit - MIN_IDENTITY_TAXA[depth])/(1.0 - MIN_IDENTITY_TAXA[depth])
                            weight = max(weight,0.0)
                            if weight > 0.0:
                                collate_hits[depth][hits[depth]] += weight #could put a transform in here

        #import ipdb; ipdb.set_trace()
        for depth in range(6,-1,-1):
            collate = collate_hits[depth]
            dWeight = sum(collate.values())

            sortCollate = sorted(collate.items(), key=operator.itemgetter(1),reverse=True)
            nL = len(collate)
            if nL > 0:
                dP = 0.0
                if dWeight > 0.0:
                    dP = float(sortCollate[0][1])/dWeight

                    if dP > args.min_fraction:
                        geneAssign[gene][depth] = (sortCollate[0][0],dP)
                        assignBack = mapBack[depth][sortCollate[0][0]]
                        depth2 = depth -1
                        for assignB in assignBack:
                            geneAssign[gene][depth2] = (assignB,1.0)
                            depth2 -= 1

                        break
                    else:
                        geneAssign[gene][depth] = ('No hits', -1.0)
                else:
                    geneAssign[gene][depth] = ('No hits', -1.0)
            else:
                geneAssign[gene][depth] = ('No hits',-1.0)

    with open(args.output_dir+"_genes.csv", "w") as text_file:
        for gene in geneAssign.keys():
            text_file.write('%s'%gene)
            for depth in range(7):
                (assign,p) = geneAssign[gene][depth]
                text_file.write(',%s->%.3f'%(assign,p))
            text_file.write('\n')
            text_file.flush()

    #collate contig predictions
    #import ipdb; ipdb.set_trace()

    contigAssign = defaultdict(dict)

    for contig, genes in contigGenes.iteritems():

        collate_hits = list()
        for depth in range(7):
            collate_hits.append(Counter())

        for gene in genes:

            for depth in range(7):
                (assignhit, genef) = geneAssign[gene][depth]

                if assignhit != 'No hits':
                    collate_hits[depth][assignhit] += lengths[gene]#*genef

        # Contigs are assigned a taxonomy based on LCA for
        # all genes assigned on at least kingdom level.
        dWeight = sum(collate_hits[0].values())
        for depth in range(7):
            collate = collate_hits[depth]
            sortCollate = sorted(collate.items(), key=operator.itemgetter(1),reverse=True)
            nL = len(collate)
            if nL > 0:
                dP = 0.0
                if dWeight > 0.0:
                    dP = float(sortCollate[0][1])/dWeight
                    if dP > args.min_fraction:
                        contigAssign[contig][depth] = (sortCollate[0][0],dP,sortCollate[0][1])
                    else:
                        contigAssign[contig][depth] = ('No hits',0.,0.)
                else:
                    contigAssign[contig][depth] = ('No hits',0.,0.)
            else:
                contigAssign[contig][depth] = ('No hits',0.,0.)

    with open(args.output_dir+"_contigs.csv", "w") as text_file:
        for contig in contigAssign.keys():
            text_file.write('%s,%f'%(contig,contigLengths[contig]))
            for depth in range(7):
                (assign,p,dF) = contigAssign[contig][depth]
                dFN = dF/contigLengths[contig]
                text_file.write(',%s->%.3f->%.3f'%(assign,p,dFN))
            text_file.write('\n')
            text_file.flush()

if __name__ == "__main__":
    main(sys.argv[1:])

<commit_msg>Fixed another used of iteritems() to items()<commit_after>#from ete2 import NCBITaxa
import argparse
import sys, getopt
import glob, os
import pandas as p
import numpy as np
import re
import operator
import gzip

from collections import defaultdict
from collections import Counter
import logging

# These are identities normalized with query coverage:
MIN_IDENTITY_TAXA = (0.40,0.50,0.60,0.70,0.80,0.90,0.95)

# This is local to aligned segment identity:
MIN_IDENTITY = 0.40

# No limit in nr of matches from file to be used
MAX_MATCHES = 1e100

def read_blast_input(blastinputfile,lengths, accession_mode=False):
    #k191_83_2       gi|973180054|gb|KUL19018.1|     71.2    73      21      0       9       81      337     409     6.6e-24 118.2

    #queryId, subjectId, percIdentity, alnLength, mismatchCount, gapOpenCount, queryStart, queryEnd, subjectStart, subjectEnd, eVal, bitScore

    matches = defaultdict(list)
    gids = Counter()
    nmatches = Counter();

    for line in open(blastinputfile):
        line = line.rstrip()

        (queryId, subjectId, percIdentity, alnLength, mismatchCount,
        gapOpenCount, queryStart, queryEnd, subjectStart, subjectEnd, eVal, bitScore) = line.split("\t")

        if accession_mode:
            gid = subjectId
        else:
            m = re.search(r"gi\|(.*?)\|.*", subjectId)
            gid = m.group(1)
        qLength = lengths[queryId]

        alnLength_in_query = abs(int(queryEnd) - int(queryStart)) + 1
        fHit = float(alnLength_in_query)/qLength
        fHit *= float(percIdentity)/100.0
        fHit = min(1.0,fHit)
        #hits[queryId] = hits[queryId] + 1
        if float(percIdentity) > MIN_IDENTITY and nmatches[queryId] < MAX_MATCHES:
            matches[queryId].append((gid,fHit))
            nmatches[queryId] += 1
            gids[gid] +=1

    return (matches, gids.keys())

def read_lineage_file(lineage_file):

    mapping = {}
    mapBack = defaultdict(lambda: defaultdict(list))
    for line in open(lineage_file):
        line = line.rstrip()

        tokens = line.split("\t")
        (taxaid, domainid, phylumid, classid, orderid, familyid, genusid, speciesid) = tokens

        mapping[int(taxaid)]=[domainid, phylumid, classid, orderid, familyid, genusid, speciesid];
        tokens.pop(0)
        for depth in range(6,0,-1):
            if tokens[depth] not in mapBack[depth] and tokens[depth] != 'None':
                for depth2 in range(depth - 1,-1,-1):
                    mapBack[depth][tokens[depth]].append(tokens[depth2])
    return (mapping,mapBack)

def read_query_length_file(query_length_file):

    lengths = {}

    for line in open(query_length_file):
        line = line.rstrip()

        (queryid, length) = line.split("\t")

        lengths[queryid] = float(length)

    return lengths

def map_gids_binary(gids, mapping_file):

    t = open(mapping_file, 'r')
    c = 0
    size = long(os.path.getsize(mapping_file))
    mapping = {}

    for gis in gids: #binary search #########################################
        c=c+1
        #print gis
        found=False
        offset=0
        chunk=size
        pos=chunk/2
        while found == False and chunk>0:
            chunk = chunk/2
            t.seek(pos)
            t.readline()
            entry = t.readline().split("\t")
            if entry[0]:
                filegi = int(entry[0])
                filetax = int(entry[1].rstrip("\n"))

                if filegi == int(gis):
                    answer = filetax
                    found = True
                elif filegi > int(gis):
                    pos = offset +(chunk/2)
                elif filegi < int(gis):
                    offset = offset+chunk
                    pos = pos + (chunk/2)

        if found == False:
            answer = -1

        mapping[gis] = answer

    return mapping

def map_accessions(accs, mapping_file):
    first = True
    mappings = dict([(acc, -1) for acc in accs])
    with open(mapping_file) as mapping_fh:
        for line in mapping_fh:
            if first:
                first = False
                continue

            _, acc_ver, taxid, _ = line.split("\t")
            # Only add taxids for the given acc
            if acc_ver in mappings:
                mappings[acc_ver] = int(taxid)

    return mappings

def main(argv):

    parser = argparse.ArgumentParser()

    parser.add_argument("blast_input_file", help="directory with blast 6 matches to taxaid database *.b6")

    parser.add_argument("query_length_file", help="tab delimited file of query lengths")

    parser.add_argument('-g','--gid_taxaid_mapping_file', help="mapping from gid to taxaid gzipped")

    parser.add_argument('-a','--acc_taxaid_mapping_file', help="mapping from accession to taxaid gzipped")

    parser.add_argument('-l','--lineage_file', help="text taxaid to lineage mapping")

    parser.add_argument('-f','--min_fraction', default=0.5, type=float, help="Minimum fraction of weights needed to assign to a particular level, default 0.5. 0.9 would be more strict.")

    parser.add_argument('-o','--output_dir', type=str, default="output",
        help=("string specifying output directory and file stubs"))

    args = parser.parse_args()

    if args.gid_taxaid_mapping_file and args.acc_taxaid_mapping_file:
        raise Exception("Both gid_taxaid_mapping_file and acc_taxaid_mapping_file are given, but only one at a time is allowed")
    elif args.gid_taxaid_mapping_file:
        accession_mode = False
    else:
        accession_mode = True

    lengths = read_query_length_file(args.query_length_file)
    logging.info("Finished reading lengths file")

    (matches,gids) = read_blast_input(args.blast_input_file,lengths,accession_mode)
    logging.info("Finished reading in blast results file")

    (lineages,mapBack) = read_lineage_file(args.lineage_file)
    logging.info("Finished reading in lineage file")

    if accession_mode:
        mapping = map_accessions(gids, args.acc_taxaid_mapping_file)
    else:
        mapping = map_gids_binary(gids, args.gid_taxaid_mapping_file)
    logging.info("Finished loading taxaid map file")

    geneAssign = defaultdict(dict)
    contigLengths = Counter()
    contigAssignDepth = list()
    for depth in range(7):
            contigAssignDepth.append(defaultdict(lambda: Counter()))

    contigGenes = defaultdict(list)
    for gene, matchs in matches.items():
        #print str(gene)
        m = re.search(r"(.*)_\d+", gene)
        contig = m.group(1)
        contigLengths[contig] += lengths[gene]
        contigGenes[contig].append(gene)

        collate_hits = list()
        for depth in range(7):
            collate_hits.append(Counter())

        added_matches = set()
        for (match,fHit) in sorted(matchs, key=lambda x: x[1], reverse=True):

            if mapping[match] > -1:
                tax_id = mapping[match]
                if tax_id not in added_matches:     # Only add the best hit per species
                    added_matches.add(tax_id)
                    if tax_id not in lineages:
                        logging.warning("Taxa id {} is missing from lineage file".format(tax_id))
                        continue
                    hits = lineages[tax_id]
                    for depth in range(7):
                        if hits[depth] != "None":
                            weight = (fHit - MIN_IDENTITY_TAXA[depth])/(1.0 - MIN_IDENTITY_TAXA[depth])
                            weight = max(weight,0.0)
                            if weight > 0.0:
                                collate_hits[depth][hits[depth]] += weight #could put a transform in here

        #import ipdb; ipdb.set_trace()
        for depth in range(6,-1,-1):
            collate = collate_hits[depth]
            dWeight = sum(collate.values())

            sortCollate = sorted(collate.items(), key=operator.itemgetter(1),reverse=True)
            nL = len(collate)
            if nL > 0:
                dP = 0.0
                if dWeight > 0.0:
                    dP = float(sortCollate[0][1])/dWeight

                    if dP > args.min_fraction:
                        geneAssign[gene][depth] = (sortCollate[0][0],dP)
                        assignBack = mapBack[depth][sortCollate[0][0]]
                        depth2 = depth -1
                        for assignB in assignBack:
                            geneAssign[gene][depth2] = (assignB,1.0)
                            depth2 -= 1

                        break
                    else:
                        geneAssign[gene][depth] = ('No hits', -1.0)
                else:
                    geneAssign[gene][depth] = ('No hits', -1.0)
            else:
                geneAssign[gene][depth] = ('No hits',-1.0)

    with open(args.output_dir+"_genes.csv", "w") as text_file:
        for gene in geneAssign.keys():
            text_file.write('%s'%gene)
            for depth in range(7):
                (assign,p) = geneAssign[gene][depth]
                text_file.write(',%s->%.3f'%(assign,p))
            text_file.write('\n')
            text_file.flush()

    #collate contig predictions
    #import ipdb; ipdb.set_trace()

    contigAssign = defaultdict(dict)

    for contig, genes in contigGenes.items():

        collate_hits = list()
        for depth in range(7):
            collate_hits.append(Counter())

        for gene in genes:

            for depth in range(7):
                (assignhit, genef) = geneAssign[gene][depth]

                if assignhit != 'No hits':
                    collate_hits[depth][assignhit] += lengths[gene]#*genef

        # Contigs are assigned a taxonomy based on LCA for
        # all genes assigned on at least kingdom level.
        dWeight = sum(collate_hits[0].values())
        for depth in range(7):
            collate = collate_hits[depth]
            sortCollate = sorted(collate.items(), key=operator.itemgetter(1),reverse=True)
            nL = len(collate)
            if nL > 0:
                dP = 0.0
                if dWeight > 0.0:
                    dP = float(sortCollate[0][1])/dWeight
                    if dP > args.min_fraction:
                        contigAssign[contig][depth] = (sortCollate[0][0],dP,sortCollate[0][1])
                    else:
                        contigAssign[contig][depth] = ('No hits',0.,0.)
                else:
                    contigAssign[contig][depth] = ('No hits',0.,0.)
            else:
                contigAssign[contig][depth] = ('No hits',0.,0.)

    with open(args.output_dir+"_contigs.csv", "w") as text_file:
        for contig in contigAssign.keys():
            text_file.write('%s,%f'%(contig,contigLengths[contig]))
            for depth in range(7):
                (assign,p,dF) = contigAssign[contig][depth]
                dFN = dF/contigLengths[contig]
                text_file.write(',%s->%.3f->%.3f'%(assign,p,dFN))
            text_file.write('\n')
            text_file.flush()

if __name__ == "__main__":
    main(sys.argv[1:])

<|endoftext|>


<commit_before>from couchdbkit.exceptions import ResourceNotFound
from datetime import datetime

from casexml.apps.case.dbaccessors import get_extension_case_ids, \
    get_indexed_case_ids, get_all_reverse_indices_info
from casexml.apps.case.models import CommCareCase
from casexml.apps.case.util import get_case_xform_ids
from corehq.apps.hqcase.dbaccessors import (
    get_case_ids_in_domain,
    get_open_case_ids,
    get_closed_case_ids,
    get_case_ids_in_domain_by_owner,
    get_case_types_for_domain,
    get_cases_in_domain_by_external_id,
)
from corehq.apps.hqcase.utils import get_case_by_domain_hq_user_id
from corehq.couchapps.dbaccessors import forms_have_multimedia
from corehq.dbaccessors.couchapps.cases_by_server_date.by_owner_server_modified_on import \
    get_case_ids_modified_with_owner_since
from corehq.dbaccessors.couchapps.cases_by_server_date.by_server_modified_on import \
    get_last_modified_dates
from corehq.form_processor.exceptions import AttachmentNotFound
from corehq.form_processor.interfaces.dbaccessors import (
    AbstractCaseAccessor, AbstractFormAccessor, AttachmentContent,
    AbstractLedgerAccessor)
from couchforms.dbaccessors import (
    get_forms_by_type,
    get_deleted_form_ids_for_user,
    get_form_ids_for_user,
    get_forms_by_id)
from couchforms.models import XFormInstance, doc_types
from dimagi.utils.couch.database import iter_docs
from dimagi.utils.parsing import json_format_datetime

class FormAccessorCouch(AbstractFormAccessor):
    @staticmethod
    def form_exists(form_id, domain=None):
        if not domain:
            return XFormInstance.get_db().doc_exist(form_id)
        else:
            try:
                xform = XFormInstance.get(form_id)
            except ResourceNotFound:
                return False

            return xform.domain == domain

    @staticmethod
    def get_form(form_id):
        return XFormInstance.get(form_id)

    @staticmethod
    def get_forms(form_ids):
        return get_forms_by_id(form_ids)

    @staticmethod
    def get_forms_by_type(domain, type_, limit, recent_first=False):
        return get_forms_by_type(domain, type_, recent_first, limit)

    @staticmethod
    def get_with_attachments(form_id):
        doc = XFormInstance.get_db().get(form_id, attachments=True)
        return doc_types()[doc['doc_type']].wrap(doc)

    @staticmethod
    def get_attachment_content(form_id, attachment_id):
        return _get_attachment_content(XFormInstance, form_id, attachment_id)

    @staticmethod
    def save_new_form(form):
        form.save()

    @staticmethod
    def update_form_problem_and_state(form):
        form.save()

    @staticmethod
    def get_deleted_form_ids_for_user(domain, user_id):
        return get_deleted_form_ids_for_user(user_id)

    @staticmethod
    def get_form_ids_for_user(domain, user_id):
        return get_form_ids_for_user(domain, user_id)

    @staticmethod
    def forms_have_multimedia(domain, app_id, xmlns):
        return forms_have_multimedia(domain, app_id, xmlns)

    @staticmethod
    def soft_delete_forms(domain, form_ids, deletion_date=None, deletion_id=None):
        return _soft_delete(XFormInstance.get_db(), form_ids, deletion_date, deletion_id)

class CaseAccessorCouch(AbstractCaseAccessor):

    @staticmethod
    def get_case(case_id):
        return CommCareCase.get(case_id)

    @staticmethod
    def get_cases(case_ids, ordered=False):
        return [
            CommCareCase.wrap(doc) for doc in iter_docs(
                CommCareCase.get_db(),
                case_ids
            )
        ]

    @staticmethod
    def get_case_xform_ids(case_id):
        return get_case_xform_ids(case_id)

    @staticmethod
    def get_case_ids_in_domain(domain, type=None):
        return get_case_ids_in_domain(domain, type=type)

    @staticmethod
    def get_case_ids_in_domain_by_owners(domain, owner_ids):
        return get_case_ids_in_domain_by_owner(domain, owner_id__in=owner_ids)

    @staticmethod
    def get_open_case_ids(domain, owner_id):
        return get_open_case_ids(domain, owner_id)

    @staticmethod
    def get_closed_case_ids(domain, owner_id):
        return get_closed_case_ids(domain, owner_id)

    @staticmethod
    def get_case_ids_modified_with_owner_since(domain, owner_id, reference_date):
        return get_case_ids_modified_with_owner_since(domain, owner_id, reference_date)

    @staticmethod
    def get_extension_case_ids(domain, case_ids):
        return get_extension_case_ids(domain, case_ids)

    @staticmethod
    def get_indexed_case_ids(domain, case_ids):
        return get_indexed_case_ids(domain, case_ids)

    @staticmethod
    def get_last_modified_dates(domain, case_ids):
        return get_last_modified_dates(domain, case_ids)

    @staticmethod
    def get_all_reverse_indices_info(domain, case_ids):
        return get_all_reverse_indices_info(domain, case_ids)

    @staticmethod
    def get_attachment_content(case_id, attachment_id):
        return _get_attachment_content(CommCareCase, case_id, attachment_id)

    @staticmethod
    def get_case_by_domain_hq_user_id(domain, user_id, case_type):
        return get_case_by_domain_hq_user_id(domain, user_id, case_type)

    @staticmethod
    def get_case_types_for_domain(domain):
        return get_case_types_for_domain(domain)

    @staticmethod
    def get_cases_by_external_id(domain, external_id, case_type=None):
        cases = get_cases_in_domain_by_external_id(domain, external_id)
        if case_type:
            return [case for case in cases if case.type == case_type]
        return cases

    @staticmethod
    def soft_delete_cases(domain, case_ids, deletion_date=None, deletion_id=None):
        return _soft_delete(CommCareCase.get_db(), case_ids, deletion_date, deletion_id)

class LedgerAccessorCouch(AbstractLedgerAccessor):
    @staticmethod
    def get_transactions_for_consumption(domain, case_id, product_id, section_id, window_start, window_end):
        from casexml.apps.stock.models import StockTransaction
        db_transactions = StockTransaction.objects.filter(
            case_id=case_id, product_id=product_id,
            report__date__gt=window_start,
            report__date__lte=window_end,
            section_id=section_id,
        ).order_by('report__date', 'pk')

        first = True
        for db_tx in db_transactions:
            # for the very first transaction, include the previous one if there as well
            # to capture the data on the edge of the window
            if first:
                previous = db_tx.get_previous_transaction()
                if previous:
                    yield previous
                first = False

            yield db_tx

def _get_attachment_content(doc_class, doc_id, attachment_id):
    try:
        resp = doc_class.get_db().fetch_attachment(doc_id, attachment_id, stream=True)
    except ResourceNotFound:
        raise AttachmentNotFound(attachment_id)

    headers = resp.resp.headers
    content_type = headers.get('Content-Type', None)
    return AttachmentContent(content_type, resp)

def _soft_delete(db, doc_ids, deletion_date=None, deletion_id=None):
    from dimagi.utils.couch.undo import DELETED_SUFFIX
    deletion_date = json_format_datetime(deletion_date or datetime.utcnow())

    def delete(doc):
        doc['doc_type'] += DELETED_SUFFIX
        doc['-deletion_id'] = deletion_id
        doc['-deletion_date'] = deletion_date
        return doc

    return _operate_on_docs(db, doc_ids, delete)

def _soft_undelete(db, doc_ids):
    from dimagi.utils.couch.undo import DELETED_SUFFIX

    def undelete(doc):
        doc_type = doc['doc_type']
        if doc_type.endswith(DELETED_SUFFIX):
            doc['doc_type'] = doc_type[:-len(DELETED_SUFFIX)]

        del doc['-deletion_id']
        del doc['-deletion_date']
        return doc

    return _operate_on_docs(db, doc_ids, undelete)

def _operate_on_docs(db, doc_ids, operation_fn):
    docs_to_save = []
    for doc in iter_docs(db, doc_ids):
        doc = operation_fn(doc)
        docs_to_save.append(doc)

        if len(docs_to_save) % 1000 == 0:
            db.bulk_save(docs_to_save)
            docs_to_save = []

    if docs_to_save:
        db.bulk_save(docs_to_save)

    return len(doc_ids)
<commit_msg>responding 404 for deleted cases<commit_after>from couchdbkit.exceptions import ResourceNotFound
from datetime import datetime

from casexml.apps.case.dbaccessors import get_extension_case_ids, \
    get_indexed_case_ids, get_all_reverse_indices_info
from casexml.apps.case.models import CommCareCase
from casexml.apps.case.util import get_case_xform_ids
from corehq.apps.hqcase.dbaccessors import (
    get_case_ids_in_domain,
    get_open_case_ids,
    get_closed_case_ids,
    get_case_ids_in_domain_by_owner,
    get_case_types_for_domain,
    get_cases_in_domain_by_external_id,
)
from corehq.apps.hqcase.utils import get_case_by_domain_hq_user_id
from corehq.couchapps.dbaccessors import forms_have_multimedia
from corehq.dbaccessors.couchapps.cases_by_server_date.by_owner_server_modified_on import \
    get_case_ids_modified_with_owner_since
from corehq.dbaccessors.couchapps.cases_by_server_date.by_server_modified_on import \
    get_last_modified_dates
from corehq.form_processor.exceptions import AttachmentNotFound, CaseNotFound
from corehq.form_processor.interfaces.dbaccessors import (
    AbstractCaseAccessor, AbstractFormAccessor, AttachmentContent,
    AbstractLedgerAccessor)
from couchforms.dbaccessors import (
    get_forms_by_type,
    get_deleted_form_ids_for_user,
    get_form_ids_for_user,
    get_forms_by_id)
from couchforms.models import XFormInstance, doc_types
from dimagi.utils.couch.database import iter_docs
from dimagi.utils.parsing import json_format_datetime

class FormAccessorCouch(AbstractFormAccessor):
    @staticmethod
    def form_exists(form_id, domain=None):
        if not domain:
            return XFormInstance.get_db().doc_exist(form_id)
        else:
            try:
                xform = XFormInstance.get(form_id)
            except ResourceNotFound:
                return False

            return xform.domain == domain

    @staticmethod
    def get_form(form_id):
        return XFormInstance.get(form_id)

    @staticmethod
    def get_forms(form_ids):
        return get_forms_by_id(form_ids)

    @staticmethod
    def get_forms_by_type(domain, type_, limit, recent_first=False):
        return get_forms_by_type(domain, type_, recent_first, limit)

    @staticmethod
    def get_with_attachments(form_id):
        doc = XFormInstance.get_db().get(form_id, attachments=True)
        return doc_types()[doc['doc_type']].wrap(doc)

    @staticmethod
    def get_attachment_content(form_id, attachment_id):
        return _get_attachment_content(XFormInstance, form_id, attachment_id)

    @staticmethod
    def save_new_form(form):
        form.save()

    @staticmethod
    def update_form_problem_and_state(form):
        form.save()

    @staticmethod
    def get_deleted_form_ids_for_user(domain, user_id):
        return get_deleted_form_ids_for_user(user_id)

    @staticmethod
    def get_form_ids_for_user(domain, user_id):
        return get_form_ids_for_user(domain, user_id)

    @staticmethod
    def forms_have_multimedia(domain, app_id, xmlns):
        return forms_have_multimedia(domain, app_id, xmlns)

    @staticmethod
    def soft_delete_forms(domain, form_ids, deletion_date=None, deletion_id=None):
        return _soft_delete(XFormInstance.get_db(), form_ids, deletion_date, deletion_id)

class CaseAccessorCouch(AbstractCaseAccessor):

    @staticmethod
    def get_case(case_id):
        case = CommCareCase.get(case_id)
        if 'Deleted' in case.doc_type:
            raise CaseNotFound
        else:
            return case

    @staticmethod
    def get_cases(case_ids, ordered=False):
        return [
            CommCareCase.wrap(doc) for doc in iter_docs(
                CommCareCase.get_db(),
                case_ids
            )
        ]

    @staticmethod
    def get_case_xform_ids(case_id):
        return get_case_xform_ids(case_id)

    @staticmethod
    def get_case_ids_in_domain(domain, type=None):
        return get_case_ids_in_domain(domain, type=type)

    @staticmethod
    def get_case_ids_in_domain_by_owners(domain, owner_ids):
        return get_case_ids_in_domain_by_owner(domain, owner_id__in=owner_ids)

    @staticmethod
    def get_open_case_ids(domain, owner_id):
        return get_open_case_ids(domain, owner_id)

    @staticmethod
    def get_closed_case_ids(domain, owner_id):
        return get_closed_case_ids(domain, owner_id)

    @staticmethod
    def get_case_ids_modified_with_owner_since(domain, owner_id, reference_date):
        return get_case_ids_modified_with_owner_since(domain, owner_id, reference_date)

    @staticmethod
    def get_extension_case_ids(domain, case_ids):
        return get_extension_case_ids(domain, case_ids)

    @staticmethod
    def get_indexed_case_ids(domain, case_ids):
        return get_indexed_case_ids(domain, case_ids)

    @staticmethod
    def get_last_modified_dates(domain, case_ids):
        return get_last_modified_dates(domain, case_ids)

    @staticmethod
    def get_all_reverse_indices_info(domain, case_ids):
        return get_all_reverse_indices_info(domain, case_ids)

    @staticmethod
    def get_attachment_content(case_id, attachment_id):
        return _get_attachment_content(CommCareCase, case_id, attachment_id)

    @staticmethod
    def get_case_by_domain_hq_user_id(domain, user_id, case_type):
        return get_case_by_domain_hq_user_id(domain, user_id, case_type)

    @staticmethod
    def get_case_types_for_domain(domain):
        return get_case_types_for_domain(domain)

    @staticmethod
    def get_cases_by_external_id(domain, external_id, case_type=None):
        cases = get_cases_in_domain_by_external_id(domain, external_id)
        if case_type:
            return [case for case in cases if case.type == case_type]
        return cases

    @staticmethod
    def soft_delete_cases(domain, case_ids, deletion_date=None, deletion_id=None):
        return _soft_delete(CommCareCase.get_db(), case_ids, deletion_date, deletion_id)

class LedgerAccessorCouch(AbstractLedgerAccessor):
    @staticmethod
    def get_transactions_for_consumption(domain, case_id, product_id, section_id, window_start, window_end):
        from casexml.apps.stock.models import StockTransaction
        db_transactions = StockTransaction.objects.filter(
            case_id=case_id, product_id=product_id,
            report__date__gt=window_start,
            report__date__lte=window_end,
            section_id=section_id,
        ).order_by('report__date', 'pk')

        first = True
        for db_tx in db_transactions:
            # for the very first transaction, include the previous one if there as well
            # to capture the data on the edge of the window
            if first:
                previous = db_tx.get_previous_transaction()
                if previous:
                    yield previous
                first = False

            yield db_tx

def _get_attachment_content(doc_class, doc_id, attachment_id):
    try:
        resp = doc_class.get_db().fetch_attachment(doc_id, attachment_id, stream=True)
    except ResourceNotFound:
        raise AttachmentNotFound(attachment_id)

    headers = resp.resp.headers
    content_type = headers.get('Content-Type', None)
    return AttachmentContent(content_type, resp)

def _soft_delete(db, doc_ids, deletion_date=None, deletion_id=None):
    from dimagi.utils.couch.undo import DELETED_SUFFIX
    deletion_date = json_format_datetime(deletion_date or datetime.utcnow())

    def delete(doc):
        doc['doc_type'] += DELETED_SUFFIX
        doc['-deletion_id'] = deletion_id
        doc['-deletion_date'] = deletion_date
        return doc

    return _operate_on_docs(db, doc_ids, delete)

def _soft_undelete(db, doc_ids):
    from dimagi.utils.couch.undo import DELETED_SUFFIX

    def undelete(doc):
        doc_type = doc['doc_type']
        if doc_type.endswith(DELETED_SUFFIX):
            doc['doc_type'] = doc_type[:-len(DELETED_SUFFIX)]

        del doc['-deletion_id']
        del doc['-deletion_date']
        return doc

    return _operate_on_docs(db, doc_ids, undelete)

def _operate_on_docs(db, doc_ids, operation_fn):
    docs_to_save = []
    for doc in iter_docs(db, doc_ids):
        doc = operation_fn(doc)
        docs_to_save.append(doc)

        if len(docs_to_save) % 1000 == 0:
            db.bulk_save(docs_to_save)
            docs_to_save = []

    if docs_to_save:
        db.bulk_save(docs_to_save)

    return len(doc_ids)
<|endoftext|>


<commit_before>"""
Try to replicate the toy dateset in the paper by Fowlkes et al., "Spectral Grouping
Using the Nystrom Method" and compute the spectrum.
"""

import numpy
import scipy.sparse
import matplotlib.pyplot as plt
from apgl.graph import SparseGraph, GeneralVertexList, GraphUtils
from exp.sandbox.Nystrom import Nystrom

numpy.set_printoptions(suppress=True, linewidth=200, precision=3)

numVertices = 150
X = numpy.zeros((numVertices, 2))

#Create circle
radius = 5
noise = 0.3
angles = numpy.random.rand(100)*2*numpy.pi
X[0:100, 0] = radius*numpy.sin(angles)
X[0:100, 1] = radius*numpy.cos(angles)

X[0:100, :] += numpy.random.randn(100, 2)*noise

#Create blob
R = 1
centre = numpy.array([5-R, 0])
X[100:, :] = centre + numpy.random.randn(50, 2)*noise

plt.figure(0)
plt.scatter(X[0:100, 0], X[0:100, 1], c="r")
plt.scatter(X[100:, 0], X[100:, 1], c="b")

#Compute weight matrix
sigma = 0.2
W = numpy.zeros((numVertices, numVertices))

for i in range(numVertices):
    for j in range(numVertices):
        W[i, j] = numpy.exp(-(numpy.linalg.norm(X[i, :] - X[j, :])**2)/(2*sigma**2))

graph = SparseGraph(GeneralVertexList(numVertices))
graph.setWeightMatrix(W)
L = graph.normalisedLaplacianSym()

#L = GraphUtils.shiftLaplacian(scipy.sparse.csr_matrix(W)).todense()
n = 100
omega, Q = numpy.linalg.eigh(L)
omega2, Q2 = Nystrom.eigpsd(L, n)

print(omega)
print(omega2)

plt.figure(1)
plt.plot(numpy.arange(omega.shape[0]), omega)
plt.plot(numpy.arange(omega2.shape[0]), omega2)
plt.show()<commit_msg>Put in functions <commit_after>"""
Try to replicate the toy dateset in the paper by Fowlkes et al., "Spectral Grouping
Using the Nystrom Method" and compute the spectrum.
"""

import numpy
import scipy.sparse
import matplotlib.pyplot as plt
from apgl.graph import SparseGraph, GeneralVertexList, GraphUtils
from exp.sandbox.Nystrom import Nystrom

numpy.set_printoptions(suppress=True, linewidth=200, precision=3)

def createDataset():
    numVertices = 150
    X = numpy.zeros((numVertices, 2))

    #Create circle
    radius = 5
    noise = 0.3
    angles = numpy.random.rand(100)*2*numpy.pi
    X[0:100, 0] = radius*numpy.sin(angles)
    X[0:100, 1] = radius*numpy.cos(angles)

    X[0:100, :] += numpy.random.randn(100, 2)*noise

    #Create blob
    R = 1
    centre = numpy.array([5-R, 0])
    X[100:, :] = centre + numpy.random.randn(50, 2)*noise

    #plt.figure(0)
    #plt.scatter(X[0:100, 0], X[0:100, 1], c="r")
    #plt.scatter(X[100:, 0], X[100:, 1], c="b")

    #Compute weight matrix
    sigma = 0.2
    W = numpy.zeros((numVertices, numVertices))

    for i in range(numVertices):
        for j in range(numVertices):
            W[i, j] = numpy.exp(-(numpy.linalg.norm(X[i, :] - X[j, :])**2)/(2*sigma**2))

    return W

def run():
    W = createDataset()

    numVertices = W.shape[0]
    graph = SparseGraph(GeneralVertexList(numVertices))
    graph.setWeightMatrix(W)
    L = graph.normalisedLaplacianSym()
    #L = GraphUtils.shiftLaplacian(scipy.sparse.csr_matrix(W)).todense()
    n = 100
    omega, Q = numpy.linalg.eigh(L)
    omega2, Q2 = Nystrom.eigpsd(L, n)

    print(omega)
    print(omega2)

    plt.figure(1)
    plt.plot(numpy.arange(omega.shape[0]), omega)
    plt.plot(numpy.arange(omega2.shape[0]), omega2)
    plt.show()

#run()<|endoftext|>


<commit_before>"""actofgoods URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.9/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url, include
from django.contrib import admin
from . import views

app_name = 'basics'

urlpatterns = [
    url(r'^$', views.actofgoods_startpage, name='actofgoods_startpage'),
    url(r'^$', views.index, name='index'),
    url(r'^login/$', views.login, name='login'),
    url(r'^profil/$', views.profil, name='profil'),
    url(r'^aboutus/$', views.aboutus, name='aboutus'),
    url(r'^privacy/$', views.privacy, name='aboutus'),

]
<commit_msg>urls privacy fixed<commit_after>"""actofgoods URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.9/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url, include
from django.contrib import admin
from . import views

app_name = 'basics'

urlpatterns = [
    url(r'^$', views.actofgoods_startpage, name='actofgoods_startpage'),
    url(r'^$', views.index, name='index'),
    url(r'^login/$', views.login, name='login'),
    url(r'^profil/$', views.profil, name='profil'),
    url(r'^aboutus/$', views.aboutus, name='aboutus'),
    url(r'^privacy/$', views.privacy, name='privacy'),

]
<|endoftext|>


<commit_before>import datetime
import calendar

from .events import events_dict
from .progressions import run_easy_progress, interval_progress, \
    hillsprint_progress, tempo_progress

class Plan:
    '''
    Represents running training plan for prescribed event and level.
    '''

    def __init__(self, event, level, current_date=datetime.date.today()):
        self.event = event
        self.distance, self.event_date = events_dict[self.event]
        self.level = level
        self.current_date = current_date

        self.start_date = self.determine_start_date()
        self.schedule_weeks = self.determine_schedule_weeks()
        self.weeks_to_event = len(self.schedule_weeks)

        self.schedule = {}
        self.calendars = []

    def __repr__(self):
        '''
        Return a more human-readable representation
        '''

        return "{0} week {1} Plan for the {2}".format(self.weeks_to_event,
                                                      self.level,
                                                      self.event)

    def determine_start_date(self):
        '''
        None -> datetime

        Return the start date of the training plan, which is assumed to be
        the next Monday following plan creation.
        '''

        days_ahead = 0 - self.current_date.weekday()
        if days_ahead <= 0:  # Target day already happened this week
            days_ahead += 7
        return self.current_date + datetime.timedelta(days_ahead)

    def determine_schedule_weeks(self):
        '''
        None -> matrix

        Return a matrix (list of lists) representing training plan period.
        Each row represents a week; week entries are datetime.date values.
        '''

        days_ahead = 6 - self.event_date.weekday()
        end_date = self.event_date + datetime.timedelta(days_ahead)
        days = [self.start_date + datetime.timedelta(n)
                for n in range(int((end_date - self.start_date).days) + 1)]
        return [days[i:i + 7] for i in range(0, len(days), 7)]

    def add_progression_to_schedule(self, day, progression):
        '''
        int, list -> None

        Populates plan schedule with given progression on specified day.
        '''
        for step, week in zip(progression, self.schedule_weeks):
            self.schedule[week[day]] = step

    def create_schedule(self, days):
        '''
        list -> None

        Creates plan schedule based on prescribed list of training days.
        '''

        def builder_dict(distance, level, weeks, days):
            distance_dict = {
                "5k": {"Beginner": self.builder_5k_beg,
                       "Intermediate": self.builder_5k_int,
                       "Advanced": self.builder_5k_beg},
                "10k": {"Beginner": self.builder_5k_beg,
                        "Intermediate": self.builder_5k_beg,
                        "Advanced": self.builder_5k_beg},
                "half": {"Beginner": self.builder_5k_beg,
                         "Intermediate": self.builder_5k_beg,
                         "Advanced": self.builder_5k_beg},
                "full": {"Beginner": self.builder_5k_beg,
                         "Intermediate": self.builder_5k_beg,
                         "Advanced": self.builder_5k_beg}
            }.get(distance, None)

            return distance_dict[level](weeks, days)

        progressions = builder_dict(
            self.distance, self.level, len(self.schedule_weeks), len(days))

        for day, progression in zip(days, progressions):
            self.add_progression_to_schedule(day, progression)

        # Populate schedule with race day
        self.schedule[self.event_date] = "RACE DAY!"

    def builder_5k_beg(self, weeks, days):
        '''
        int, int -> matrix

        Return matrix (list of lists) representing 5k Beginner progressions.
        Based on specified number of weeks and for specified number of training
        days a week.
        '''

        progressions = []

        if days > 0:
            progressions.append(list(run_easy_progress(weeks)))
        if days > 1:
            ints = list(interval_progress(weeks, start_week=0, step=2))
            hills = list(hillsprint_progress(weeks, start_week=1, step=2))
            progress = [val for pair in zip(ints, hills) for val in pair]
            progressions.append(progress)
        if days > 2:
            progressions.append(list(run_easy_progress(weeks)))

        return progressions

    def builder_5k_int(self, weeks, days):
        '''
        int, int -> matrix

        Return matrix (list of lists) representing 5k Intermediate progressions.
        Based on specified number of weeks and for specified number of training
        days a week.
        '''

        progressions = []

        if days > 0:
            tempos = list(tempo_progress(weeks, start_week=0, step=2, freq=3))
            ints = list(interval_progress(weeks, start_week=1, step=2))
            progress = [val for pair in zip(tempos, ints) for val in pair]
            progressions.append(progress)
        if days > 1:
            progressions.append(list(run_easy_progress(weeks)))
        if days > 2:
            progressions.append(
                list(hillsprint_progress(weeks, start_week=0, step=1)))

        return progressions

    def create_cals(self):
        '''
        Create list of calendars to render schedule.
        '''

        year = datetime.date.today().year
        month = 0
        for entry in self.schedule:
            if entry.month > month:
                year = entry.year
                month = entry.month
                self.calendars.append(WorkoutCalendar(
                    self.schedule).formatmonth(year, month))

class WorkoutCalendar(calendar.HTMLCalendar):
    '''
    A calendar renderer.
    '''

    def __init__(self, schedule):
        super(WorkoutCalendar, self).__init__()
        self.schedule = schedule

    def formatday(self, day, weekday):

        # days belonging to last or next month are rendered empty
        if day == 0:
            return self.day_cell('noday', ' ')

        date_obj = datetime.date(self.year, self.month, day)
        cssclass = self.cssclasses[weekday]
        if datetime.date.today() == date_obj:
            cssclass += ' today'

        # There are no logs for this day, doesn't need special attention
        if date_obj not in self.schedule:
            return self.day_cell(cssclass, day)

        entry = self.schedule.get(date_obj)

        # Format race day
        if entry == "RACE DAY!":
            formatted_date = str(day)
            body = []
            body.append('
 {0}
'.format(formatted_date))
            body.append(
                ''.format('btn-info'))
            body.append(entry)
            return self.day_cell(cssclass, '{0}'.format(''.join(body)))

        background_css = entry.background_css
        formatted_date = str(day)
        body = []
        body.append('
 {0}
'.format(formatted_date))
        body.append(
            ''.format(background_css))

        body.append(repr(entry))
        body.append('')

        return self.day_cell(cssclass, '{0}'.format(''.join(body)))

    def formatmonth(self, year, month):
        '''
        Format the table header. This is basically the same code from python's
        calendar module but with additional bootstrap classes
        '''
        self.year, self.month = year, month
        out = []
        out.append('
\n')
        out.append(self.formatmonthname(year, month))
        out.append('\n')
        out.append(self.formatweekheader())
        out.append('\n')
        for week in self.monthdays2calendar(year, month):
            out.append(self.formatweek(week))
            out.append('\n')
        out.append('
\n')
        return ''.join(out)

    def day_cell(self, cssclass, body):
        '''
        Renders a day cell
        '''
        return '{1}	'.format(cssclass, body)
<commit_msg>improve calendar presentation<commit_after>import datetime
import calendar

from .events import events_dict
from .progressions import run_easy_progress, interval_progress, \
    hillsprint_progress, tempo_progress

class Plan:
    '''
    Represents running training plan for prescribed event and level.
    '''

    def __init__(self, event, level, current_date=datetime.date.today()):
        self.event = event
        self.distance, self.event_date = events_dict[self.event]
        self.level = level
        self.current_date = current_date

        self.start_date = self.determine_start_date()
        self.schedule_weeks = self.determine_schedule_weeks()
        self.weeks_to_event = len(self.schedule_weeks)

        self.schedule = {}
        self.calendars = []

    def __repr__(self):
        '''
        Return a more human-readable representation
        '''

        return "{0} week {1} Plan for the {2}".format(self.weeks_to_event,
                                                      self.level,
                                                      self.event)

    def determine_start_date(self):
        '''
        None -> datetime

        Return the start date of the training plan, which is assumed to be
        the next Monday following plan creation.
        '''

        days_ahead = 0 - self.current_date.weekday()
        if days_ahead <= 0:  # Target day already happened this week
            days_ahead += 7
        return self.current_date + datetime.timedelta(days_ahead)

    def determine_schedule_weeks(self):
        '''
        None -> matrix

        Return a matrix (list of lists) representing training plan period.
        Each row represents a week; week entries are datetime.date values.
        '''

        days_ahead = 6 - self.event_date.weekday()
        end_date = self.event_date + datetime.timedelta(days_ahead)
        days = [self.start_date + datetime.timedelta(n)
                for n in range(int((end_date - self.start_date).days) + 1)]
        return [days[i:i + 7] for i in range(0, len(days), 7)]

    def add_progression_to_schedule(self, day, progression):
        '''
        int, list -> None

        Populates plan schedule with given progression on specified day.
        '''
        for step, week in zip(progression, self.schedule_weeks):
            self.schedule[week[day]] = step

    def create_schedule(self, days):
        '''
        list -> None

        Creates plan schedule based on prescribed list of training days.
        '''

        def builder_dict(distance, level, weeks, days):
            distance_dict = {
                "5k": {"Beginner": self.builder_5k_beg,
                       "Intermediate": self.builder_5k_int,
                       "Advanced": self.builder_5k_beg},
                "10k": {"Beginner": self.builder_5k_beg,
                        "Intermediate": self.builder_5k_beg,
                        "Advanced": self.builder_5k_beg},
                "half": {"Beginner": self.builder_5k_beg,
                         "Intermediate": self.builder_5k_beg,
                         "Advanced": self.builder_5k_beg},
                "full": {"Beginner": self.builder_5k_beg,
                         "Intermediate": self.builder_5k_beg,
                         "Advanced": self.builder_5k_beg}
            }.get(distance, None)

            return distance_dict[level](weeks, days)

        progressions = builder_dict(
            self.distance, self.level, len(self.schedule_weeks), len(days))

        for day, progression in zip(days, progressions):
            self.add_progression_to_schedule(day, progression)

        # Populate schedule with race day
        self.schedule[self.event_date] = "RACE DAY!"

    def builder_5k_beg(self, weeks, days):
        '''
        int, int -> matrix

        Return matrix (list of lists) representing 5k Beginner progressions.
        Based on specified number of weeks and for specified number of training
        days a week.
        '''

        progressions = []

        if days > 0:
            progressions.append(list(run_easy_progress(weeks)))
        if days > 1:
            ints = list(interval_progress(weeks, start_week=0, step=2))
            hills = list(hillsprint_progress(weeks, start_week=1, step=2))
            progress = [val for pair in zip(ints, hills) for val in pair]
            progressions.append(progress)
        if days > 2:
            progressions.append(list(run_easy_progress(weeks)))

        return progressions

    def builder_5k_int(self, weeks, days):
        '''
        int, int -> matrix

        Return matrix (list of lists) representing 5k Intermediate progressions.
        Based on specified number of weeks and for specified number of training
        days a week.
        '''

        progressions = []

        if days > 0:
            tempos = list(tempo_progress(weeks, start_week=0, step=2, freq=3))
            ints = list(interval_progress(weeks, start_week=1, step=2))
            progress = [val for pair in zip(tempos, ints) for val in pair]
            progressions.append(progress)
        if days > 1:
            progressions.append(list(run_easy_progress(weeks)))
        if days > 2:
            progressions.append(
                list(hillsprint_progress(weeks, start_week=0, step=1)))

        return progressions

    def create_cals(self):
        '''
        Create list of calendars to render schedule.
        '''

        year = datetime.date.today().year
        month = 0
        for entry in self.schedule:
            if entry.year > year:
                year = entry.year
                month = entry.month
            if entry.month > month:
                month = entry.month
            self.calendars.append(WorkoutCalendar(
                self.schedule).formatmonth(year, month))

class WorkoutCalendar(calendar.HTMLCalendar):
    '''
    A calendar renderer.
    '''

    def __init__(self, schedule):
        super(WorkoutCalendar, self).__init__()
        self.schedule = schedule

    def formatday(self, day, weekday):

        # days belonging to last or next month are rendered empty
        if day == 0:
            return self.day_cell('noday', ' ')

        date_obj = datetime.date(self.year, self.month, day)
        cssclass = self.cssclasses[weekday]
        if datetime.date.today() == date_obj:
            cssclass += ' today'

        # There are no logs for this day, doesn't need special attention
        if date_obj not in self.schedule:
            return self.day_cell(cssclass, day)

        entry = self.schedule.get(date_obj)

        # Format race day
        if entry == "RACE DAY!":
            formatted_date = str(day)
            body = []
            body.append('
 {0}
'.format(formatted_date))
            body.append(
                ''.format('btn-info'))
            body.append(entry)
            return self.day_cell(cssclass, '{0}'.format(''.join(body)))

        background_css = entry.background_css
        formatted_date = str(day)
        body = []
        body.append('
 {0}
'.format(formatted_date))
        body.append(
            ''.format(background_css))

        body.append(repr(entry))
        body.append('')

        return self.day_cell(cssclass, '{0}'.format(''.join(body)))

    def formatmonth(self, year, month):
        '''
        Format the table header. This is basically the same code from python's
        calendar module but with additional bootstrap classes
        '''
        self.year, self.month = year, month
        out = []
        out.append('
\n')
        out.append(self.formatmonthname(year, month))
        out.append('\n')
        out.append(self.formatweekheader())
        out.append('\n')
        for week in self.monthdays2calendar(year, month):
            out.append(self.formatweek(week))
            out.append('\n')
        out.append('
\n')
        return ''.join(out)

    def day_cell(self, cssclass, body):
        '''
        Renders a day cell
        '''
        return '{1}	'.format(cssclass, body)
<|endoftext|>