<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# import string
# import os.path
import re

from lxml import html
from xdfile.utils import info, debug, error
import xdfile

SPLIT_REBUS_TITLES = "CRYPTOCROSSWORD TIC-TAC-TOE".split()

def stringify_children(node):
    s = node.text
    if s is None:
        s = ''
    # for child in node:
    #     s += etree.tostring(child, encoding='unicode')
    return s

# content is unicode()
def parse_xwordinfo(content, filename):
    content = content.decode('utf-8')

    REBUS_LONG_HANDS = {'NINE': '9',
                        'EIGHT': '8',
                        'SEVEN': '7',
                        'SIX': '6',
                        'FIVE': '5',
                        'FOUR': '4',
                        'THREE': '3',
                        'TWO': '2',
                        'ONE': '1',
                        'ZERO': '0',
                        'AUGHT': '0',
                        'AMPERSAND': '&',
                        'AND': '&',
                        'ASTERISK': '*',
                        'PERCENT': '%',
                        'STAR': '*',
                        'AT': '@',
                        'DOLLAR': '$',
                        'PLUS': '+',
                        'CENT': 'c',
                        # 'DASH': '-',
                        # 'DOT': '●'
                        }
    rsh = 'zyxwvutsrqponmlkjihgfedcba♚♛♜♝♞♟⚅⚄⚃⚂⚁⚀♣♦♥♠Фθиλπφя+&%$@?*0987654321'
    REBUS_SHORT_HANDS = list(rsh)

    content = content.replace("", "{*")
    content = content.replace("", "*}")
    content = content.replace("", "{/")
    content = content.replace("", "/}")
    content = content.replace("", "{/")
    content = content.replace("", "/}")
    content = content.replace("", "{_")
    content = content.replace("", "_}")
    content = content.replace("", "{-")
    content = content.replace("", "-}")
    content = content.replace("’", "'")
    content = content.replace('“', '"')
    # content = content.replace('–', '-')

    if "CPHContent_" in content:
        xwiprefix = '#CPHContent_'
    else:
        xwiprefix = '#'

    root = html.fromstring(content)

    ## debug("ROOT: %s" % root)

    special_type = ''
    rebus = {}
    rebus_order = []

    xd = xdfile.xdfile('', filename)

    # get crossword info
    title = root.cssselect(xwiprefix + 'TitleLabel')[0].text.strip()
    try:
        subtitle = root.cssselect(xwiprefix + 'SubTitleLabel')[0].text.strip()
        subtitle = ' [%s]' % subtitle
    except:
        subtitle = ""

    # author = root.cssselect(xwiprefix + 'AuthorLabel')[0].text.strip()
    # editor = root.cssselect(xwiprefix + 'EditorLabel')[0].text.strip()
    try:
        xd.notes = stringify_children(root.cssselect(xwiprefix + 'NotepadDiv')[0])
    except Exception as e:
        xd.notes = ""
        debug('Exception %s' % e)

    xd.set_header("Title", '%s%s' % (title, subtitle))
    xd.set_header("Author", root.cssselect(xwiprefix + 'AuthorLabel')[0].text.strip())
    xd.set_header("Editor", root.cssselect(xwiprefix + 'EditorLabel')[0].text.strip())

    xd.notes = xd.notes.replace("
", "\n")
    xd.notes = xd.notes.replace("Notepad:", "\n")
    xd.notes = xd.notes.replace("
", "\n")
    xd.notes = xd.notes.strip()

    puzzle_table = root.cssselect(xwiprefix + 'PuzTable tr') or root.cssselect('#PuzTable tr')

    for row in puzzle_table:
        row_data = ""
        for cell in row.cssselect('td'):
            # check if the cell is special - with a shade or a circle
            cell_class = cell.get('class')
            cell_type = ''
            if cell_class == 'bigshade':
                cell_type = 'shaded'
            elif cell_class == 'bigcircle':
                cell_type = 'circle'

            letter = cell.cssselect('div.letter')
            letter = (len(letter) and letter[0].text) or xdfile.BLOCK_CHAR

            # handle rebuses
            if letter == xdfile.BLOCK_CHAR:
                subst = cell.cssselect('div.subst2')
                subst = (len(subst) and subst[0].text) or ''
                if not subst:
                    subst = cell.cssselect('div.subst')
                    if subst:
                        if title in SPLIT_REBUS_TITLES:
                            subst = "/".join(list(subst[0].text))
                        else:
                            subst = subst[0].text
                    else:
                        subst = ''

                if subst:
                    if subst not in rebus:
                        if subst in REBUS_LONG_HANDS:
                            rebus_val = REBUS_LONG_HANDS[subst]
                            if rebus_val in REBUS_SHORT_HANDS:
                                REBUS_SHORT_HANDS.remove(rebus_val)
                        else:
                            rebus_val = REBUS_SHORT_HANDS.pop()
                        rebus[subst] = rebus_val
                        rebus_order.append(subst)
                    letter = rebus[subst]

            if cell_type:
                # the special cell's letter should be represented in lower case
                letter = letter.lower()
                if not special_type:
                    # hopefully there shouldn't be both shades and circles in
                    # the same puzzle - if that is the case, only the last value
                    # will be put up in the header
                    special_type = cell_type

            row_data += letter
        xd.grid.append(row_data)

    if len(rebus):
        rebus = ["%s=%s" % (rebus[x], x.upper()) for x in rebus_order]
        xd.set_header("Rebus", ','.join(rebus))
    if special_type:
        xd.set_header("Special", special_type)

    # add clues
    across_clues = _fetch_clues(xd, 'A', root, xwiprefix + 'AcrossClues', rebus)
    down_clues = _fetch_clues(xd, 'D', root, xwiprefix + 'DownClues', rebus)

    return xd

def _fetch_clues(xd, clueprefix, root, css_identifier, rebus):
    PT_CLUE = re.compile(r'(\d+)\. ?(.*)')
    text = number = solution = None
    answer_next = False
    cluespan = root.cssselect(css_identifier)
    if not cluespan:
        return
    for content in cluespan[0].itertext():
        content = content.strip()
        if answer_next:
            assert not solution
            solution = content

            # replace rebuses with appropriate identifiers (numbers)
            for item in rebus:
                if item in content:
                    pass
                    # TODO: where is 'index'???
                    # content = content.replace(item, str(index + 1))

            solution = content
            xd.clues.append(((clueprefix, number), text, solution))
            text = number = solution = None
            answer_next = False
        else:
            if content[-2:] == " :":
                answer_next = True
                content = content[:-2]

            match = re.match(PT_CLUE, content)
            if match:
                number = int(match.group(1))
                text = match.group(2)
            else:
                if text is None:  # a special one
                    number = content
                    text = ""
                else:
                    text += " " + content

if __name__ == "__main__":
    import sys
    from .utils import find_files
    for fn, contents in find_files(*sys.argv[1:]):
        xd = parse_xwordinfo(contents, fn)
        print("--- %s ---" % fn)
        print(xd.to_unicode())
<commit_msg>fix xwi parsing<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import re
from lxml import html, etree
from datetime import datetime
from xdfile.utils import info, debug, error
import xdfile

SPLIT_REBUS_TITLES = "CRYPTOCROSSWORD TIC-TAC-TOE".split()

class XWordInfoParseError(Exception):
    pass

# content is unicode()
def parse_xwordinfo(content, filename):
    content = content.decode('utf-8')

    REBUS_LONG_HANDS = {'NINE': '9',
                        'EIGHT': '8',
                        'SEVEN': '7',
                        'SIX': '6',
                        'FIVE': '5',
                        'FOUR': '4',
                        'THREE': '3',
                        'TWO': '2',
                        'ONE': '1',
                        'ZERO': '0',
                        'AUGHT': '0',
                        'AMPERSAND': '&',
                        'AND': '&',
                        'ASTERISK': '*',
                        'PERCENT': '%',
                        'STAR': '*',
                        'AT': '@',
                        'DOLLAR': '$',
                        'PLUS': '+',
                        'CENT': 'c',
                        # 'DASH': '-',
                        # 'DOT': '●'
                        }
    rsh = 'zyxwvutsrqponmlkjihgfedcba♚♛♜♝♞♟⚅⚄⚃⚂⚁⚀♣♦♥♠Фθиλπφя+&%$@?*0987654321'
    REBUS_SHORT_HANDS = list(rsh)

    content = content.replace("", "{*")
    content = content.replace("", "*}")
    content = content.replace("", "{/")
    content = content.replace("", "/}")
    content = content.replace("", "{/")
    content = content.replace("", "/}")
    content = content.replace("", "{_")
    content = content.replace("", "_}")
    content = content.replace("", "{-")
    content = content.replace("", "-}")
    content = content.replace("’", "'")
    content = content.replace('“', '"')
    # content = content.replace('–', '-')

    if "CPHContent_" in content:
        xwiprefix = '#CPHContent_'
    else:
        xwiprefix = '#'

    root = html.fromstring(content)

    ## debug("ROOT: %s" % root)

    special_type = ''
    rebus = {}
    rebus_order = []

    xd = xdfile.xdfile('', filename)

    # get crossword info
    title = root.cssselect('#PuzTitle')[0].text.strip()
    try:
        subtitle = root.cssselect(xwiprefix + 'SubTitle')[0].text.strip()
        subtitle = ' [%s]' % subtitle
    except:
        subtitle = ""

    author = root.cssselect('.aegrid div')[1].text.strip()
    editor = root.cssselect('.aegrid div')[3].text.strip()

    copyright = root.cssselect(xwiprefix + 'Copyright')[0].text.strip()

    xd.set_header("Title", '%s%s' % (title, subtitle))
    xd.set_header("Author", author)
    xd.set_header("Editor", editor)
    xd.set_header("Copyright", copyright)

    # nyt title normally has date as e.g. January 1, 2020
    date_re = "(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},\s+\d{4}"
    try:
        m = re.search(date_re, subtitle if subtitle else title)
        date_string = m.group(0)
        date = datetime.strptime(date_string, "%B %d, %Y")
        xd.set_header("Date", date.strftime("%Y-%m-%d"))
    except:
        pass

    _process_notes(xd, xwiprefix, root) # add header for notes, if any

    puzzle_table = root.cssselect(xwiprefix + 'PuzTable tr') or root.cssselect('#PuzTable tr')

    for row in puzzle_table:
        row_data = ""
        for cell in row.cssselect('td'):
            # check if the cell is special - with a shade or a circle
            cell_class = cell.get('class')
            cell_type = ''
            if cell_class == 'shade':
                cell_type = 'shaded'
            elif cell_class == 'bigcircle':
                cell_type = 'circle'

            letter = cell.cssselect('div.letter')
            letter = (len(letter) and letter[0].text) or xdfile.BLOCK_CHAR

            # handle rebuses
            if letter == xdfile.BLOCK_CHAR:
                subst = cell.cssselect('div.subst2')
                subst = (len(subst) and subst[0].text) or ''
                if not subst:
                    subst = cell.cssselect('div.subst')
                    if subst:
                        if title in SPLIT_REBUS_TITLES:
                            subst = "/".join(list(subst[0].text))
                        else:
                            subst = subst[0].text
                    else: # check if color rebus
                        cell_string = etree.tostring(cell).decode('utf-8')
                        m = re.search("background-color:([A-Z]+);", cell_string)
                        if m:
                            subst = m.group(1)
                        else:
                            subst = ''

                if subst:
                    if subst not in rebus:
                        if subst in REBUS_LONG_HANDS:
                            rebus_val = REBUS_LONG_HANDS[subst]
                            if rebus_val in REBUS_SHORT_HANDS:
                                REBUS_SHORT_HANDS.remove(rebus_val)
                        else:
                            rebus_val = REBUS_SHORT_HANDS.pop()
                        rebus[subst] = rebus_val
                        rebus_order.append(subst)
                    letter = rebus[subst]

            if cell_type:
                # the special cell's letter should be represented in lower case
                letter = letter.lower()
                if not special_type:
                    # hopefully there shouldn't be both shades and circles in
                    # the same puzzle - if that is the case, only the last value
                    # will be put up in the header
                    special_type = cell_type

            row_data += letter
        xd.grid.append(row_data)

    if len(rebus):
        rebus = ["%s=%s" % (rebus[x], x.upper()) for x in rebus_order]
        xd.set_header("Rebus", ','.join(rebus))
    if special_type:
        xd.set_header("Special", special_type)

    across_div = root.cssselect('#ACluesPan') or root.cssselect(xwiprefix + 'ACluesPan')
    down_div = root.cssselect('#DCluesPan') or root.cssselect(xwiprefix + 'DCluesPan')

    if across_div and down_div: # normal puzzle
        _process_clues(xd, 'A', across_div) # add across clues
        _process_clues(xd, 'D', down_div) # add down clues
    elif across_div: # uniclue puzzle?
        _process_uniclues(xd, across_div)
    else:
        raise XWordInfoParseError("No clue divs found.")

    return xd

def _process_notes(xd, xwiprefix, root):
    note_div = root.cssselect('#notepad') or root.cssselect(xwiprefix + 'NotepadDiv')
    if note_div:
        note_div_string = etree.tostring(note_div[0]).decode('utf-8')
        note_div_string = note_div_string.replace("
", "\n")
        note_div_string = note_div_string.replace("{*Notepad:*}", "\n")
        note_div_string = note_div_string.replace("
", "\n")
        note_div_string = note_div_string.strip()
        note_div = html.fromstring(note_div_string)
        note_text = note_div.text_content()
        note_text = note_text.replace("\n", "   ")
        note_text = note_text.strip()
        xd.set_header("Notes", note_text)
    elif root.cssselect(xwiprefix + 'UnicluePan'):
        note_text = ("This was published as a uniclue puzzle in print. " +
        "All the clues appear in a single list, combining Across and Down. " +
        "When two answers share a number, they also share a clue.")
        xd.set_header("Notes", note_text)

def _process_clues(xd, clueprefix, clues_div):
    error_text = 'Parsing %s clues failed. ' % ('Across' if clueprefix == 'A' else 'Down')
    numclue_divs = clues_div[0].cssselect('.numclue div')
    if len(numclue_divs) % 2 != 0:
        raise XWordInfoParseError(error_text +
            'Either the number of numbers does not match the ' +
            'number of clues, or there are additional unknown divs.')
    for i in range(0, len(numclue_divs), 2):
        num = numclue_divs[i].text
        clue_div = numclue_divs[i + 1]
        clue_end = clue_div.text.rfind(' :')
        if clue_end < 0:
            raise XWordInfoParseError(error_text +
                'Malformed clue for number %s.' % num)
        clue = clue_div.text[:clue_end]
        anchor = clue_div.cssselect('a')
        if len(anchor) != 1:
            raise XWordInfoParseError(error_text +
                'Not exactly one anchor in clue div for number %s.' % num)
        else:
            answer = anchor[0].text
            xd.clues.append(((clueprefix, num), clue, answer))

def _process_uniclues(xd, clues_div):
    error_text = 'Parsing suspected uniclues failed. '
    grid_answers = xd.iteranswers()
    down_clues = []
    numclue_divs = clues_div[0].cssselect('.numclue div')
    if len(numclue_divs) % 2 != 0:
        raise XWordInfoParseError(error_text +
            'Either the number of numbers does not match the ' +
            'number of clues, or there are additional unknown divs.')
    for i in range(0, len(numclue_divs), 2):
        num = numclue_divs[i].text
        clue_div = numclue_divs[i + 1]
        clue_end = clue_div.text.rfind(' :')
        if clue_end < 0:
            raise XWordInfoParseError(error_text +
                'Malformed clue for number %s.' % num)
        clue = clue_div.text[:clue_end]
        anchor = clue_div.cssselect('a')
        if not anchor or len(anchor) > 2:
            raise XWordInfoParseError(error_text +
                'Neither 1 nor 2 anchors in clue div for number %s.' % num)
        for a in anchor:
            answer = a.text
            direction, grid_num, _ = next(grid_answers)
            if direction == 'A':
                xd.clues.append(((direction, grid_num), clue, answer))
            else:
                down_clues.append(((direction, grid_num), clue, answer))
    for clue in down_clues:
        xd.clues.append(clue)

if __name__ == "__main__":
    import sys
    from .utils import find_files
    for fn, contents in find_files(*sys.argv[1:]):
        xd = parse_xwordinfo(contents, fn)
        print("--- %s ---" % fn)
        print(xd.to_unicode())
<|endoftext|>


<commit_before>import logging
import os
import re
import sys
import traceback
import yaml
import types

from sanitize import sanitize
from recursivedefaultdict import recursivedefaultdict
import env
import json
import local
from to_xml import to_xml

from google.appengine.api import users, memcache
from google.appengine.api.datastore_errors import NeedIndexError
import google.appengine.ext.webapp
import jinja2

class NotFoundException(Exception):
  pass

class RequestHandler(google.appengine.ext.webapp.RequestHandler):
  # These constants are used to locate the default templates.
  HANDLERS_BASE = 'handlers'
  TEMPLATES_BASE = 'templates'
  NOT_FOUND_HTML = 'not_found.html'
  ERROR_HTML = 'error.html'

  def __init__(self):
    self.__response_dict__ = recursivedefaultdict()
    self.jinja = jinja2.Environment(loader=jinja2.FileSystemLoader(self.TEMPLATES_BASE))

  @classmethod
  def with_page(cls, page):
    if isinstance(page, types.ModuleType):
      return type(page.__file__, (cls,), dict(page=page))
    if isinstance(page, str):
      try:
        __import__(page)
        return cls.with_page(page=sys.modules[page])
      except ImportError:
        logging.error("missing handler for %s", page)

  @classmethod
  def path_with_page(cls, path, page):
    return (path+"\.?(yaml|json|xml|atom)?", cls.with_page(page))

  def response_dict(self, **kwargs):
    """Returns the response dictionary and sets the given values."""
    if kwargs:
      self.__response_dict__.update(**kwargs)
    return self.__response_dict__

  def url_args(self):
    """Returns the URL arguments in the WSGI paths."""
    return self.__url_args__

  def url_arg(self, index):
    """Returns the URL arg at the given index or None."""
    args = self.url_args()
    if args and 0 <= index < len(args):
       return args[index]

  def get(self, *args):
    """Responds to GET requests from WSGIApplication."""
    if self.has_param('post'):
      return self.post(*args)
    # check for trailing slashes
    match = re.compile('^(/.*[^/])/+$').search(self.request.path)
    if match and match.groups(1):
      # strip trailing slashes and redirect
      return self.redirect(match.group(1))
    # check if we can respond
    if hasattr(self.page, 'get'):
      # run the handler and get the template path
      path = self.handle(self.page.get, *args)
    else:
      # return with 405
      path = self.not_found(status=405)
    # render the template
    if self.is_atom():
      # for atom
      self.response.headers['Content-Type'] = "application/atom+xml; charset=UTF-8"
      self.render(path, 'atom')
    else:
      # for html
      self.render(path)

  def post(self, *args):
    """Responds to POST requests from WSGIApplication"""
    self.__url_args__ = args
    # check if we can post
    if hasattr(self.page, 'post'):
      # run the handler and get the template path
      path = self.handle(self.page.post, *args)
    else:
      # return with 405
      path = self.not_found(status=405)
    # if we encountered errors, run the get handler
    if self.has_errors():
      self.get(*args)
    else:
      # otherwise render the template
      self.render(path)

  def has_errors(self):
    """Returns if the response dictionary contains form errors."""
    return 'errors' in self.response_dict()

  def is_admin(self):
    """Returns if the current user is an admin."""
    return users.is_current_user_admin()

  def has_param(self, param):
    return len(self.request.get_all(param)) > 0

  def is_json(self):
    """Returns if the current request is for JSON."""
    return self.has_param('json') or self.request.path.endswith('.json')

  def is_yaml(self):
    """Returns if the current request is for YAML."""
    return self.has_param('yaml') or self.request.path.endswith('.yaml')

  def is_xml(self):
    """Returns if the current request is for XML."""
    return self.has_param('xml') or self.request.path.endswith('.xml')

  def is_atom(self):
    """Returns if the current request is for Atom."""
    return self.has_param('atom') or self.request.path.endswith('.atom')

  def logout_url(self):
    """Returns the logout URL of the current request."""
    return users.create_logout_url(self.request.uri)

  def login_url(self):
    """Returns the login URL of the current request."""
    return users.create_login_url(self.request.uri)

  def host(self):
    """Returns the current host's name."""
    return os.environ.get('HTTP_HOST')

  def cache_key(self, page=None, vary=None):
    page_name = self.page_name(page=page)
    return '-'.join([str(x) for x in (page_name, vary) if x])

  def cached(self, vary=None):
    """Returns if the current page is cached and updates the response dict with the cached values."""
    if self.has_param('no_cache'):
      return
    cached = memcache.get(
      key=self.cache_key(vary=vary),
      namespace="handler-cache")
    if cached:
      # update the response
      self.response_dict(**cached)
      return True

  def cache(self, time=0, vary=None, **kwargs):
    """Caches and updates the response dict with the given values for the current page."""
    memcache.set(
      key=self.cache_key(vary=vary),
      value=kwargs,
      time=time,
      namespace="handler-cache")
    # update the response
    self.response_dict(**kwargs)

  def invalidate(self, page=None, vary=None):
    """Invalidates the cache for given page or the current page."""
    memcache.delete(
      key=self.cache_key(vary=vary, page=page),
      namespace="handler-cache")

  def page_name(self, page=None):
    """Returns the name of the given page or the current page."""
    if not page:
      page = self.page
    match = re.compile("%s/([^.]*)" % self.HANDLERS_BASE).search(page.__file__)
    if match:
      return match.group(1)

  def default_template(self, ext="html"):
    """Returns the path for the current page's default template."""
    name = self.page_name()
    if name:
      return "%s.%s" % (name, ext)
    raise Exception("failed to build default template for %s" % page)

  def handle(self, method, *args):
    """Invokes the given method and return the template path to render."""
    self.__url_args__ = args
    try:
      return method(self, self.response_dict())
    except NotFoundException:
      return self.not_found()
    except NeedIndexError:
      return self.not_found(status=503)
    except:
      return self.handle_error()

  def handle_error(self):
    """Prepares a traceback for 500 errors, returns error template path."""
    (error_type, error, tb) = sys.exc_info()
    tb_formatted = traceback.format_tb(tb)
    error_type = error_type.mro()[0].__name__
    self.response_dict(
      error=error,
      error_type=error_type,
      tb_formatted=tb_formatted)
    logging.error("%s: %s", error_type, error)
    self.set_status(status=500)
    return self.ERROR_HTML

  def render(self, path, base="html"):
    """Renders the given template or the default template, or JSON(P)/YAML."""
    if self.is_json():
      sanitized = sanitize(self.response_dict())
      json_str = json.write(sanitized)
      callback = self.request.get('callback')
      if re.match("^[_a-z]([_a-z0-9])*$", callback, re.IGNORECASE):
        json_str = "%s(%s)" % (callback, json_str) # jsonp
      self.response.headers['Content-Type'] = "text/javascript; charset=UTF-8"
      self.response.out.write(json_str)
      return
    if self.is_yaml():
      sanitized = sanitize(self.response_dict())
      yaml_str = yaml.safe_dump(sanitized, default_flow_style=False)
      self.response.headers['Content-Type'] = "text/plain; charset=UTF-8"
      self.response.out.write(yaml_str)
      return
    if self.is_xml():
      sanitized = sanitize(self.response_dict())
      xml_str = to_xml(value=sanitized, root="response")
      self.response.headers['Content-Type'] = "text/xml; charset=UTF-8"
      self.response.out.write(xml_str)
      return
    if not path:
      path = self.default_template(ext=base)
    full_path = os.path.join(self.TEMPLATES_BASE, path)
    if self.file_exists(full_path):
      try:
        # the template might find these handy
        self.response_dict(
          handler=self,
          is_dev=env.is_dev()
        )
        template = self.jinja.get_template(path)
        rendered = template.render(**self.response_dict())
        self.response.out.write(rendered)
      except jinja2.TemplateError, error:
        self.response.headers['Content-Type'] = 'text/plain'
        message = "Template syntax error: %s" % error
        logging.critical(message)
        (error_type, error, tb) = sys.exc_info()
        tb_formatted = traceback.format_tb(tb)
        self.response.out.write("\n".join([message]+tb_formatted))
    elif path is self.ERROR_HTML:
      self.response.headers['Content-Type'] = 'text/plain'
      self.response.out.write("%s %s" % (self.get_status(), self.get_error()))
    elif path is self.NOT_FOUND_HTML:
      self.response.headers['Content-Type'] = 'text/plain'
      self.response.out.write("%s %s" % (self.get_status(), 'not found'))
    else:
      self.response.headers['Content-Type'] = 'text/plain'
      message = "Template not found: %s" % path
      logging.critical(message)
      self.response.out.write(message)

  def file_exists(self, path):
    return os.path.exists(path)

  def redirect(self, *args):
    """Redirects to the given location (unless in JSON/YAML mode)."""
    if not self.is_json() and not self.is_yaml():
      super(RequestHandler, self).redirect(*args)

  def not_found(self, status=404):
    """Returns generic not-found template."""
    self.set_status(status=status)
    return self.NOT_FOUND_HTML

  def set_status(self, status):
    """Sets the response status."""
    self.response_dict(status=status)
    self.response.set_status(code=status)

  def get_status(self):
    """Returns the response status."""
    response_dict = self.response_dict()
    return response_dict.status

  def get_error(self):
    """Returns the response error message if any."""
    response_dict = self.response_dict()
    return response_dict.error

  def form_error(self, **kwargs):
    """Set the given form errors."""
    response = self.response_dict()
    for key, value in kwargs.iteritems():
      response.errors[key] = value

  def config(self):
    """Returns the local configuration."""
    return local.config()
<commit_msg>don't render when redirecting after a post<commit_after>import logging
import os
import re
import sys
import traceback
import yaml
import types

from sanitize import sanitize
from recursivedefaultdict import recursivedefaultdict
import env
import json
import local
from to_xml import to_xml

from google.appengine.api import users, memcache
from google.appengine.api.datastore_errors import NeedIndexError
import google.appengine.ext.webapp
import jinja2

class NotFoundException(Exception):
  pass

class RequestHandler(google.appengine.ext.webapp.RequestHandler):
  # These constants are used to locate the default templates.
  HANDLERS_BASE = 'handlers'
  TEMPLATES_BASE = 'templates'
  NOT_FOUND_HTML = 'not_found.html'
  ERROR_HTML = 'error.html'

  def __init__(self):
    self.__response_dict__ = recursivedefaultdict()
    self.jinja = jinja2.Environment(loader=jinja2.FileSystemLoader(self.TEMPLATES_BASE))

  @classmethod
  def with_page(cls, page):
    if isinstance(page, types.ModuleType):
      return type(page.__file__, (cls,), dict(page=page))
    if isinstance(page, str):
      try:
        __import__(page)
        return cls.with_page(page=sys.modules[page])
      except ImportError:
        logging.error("missing handler for %s", page)

  @classmethod
  def path_with_page(cls, path, page):
    return (path+"\.?(yaml|json|xml|atom)?", cls.with_page(page))

  def response_dict(self, **kwargs):
    """Returns the response dictionary and sets the given values."""
    if kwargs:
      self.__response_dict__.update(**kwargs)
    return self.__response_dict__

  def url_args(self):
    """Returns the URL arguments in the WSGI paths."""
    return self.__url_args__

  def url_arg(self, index):
    """Returns the URL arg at the given index or None."""
    args = self.url_args()
    if args and 0 <= index < len(args):
       return args[index]

  def get(self, *args):
    """Responds to GET requests from WSGIApplication."""
    if self.has_param('post'):
      return self.post(*args)
    # check for trailing slashes
    match = re.compile('^(/.*[^/])/+$').search(self.request.path)
    if match and match.groups(1):
      # strip trailing slashes and redirect
      return self.redirect(match.group(1))
    # check if we can respond
    if hasattr(self.page, 'get'):
      # run the handler and get the template path
      path = self.handle(self.page.get, *args)
    else:
      # return with 405
      path = self.not_found(status=405)
    # render the template
    if self.is_atom():
      # for atom
      self.response.headers['Content-Type'] = "application/atom+xml; charset=UTF-8"
      self.render(path, 'atom')
    else:
      # for html
      self.render(path)

  def post(self, *args):
    """Responds to POST requests from WSGIApplication"""
    self.__url_args__ = args
    # check if we can post
    if hasattr(self.page, 'post'):
      # run the handler and get the template path
      path = self.handle(self.page.post, *args)
    else:
      # return with 405
      path = self.not_found(status=405)
    # if we encountered errors, run the get handler
    if self.has_errors():
      self.get(*args)
    elif self._is_redirect and 'Location' in self.response.headers:
      location = self.response.headers['Location'];
      self.response.out.write("""Moved to %s\n""" % (location, location))
    else:
      # otherwise render the template
      self.render(path)

  def has_errors(self):
    """Returns if the response dictionary contains form errors."""
    return 'errors' in self.response_dict()

  def is_admin(self):
    """Returns if the current user is an admin."""
    return users.is_current_user_admin()

  def has_param(self, param):
    return len(self.request.get_all(param)) > 0

  def is_json(self):
    """Returns if the current request is for JSON."""
    return self.has_param('json') or self.request.path.endswith('.json')

  def is_yaml(self):
    """Returns if the current request is for YAML."""
    return self.has_param('yaml') or self.request.path.endswith('.yaml')

  def is_xml(self):
    """Returns if the current request is for XML."""
    return self.has_param('xml') or self.request.path.endswith('.xml')

  def is_atom(self):
    """Returns if the current request is for Atom."""
    return self.has_param('atom') or self.request.path.endswith('.atom')

  def logout_url(self):
    """Returns the logout URL of the current request."""
    return users.create_logout_url(self.request.uri)

  def login_url(self):
    """Returns the login URL of the current request."""
    return users.create_login_url(self.request.uri)

  def host(self):
    """Returns the current host's name."""
    return os.environ.get('HTTP_HOST')

  def cache_key(self, page=None, vary=None):
    page_name = self.page_name(page=page)
    return '-'.join([str(x) for x in (page_name, vary) if x])

  def cached(self, vary=None):
    """Returns if the current page is cached and updates the response dict with the cached values."""
    if self.has_param('no_cache'):
      return
    cached = memcache.get(
      key=self.cache_key(vary=vary),
      namespace="handler-cache")
    if cached:
      # update the response
      self.response_dict(**cached)
      return True

  def cache(self, time=0, vary=None, **kwargs):
    """Caches and updates the response dict with the given values for the current page."""
    memcache.set(
      key=self.cache_key(vary=vary),
      value=kwargs,
      time=time,
      namespace="handler-cache")
    # update the response
    self.response_dict(**kwargs)

  def invalidate(self, page=None, vary=None):
    """Invalidates the cache for given page or the current page."""
    memcache.delete(
      key=self.cache_key(vary=vary, page=page),
      namespace="handler-cache")

  def page_name(self, page=None):
    """Returns the name of the given page or the current page."""
    if not page:
      page = self.page
    match = re.compile("%s/([^.]*)" % self.HANDLERS_BASE).search(page.__file__)
    if match:
      return match.group(1)

  def default_template(self, ext="html"):
    """Returns the path for the current page's default template."""
    name = self.page_name()
    if name:
      return "%s.%s" % (name, ext)
    raise Exception("failed to build default template for %s" % page)

  def handle(self, method, *args):
    """Invokes the given method and return the template path to render."""
    self.__url_args__ = args
    try:
      return method(self, self.response_dict())
    except NotFoundException:
      return self.not_found()
    except NeedIndexError:
      return self.not_found(status=503)
    except:
      return self.handle_error()

  def handle_error(self):
    """Prepares a traceback for 500 errors, returns error template path."""
    (error_type, error, tb) = sys.exc_info()
    tb_formatted = traceback.format_tb(tb)
    error_type = error_type.mro()[0].__name__
    self.response_dict(
      error=error,
      error_type=error_type,
      tb_formatted=tb_formatted)
    logging.error("%s: %s", error_type, error)
    self.set_status(status=500)
    return self.ERROR_HTML

  def render(self, path, base="html"):
    """Renders the given template or the default template, or JSON(P)/YAML."""
    if self.is_json():
      sanitized = sanitize(self.response_dict())
      json_str = json.write(sanitized)
      callback = self.request.get('callback')
      if re.match("^[_a-z]([_a-z0-9])*$", callback, re.IGNORECASE):
        json_str = "%s(%s)" % (callback, json_str) # jsonp
      self.response.headers['Content-Type'] = "text/javascript; charset=UTF-8"
      self.response.out.write(json_str)
      return
    if self.is_yaml():
      sanitized = sanitize(self.response_dict())
      yaml_str = yaml.safe_dump(sanitized, default_flow_style=False)
      self.response.headers['Content-Type'] = "text/plain; charset=UTF-8"
      self.response.out.write(yaml_str)
      return
    if self.is_xml():
      sanitized = sanitize(self.response_dict())
      xml_str = to_xml(value=sanitized, root="response")
      self.response.headers['Content-Type'] = "text/xml; charset=UTF-8"
      self.response.out.write(xml_str)
      return
    if not path:
      path = self.default_template(ext=base)
    full_path = os.path.join(self.TEMPLATES_BASE, path)
    if self.file_exists(full_path):
      try:
        # the template might find these handy
        self.response_dict(
          handler=self,
          is_dev=env.is_dev()
        )
        template = self.jinja.get_template(path)
        rendered = template.render(**self.response_dict())
        self.response.out.write(rendered)
      except jinja2.TemplateError, error:
        self.response.headers['Content-Type'] = 'text/plain'
        message = "Template syntax error: %s" % error
        logging.critical(message)
        (error_type, error, tb) = sys.exc_info()
        tb_formatted = traceback.format_tb(tb)
        self.response.out.write("\n".join([message]+tb_formatted))
    elif path is self.ERROR_HTML:
      self.response.headers['Content-Type'] = 'text/plain'
      self.response.out.write("%s %s" % (self.get_status(), self.get_error()))
    elif path is self.NOT_FOUND_HTML:
      self.response.headers['Content-Type'] = 'text/plain'
      self.response.out.write("%s %s" % (self.get_status(), 'not found'))
    else:
      self.response.headers['Content-Type'] = 'text/plain'
      message = "Template not found: %s" % path
      logging.critical(message)
      self.response.out.write(message)

  def file_exists(self, path):
    return os.path.exists(path)

  def redirect(self, *args):
    """Redirects to the given location (unless in JSON/YAML mode)."""
    if not self.is_json() and not self.is_yaml():
      self._is_redirect = 1
      super(RequestHandler, self).redirect(*args)

  def not_found(self, status=404):
    """Returns generic not-found template."""
    self.set_status(status=status)
    return self.NOT_FOUND_HTML

  def set_status(self, status):
    """Sets the response status."""
    self.response_dict(status=status)
    self.response.set_status(code=status)

  def get_status(self):
    """Returns the response status."""
    response_dict = self.response_dict()
    return response_dict.status

  def get_error(self):
    """Returns the response error message if any."""
    response_dict = self.response_dict()
    return response_dict.error

  def form_error(self, **kwargs):
    """Set the given form errors."""
    response = self.response_dict()
    for key, value in kwargs.iteritems():
      response.errors[key] = value

  def config(self):
    """Returns the local configuration."""
    return local.config()
<|endoftext|>


<commit_before>__version__ = '0.9.2'
<commit_msg>bump version, no changelog yet<commit_after>__version__ = '0.10.0'
<|endoftext|>


<commit_before>from time import sleep

from django.core.mail import send_mail
from datetime import date
from django.db.models import Q
from django.http import HttpResponse
from django.utils import timezone
from history.models import Distinction, Officer
from requirements.models import (
                    ProgressItem,
                    DistinctionType,
                    Requirement,
                    EventCategory
)
from mig_main.models import AcademicTerm, MemberProfile
from mig_main.utility import UnicodeWriter

def get_active_members(term):
    members = MemberProfile.get_actives()
    terms = [term, term.get_previous_full_term()]
    officer_terms = Officer.objects.filter(term__in=[term])
    distinctions = Distinction.objects.filter(term__in=terms)
    query_active_status = Q(distinction__in=distinctions)
    query_initiated_last_term = Q(init_term=term.get_previous_full_term())
    query_officer = Q(officer__in=officer_terms)
    query_alumni = (Q(standing__name='Alumni') |
                    Q(expect_grad_date__lt=date.today()))
    query = query_officer | (
            (query_active_status | query_initiated_last_term) & ~query_alumni
    )

    return members.filter(query).distinct()

def get_members_who_graduated():
    members = MemberProfile.get_actives().exclude(standing__name='Alumni')

    return members.filter(expect_grad_date__lt=date.today()).distinct()

def get_active_members_who_came_to_something(term):
    members = get_active_members(term)
    progress_items = ProgressItem.objects.filter(
                                    term=term,
                                    event_type__name='Meeting Attendance'
    )
    return members.filter(progressitem__in=progress_items).distinct()

def get_active_members_only_if_they_come(term, is_last_voting_meeting=False):
    members = MemberProfile.get_actives()
    terms = [term, term.get_previous_full_term()]
    officer_terms = Officer.objects.filter(term__in=[term])
    distinctions = Distinction.objects.filter(term__in=terms)
    query_active_status = Q(distinction__in=distinctions)
    query_initiated_last_term = Q(init_term=term.get_previous_full_term())
    query_officer = Q(officer__in=officer_terms)
    query_alumni = (Q(standing__name='Alumni') |
                    Q(expect_grad_date__lt=date.today()))
    query_all_active = query_officer | (
            (query_active_status | query_initiated_last_term) & ~ query_alumni
    )
    progress_items = ProgressItem.objects.filter(
                                    term=term,
                                    event_type__name='Meeting Attendance'
    )
    query_actives_absent_now = ~Q(progressitem__in=progress_items)
    # figure out actives will gain status if they have the meeting credit

    query_inactives_need_meeting = (
        ~query_officer & ~query_active_status & ~query_initiated_last_term & ~query_alumni
    )
    query = (query_all_active & query_actives_absent_now)
    set1 = set(members.filter(query).distinct()[:])
    active_dist = DistinctionType.objects.get(name='Active')
    temp_active_ok = not is_last_voting_meeting
    set2 = set(
            members.filter(
                    query_inactives_need_meeting).distinct()[:]
        ).intersection(
            set(
                active_dist.get_actives_with_status(
                                            term,
                                            temp_active_ok=temp_active_ok
                )
            )
        )

    return list(set1.union(set2))

def get_quorum_list():
    term = AcademicTerm.get_current_term()
    all_actives = MemberProfile.get_actives()
    active_actives = get_active_members(term)
    members_who_graduated = get_members_who_graduated()
    actual_actives = get_active_members_who_came_to_something(term)
    potential_actives = get_active_members_only_if_they_come(term)
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="MemberStatus.csv"'

    writer = UnicodeWriter(response)
    writer.writerow([
            'First Name',
            'Last Name',
            'uniqname',
            'Active?',
            'Alumni?',
            'Present'
    ])
    for m in all_actives:
        if m in potential_actives:
            active = 'If present'
        elif m in actual_actives:
            active = 'Yes'
        elif m.standing.name == 'Alumni':
            active = 'Confirm Manually'
        else:
            active = 'No'
        if m in members_who_graduated:
            alum_text = 'Maybe'
        elif m.standing.name == 'Alumni':
            alum_text = 'Yes'
        else:
            alum_text = 'No'
        writer.writerow([
                m.first_name,
                m.last_name,
                m.uniqname,
                active,
                alum_text,
                ''
        ])
    return response

def get_quorum_list_elections():
    term = AcademicTerm.get_current_term()
    all_actives = MemberProfile.get_actives()
    electees = MemberProfile.get_electees()
    active_actives = get_active_members(term)
    members_who_graduated = get_members_who_graduated()
    actual_actives = get_active_members_who_came_to_something(term)
    potential_actives = get_active_members_only_if_they_come(
                                            term,
                                            is_last_voting_meeting=True
    )
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="MemberStatus.csv"'

    writer = UnicodeWriter(response)
    writer.writerow([
            'First Name',
            'Last Name',
            'uniqname',
            'Active?',
            'Alumni?',
            'Present'
    ])
    for m in all_actives:
        if m in potential_actives:
            active = 'If present'
        elif m in actual_actives:
            active = 'Yes'
        elif m.standing.name == 'Alumni':
            active = 'Confirm Manually'
        else:
            active = 'No'
        if m in members_who_graduated:
            alum_text = 'Maybe'
        elif m.standing.name == 'Alumni':
            alum_text = 'Yes'
        else:
            alum_text = 'No'
        writer.writerow([
                m.first_name,
                m.last_name,
                m.uniqname,
                active,
                alum_text,
                ''
        ])
    for m in electees:
        writer.writerow([
                m.first_name,
                m.last_name,
                m.uniqname,
                'Electee',
                'No',
                ''
        ])
    return response

def email_active_status(meeting, is_elections):
    term = AcademicTerm.get_current_term()
    all_actives = MemberProfile.get_actives()
    active_actives = get_active_members(term)
    members_who_graduated = get_members_who_graduated()
    actual_actives = get_active_members_who_came_to_something(term)
    potential_actives = get_active_members_only_if_they_come(
                            term,
                            is_last_voting_meeting=is_elections
    )
    body_template = r'''Hi %(member)s,
This is a friendly reminder that we have a critical voting meeting -
%(name)s - %(date)s
in %(location)s and we need to have a quorum of active members
present. Please remember to bring your iClickers. In addition, we will
be voting on the ratification ballot for the TBP Association ("nationals")
so active member attendance is very important.

Our records indicate that %(status)s

%(alumni)s

If you believe this to be in error, or especially if you are no longer on
campus, please let us know by emailing tbp-website@umich.edu or by speaking
to the president or website chair at the meeting tonight.

Thanks,
The TBP Website'''
    emailed_people = []
    location = meeting.get_locations()
    if len(location)>0:
        location = location[0]
    else:
        location = 'TBD'
    start_end = meeting.get_start_and_end()
    start = timezone.localtime(start_end['start']).strftime('%A (%b %d) at %I:%M%p')
    start_string = 'this '+start
    event_name = meeting.name
    print event_name
    print location
    sleep(5)
    for m in all_actives:
        print 'emailing ' + m.uniqname+'...'
        sleep(1)
        short_code = 'active'
        status_code = ''
        if m in potential_actives:
            status_text = (' you will be considered active and eligible '
                           'to vote upon attending the meeting. While '
                           'your absence will not count against quorum, '
                           'please be advised that voting meetings are '
                           'required to achieve DA/PA status.')
            short_code = 'conditional active'
        elif m in actual_actives:
            status_text = (' you are an active member. You will be '
                           'eligible to vote at the meeting and will '
                           'count against quorum if you cannot or do not '
                           'attend tonight.')
        elif m.standing.name == 'Alumni':
            continue
        else:
            status_text = (' you are no longer active in the chapter. '
                           'You are welcome to attend the meeting, but '
                           'you will be unable to vote.')
            short_code = 'not active'
            if m in members_who_graduated:
                status_text += (' This may be that you are listed as '
                                'having graduated. Alumni may specially '
                                'request active status, but may not vote '
                                'on candidate election')
                short_code = 'non-active alum'
        if m in members_who_graduated:
            alum_text = ('Our records additionally indicate that you have '
                         'likely graduated but are not yet listed as an '
                         'alumni. If this is the case, please let us know '
                         'and update your website profile accordingly. If '
                         'not please update your expected graduation date '
                         'accordingly. Those listed as alumni will be '
                         'ineligible to vote on candidate election.')
            status_code = 'graduated'
        elif m.standing.name == 'Alumni':
            alum_text = (' Our records have you noted as an alumni. Note '
                         'that regardless of active status, alumni may '
                         'not vote on candidate election or changes to '
                         'the initiation fee.')
            status_code = 'alum'
        else:
            alum_text = ''
        body = body_template % {
                    'member': m.first_name,
                    'status': status_text,
                    'alumni': alum_text,
                    'location': location,
                    'date': start_string,
                    'name': event_name
        }
        emailed_people.append([m.uniqname, short_code, status_code])
        send_mail(
            '[TBP] Voting meeting active status update.',
            body,
            'tbp.mi.g@gmail.com',
            [m.get_email()],
            fail_silently=False
        )
    web_summary_body = 'The following members were emailed:\n\n'
    web_summary_body += '\n'.join(
            ['\t\t'.join(sub_list) for sub_list in emailed_people])
    send_mail(
        '[TBP] Voting meeting active status update - summary.',
        web_summary_body,
        'tbp.mi.g@gmail.com',
        ['tbp-website@umich.edu'],
        fail_silently=False
    )<commit_msg>updated active email<commit_after>from time import sleep

from django.core.mail import send_mail
from datetime import date
from django.db.models import Q
from django.http import HttpResponse
from django.utils import timezone
from history.models import Distinction, Officer
from requirements.models import (
                    ProgressItem,
                    DistinctionType,
                    Requirement,
                    EventCategory
)
from mig_main.models import AcademicTerm, MemberProfile
from mig_main.utility import UnicodeWriter

def get_active_members(term):
    members = MemberProfile.get_actives()
    terms = [term, term.get_previous_full_term()]
    officer_terms = Officer.objects.filter(term__in=[term])
    distinctions = Distinction.objects.filter(term__in=terms)
    query_active_status = Q(distinction__in=distinctions)
    query_initiated_last_term = Q(init_term=term.get_previous_full_term())
    query_officer = Q(officer__in=officer_terms)
    query_alumni = (Q(standing__name='Alumni') |
                    Q(expect_grad_date__lt=date.today()))
    query = query_officer | (
            (query_active_status | query_initiated_last_term) & ~query_alumni
    )

    return members.filter(query).distinct()

def get_members_who_graduated():
    members = MemberProfile.get_actives().exclude(standing__name='Alumni')

    return members.filter(expect_grad_date__lt=date.today()).distinct()

def get_active_members_who_came_to_something(term):
    members = get_active_members(term)
    progress_items = ProgressItem.objects.filter(
                                    term=term,
                                    event_type__name='Meeting Attendance'
    )
    return members.filter(progressitem__in=progress_items).distinct()

def get_active_members_only_if_they_come(term, is_last_voting_meeting=False):
    members = MemberProfile.get_actives()
    terms = [term, term.get_previous_full_term()]
    officer_terms = Officer.objects.filter(term__in=[term])
    distinctions = Distinction.objects.filter(term__in=terms)
    query_active_status = Q(distinction__in=distinctions)
    query_initiated_last_term = Q(init_term=term.get_previous_full_term())
    query_officer = Q(officer__in=officer_terms)
    query_alumni = (Q(standing__name='Alumni') |
                    Q(expect_grad_date__lt=date.today()))
    query_all_active = query_officer | (
            (query_active_status | query_initiated_last_term) & ~ query_alumni
    )
    progress_items = ProgressItem.objects.filter(
                                    term=term,
                                    event_type__name='Meeting Attendance'
    )
    query_actives_absent_now = ~Q(progressitem__in=progress_items)
    # figure out actives will gain status if they have the meeting credit

    query_inactives_need_meeting = (
        ~query_officer & ~query_active_status & ~query_initiated_last_term & ~query_alumni
    )
    query = (query_all_active & query_actives_absent_now)
    set1 = set(members.filter(query).distinct()[:])
    active_dist = DistinctionType.objects.get(name='Active')
    temp_active_ok = not is_last_voting_meeting
    set2 = set(
            members.filter(
                    query_inactives_need_meeting).distinct()[:]
        ).intersection(
            set(
                active_dist.get_actives_with_status(
                                            term,
                                            temp_active_ok=temp_active_ok
                )
            )
        )

    return list(set1.union(set2))

def get_quorum_list():
    term = AcademicTerm.get_current_term()
    all_actives = MemberProfile.get_actives()
    active_actives = get_active_members(term)
    members_who_graduated = get_members_who_graduated()
    actual_actives = get_active_members_who_came_to_something(term)
    potential_actives = get_active_members_only_if_they_come(term)
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="MemberStatus.csv"'

    writer = UnicodeWriter(response)
    writer.writerow([
            'First Name',
            'Last Name',
            'uniqname',
            'Active?',
            'Alumni?',
            'Present'
    ])
    for m in all_actives:
        if m in potential_actives:
            active = 'If present'
        elif m in actual_actives:
            active = 'Yes'
        elif m.standing.name == 'Alumni':
            active = 'Confirm Manually'
        else:
            active = 'No'
        if m in members_who_graduated:
            alum_text = 'Maybe'
        elif m.standing.name == 'Alumni':
            alum_text = 'Yes'
        else:
            alum_text = 'No'
        writer.writerow([
                m.first_name,
                m.last_name,
                m.uniqname,
                active,
                alum_text,
                ''
        ])
    return response

def get_quorum_list_elections():
    term = AcademicTerm.get_current_term()
    all_actives = MemberProfile.get_actives()
    electees = MemberProfile.get_electees()
    active_actives = get_active_members(term)
    members_who_graduated = get_members_who_graduated()
    actual_actives = get_active_members_who_came_to_something(term)
    potential_actives = get_active_members_only_if_they_come(
                                            term,
                                            is_last_voting_meeting=True
    )
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="MemberStatus.csv"'

    writer = UnicodeWriter(response)
    writer.writerow([
            'First Name',
            'Last Name',
            'uniqname',
            'Active?',
            'Alumni?',
            'Present'
    ])
    for m in all_actives:
        if m in potential_actives:
            active = 'If present'
        elif m in actual_actives:
            active = 'Yes'
        elif m.standing.name == 'Alumni':
            active = 'Confirm Manually'
        else:
            active = 'No'
        if m in members_who_graduated:
            alum_text = 'Maybe'
        elif m.standing.name == 'Alumni':
            alum_text = 'Yes'
        else:
            alum_text = 'No'
        writer.writerow([
                m.first_name,
                m.last_name,
                m.uniqname,
                active,
                alum_text,
                ''
        ])
    for m in electees:
        writer.writerow([
                m.first_name,
                m.last_name,
                m.uniqname,
                'Electee',
                'No',
                ''
        ])
    return response

def email_active_status(meeting, is_elections):
    term = AcademicTerm.get_current_term()
    all_actives = MemberProfile.get_actives()
    active_actives = get_active_members(term)
    members_who_graduated = get_members_who_graduated()
    actual_actives = get_active_members_who_came_to_something(term)
    potential_actives = get_active_members_only_if_they_come(
                            term,
                            is_last_voting_meeting=is_elections
    )
    body_template = r'''Hi %(member)s,
This is a friendly reminder that we have a critical voting meeting -
%(name)s - %(date)s
in %(location)s and we need to have a quorum of active members
present. Please remember to bring your iClickers.

Our records indicate that %(status)s

%(alumni)s

If you believe this to be in error, or especially if you are no longer on
campus, please let us know by emailing tbp-website@umich.edu or by speaking
to the president or website chair at the meeting tonight.

Thanks,
The TBP Website'''
    emailed_people = []
    location = meeting.get_locations()
    if len(location)>0:
        location = location[0]
    else:
        location = 'TBD'
    start_end = meeting.get_start_and_end()
    start = timezone.localtime(start_end['start']).strftime('%A (%b %d) at %I:%M%p')
    start_string = 'this '+start
    event_name = meeting.name
    print event_name
    print location
    sleep(5)
    for m in all_actives:
        print 'emailing ' + m.uniqname+'...'
        sleep(1)
        short_code = 'active'
        status_code = ''
        if m in potential_actives:
            status_text = (' you will be considered active and eligible '
                           'to vote upon attending the meeting. While '
                           'your absence will not count against quorum, '
                           'please be advised that voting meetings are '
                           'required to achieve DA/PA status.')
            short_code = 'conditional active'
        elif m in actual_actives:
            status_text = (' you are an active member. You will be '
                           'eligible to vote at the meeting and will '
                           'count against quorum if you cannot or do not '
                           'attend tonight.')
        elif m.standing.name == 'Alumni':
            continue
        else:
            status_text = (' you are no longer active in the chapter. '
                           'You are welcome to attend the meeting, but '
                           'you will be unable to vote.')
            short_code = 'not active'
            if m in members_who_graduated:
                status_text += (' This may be that you are listed as '
                                'having graduated. Alumni may specially '
                                'request active status, but may not vote '
                                'on candidate election')
                short_code = 'non-active alum'
        if m in members_who_graduated:
            alum_text = ('Our records additionally indicate that you have '
                         'likely graduated but are not yet listed as an '
                         'alumni. If this is the case, please let us know '
                         'and update your website profile accordingly. If '
                         'not please update your expected graduation date '
                         'accordingly. Those listed as alumni will be '
                         'ineligible to vote on candidate election.')
            status_code = 'graduated'
        elif m.standing.name == 'Alumni':
            alum_text = (' Our records have you noted as an alumni. Note '
                         'that regardless of active status, alumni may '
                         'not vote on candidate election or changes to '
                         'the initiation fee.')
            status_code = 'alum'
        else:
            alum_text = ''
        body = body_template % {
                    'member': m.first_name,
                    'status': status_text,
                    'alumni': alum_text,
                    'location': location,
                    'date': start_string,
                    'name': event_name
        }
        emailed_people.append([m.uniqname, short_code, status_code])
        send_mail(
            '[TBP] Voting meeting active status update.',
            body,
            'tbp.mi.g@gmail.com',
            [m.get_email()],
            fail_silently=False
        )
    web_summary_body = 'The following members were emailed:\n\n'
    web_summary_body += '\n'.join(
            ['\t\t'.join(sub_list) for sub_list in emailed_people])
    send_mail(
        '[TBP] Voting meeting active status update - summary.',
        web_summary_body,
        'tbp.mi.g@gmail.com',
        ['tbp-website@umich.edu'],
        fail_silently=False
    )<|endoftext|>


<commit_before>import errno
import json
from os.path import dirname, join
import re
import sys
from urlparse import urlsplit

from mock import Mock
from slumber.exceptions import HttpClientError

example_popit_data_directory = join(
    dirname(__file__), '..', 'example-popit-data'
)

def get_example_popit_json(basename):
    with open(
        join(example_popit_data_directory, basename)
    ) as f:
        return json.load(f)

# In many cases we could get away without these fake PopIt API
# collection objects; for example, if you only need to return data on
# one organization in your test, and do a fetch of all people, you
# could do that with:
#
#   In [1]: from mock import Mock
#
#   In [2]: api = Mock(**{
#      ...:     'organizations.return_value.get.return_value': 'got-particular-org',
#      ...:     'persons.get.return_value': 'got-all-people'
#      ...: })
#
#   In [3]: api.organizations('national-assembly').get()
#   Out[3]: 'got-particular-org'
#
#   In [4]: api.organizations.get()
#   Out[4]: <Mock name='mock.organizations.get()' id='140711747445328'>
#
#   In [5]: api.persons.get()
#   Out[5]: 'got-all-people'
#
#   In [6]: api.persons('john-doe').get()
#   Out[6]: <Mock name='mock.persons().get()' id='140711747546640'>
#
# But that would return 'got-particular-org' no matter what ID was
# passed to api.organizations.

class FakeCollection(object):

    # This is the version of get() that's called when invoked on an
    # instance of this class (i.e. to get a single item from a
    # collection)
    def _instance_get(self, **kwargs):
        try:
            return get_example_popit_json(
                '{0}_{1}_embed={2}.json'.format(
                    self.collection,
                    self.object_id,
                    kwargs.get('embed', 'membership')))
        except IOError as e:
            if e.errno == errno.ENOENT:
                raise HttpClientError('Client Error 404')
            else:
                raise

    def __init__(self, *args):
        self.get = self._instance_get
        self.object_id = args[0] if len(args) == 1 else None

    # This is the version of get() that's called when invoked as a
    # class method (i.e. to get everything in a collection)
    @classmethod
    def get(cls, **kwargs):
        return get_example_popit_json(
            'generic_{0}_embed={1}.json'.format(
                cls.collection,
                kwargs.get('embed', 'membership')
            )
        )

    def delete(self):
        raise Exception("Not implemented: you should patch this")

    def put(self, data):
        raise Exception("Not implemented: you should patch this")

    @staticmethod
    def post(data):
        raise Exception("Not implemented: you should patch this")

class FakePersonCollection(FakeCollection):
    collection = 'persons'

class FakeOrganizationCollection(FakeCollection):
    collection = 'organizations'

class FakePostCollection(FakeCollection):
    collection = 'posts'
<commit_msg>Add a FakeMembershipCollection to the test helper classes<commit_after>import errno
import json
from os.path import dirname, join
import re
import sys
from urlparse import urlsplit

from mock import Mock
from slumber.exceptions import HttpClientError

example_popit_data_directory = join(
    dirname(__file__), '..', 'example-popit-data'
)

def get_example_popit_json(basename):
    with open(
        join(example_popit_data_directory, basename)
    ) as f:
        return json.load(f)

# In many cases we could get away without these fake PopIt API
# collection objects; for example, if you only need to return data on
# one organization in your test, and do a fetch of all people, you
# could do that with:
#
#   In [1]: from mock import Mock
#
#   In [2]: api = Mock(**{
#      ...:     'organizations.return_value.get.return_value': 'got-particular-org',
#      ...:     'persons.get.return_value': 'got-all-people'
#      ...: })
#
#   In [3]: api.organizations('national-assembly').get()
#   Out[3]: 'got-particular-org'
#
#   In [4]: api.organizations.get()
#   Out[4]: <Mock name='mock.organizations.get()' id='140711747445328'>
#
#   In [5]: api.persons.get()
#   Out[5]: 'got-all-people'
#
#   In [6]: api.persons('john-doe').get()
#   Out[6]: <Mock name='mock.persons().get()' id='140711747546640'>
#
# But that would return 'got-particular-org' no matter what ID was
# passed to api.organizations.

class FakeCollection(object):

    # This is the version of get() that's called when invoked on an
    # instance of this class (i.e. to get a single item from a
    # collection)
    def _instance_get(self, **kwargs):
        try:
            return get_example_popit_json(
                '{0}_{1}_embed={2}.json'.format(
                    self.collection,
                    self.object_id,
                    kwargs.get('embed', 'membership')))
        except IOError as e:
            if e.errno == errno.ENOENT:
                raise HttpClientError('Client Error 404')
            else:
                raise

    def __init__(self, *args):
        self.get = self._instance_get
        self.object_id = args[0] if len(args) == 1 else None

    # This is the version of get() that's called when invoked as a
    # class method (i.e. to get everything in a collection)
    @classmethod
    def get(cls, **kwargs):
        return get_example_popit_json(
            'generic_{0}_embed={1}.json'.format(
                cls.collection,
                kwargs.get('embed', 'membership')
            )
        )

    def delete(self):
        raise Exception("Not implemented: you should patch this")

    def put(self, data):
        raise Exception("Not implemented: you should patch this")

    @staticmethod
    def post(data):
        raise Exception("Not implemented: you should patch this")

class FakePersonCollection(FakeCollection):
    collection = 'persons'

class FakeOrganizationCollection(FakeCollection):
    collection = 'organizations'

class FakePostCollection(FakeCollection):
    collection = 'posts'

class FakeMembershipCollection(FakeCollection):
    collection = 'memberships'
<|endoftext|>


<commit_before>from builtins import str
import sadi
import rdflib
import setlr
from datetime import datetime
from depot.manager import DepotManager

from .update_change_service import UpdateChangeService
from nanopub import Nanopublication
from datastore import create_id
import flask
from flask import render_template
from flask import render_template_string
import logging

import sys, traceback

import database

import tempfile

from depot.io.interfaces import StoredFile

from whyis.namespace import *

import hashlib
from uuid import uuid4

from rdflib import compare

pexp = rdflib.Namespace('tag:tw.rpi.edu,2011:expression_rgda1-sha256-')
pmanif = rdflib.Namespace('tag:tw.rpi.edu,2011:manifestation_sha256-')
nfo = rdflib.Namespace('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#')

def rgda1_digest(graph):
    g = rdflib.Graph()
    g += graph
    ig = compare.to_isomorphic(g)
    return ig.graph_digest()

def uuid(graph):
    return int(uuid4())

def sha256(f):
    h = hashlib.sha256()
    for chunk in iter(lambda: f.read(4096), b""):
        h.update(chunk)
    hd = h.hexdigest()
    result = int(hd, 16)
    return result

class FRIRArchiver(UpdateChangeService):
    activity_class = whyis.Archive

    def __init__(self, expression_digest=rgda1_digest, manifestation_digest=sha256):
        self.expression_digest = expression_digest
        self.manifestation_digest = manifestation_digest
        self.nanopub_depot = DepotManager.get('nanopublications')

    def getInputClass(self):
        return np.Nanopublication

    def getOutputClass(self):
        return whyis.ArchivedNanopublication

    def get_query(self):
        return '''
prefix setl: <http://purl.org/twc/vocab/setl/>
select distinct ?resource where {
  ?resource a np:Nanopublication.
  filter not exists {
    ?resource a whyis:FRIRNanopublication.
  }
  filter not exists {
    ?resource a whyis:ArchivedNanopublication.
  }
}'''

    def process_nanopub(self, i, o, new_np):
        assertion = i.graph
        nanopub = Nanopublication(i.graph.store, i.identifier)
        quads = nanopub.serialize(format="nquads")
        i.identifier.split('/')[-1]
        fileid = self.nanopub_depot.create(quads, i.identifier.split('/')[-1]+'.nq', "application/n-quads")
        o.add(rdflib.RDF.type, whyis.ArchivedNanopublication)
        expressions = dict([(part.identifier, self.expression_digest(part))
                            for part in [nanopub.assertion, nanopub.provenance, nanopub.pubinfo]])
        expressions[nanopub.identifier] = sum(expressions.values())
        nanopub_expression_uri = pexp[hex(expressions[nanopub.identifier])[2:]]
        for work, expression in expressions.items():
            exp = pexp[hex(expression)[2:]]
            o.graph.add((work, frbr.realization, exp))
            o.graph.add((work, rdflib.RDF.type, frbr.Work))
            o.graph.add((exp, rdflib.RDF.type, frbr.Expression))

        with self.nanopub_depot.get(fileid) as stored_file:
            manifestation_id = self.manifestation_digest(stored_file)
        manifestation = pmanif[hex(manifestation_id)[2:]]
        o.graph.add((nanopub_expression_uri, frbr.embodiment, manifestation))

        o.graph.add((manifestation, rdflib.RDF.type, pv.File))
        o.graph.add((manifestation, whyis.hasFileID, rdflib.Literal(fileid)))
        o.graph.add((manifestation, dc.created, rdflib.Literal(datetime.utcnow())))
        o.graph.add((manifestation, NS.ov.hasContentType, rdflib.Literal("application/n-quads")))
        o.graph.add((manifestation, rdflib.RDF.type, NS.mediaTypes["application/n-quads"]))
        o.graph.add((NS.mediaTypes["application/n-quads"], rdflib.RDF.type, dc.FileFormat))
<commit_msg>Initializing the nanopub depot before building the app means the nanopub depot will be missing.<commit_after>from builtins import str
import sadi
import rdflib
import setlr
from datetime import datetime
from depot.manager import DepotManager

from .update_change_service import UpdateChangeService
from nanopub import Nanopublication
from datastore import create_id
import flask
from flask import render_template
from flask import render_template_string
import logging

import sys, traceback

import database

import tempfile

from depot.io.interfaces import StoredFile

from whyis.namespace import *

import hashlib
from uuid import uuid4

from rdflib import compare

pexp = rdflib.Namespace('tag:tw.rpi.edu,2011:expression_rgda1-sha256-')
pmanif = rdflib.Namespace('tag:tw.rpi.edu,2011:manifestation_sha256-')
nfo = rdflib.Namespace('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#')

def rgda1_digest(graph):
    g = rdflib.Graph()
    g += graph
    ig = compare.to_isomorphic(g)
    return ig.graph_digest()

def uuid(graph):
    return int(uuid4())

def sha256(f):
    h = hashlib.sha256()
    for chunk in iter(lambda: f.read(4096), b""):
        h.update(chunk)
    hd = h.hexdigest()
    result = int(hd, 16)
    return result

class FRIRArchiver(UpdateChangeService):
    activity_class = whyis.Archive

    def __init__(self, expression_digest=rgda1_digest, manifestation_digest=sha256):
        self.expression_digest = expression_digest
        self.manifestation_digest = manifestation_digest

    def getInputClass(self):
        return np.Nanopublication

    def getOutputClass(self):
        return whyis.ArchivedNanopublication

    def get_query(self):
        return '''
prefix setl: <http://purl.org/twc/vocab/setl/>
select distinct ?resource where {
  ?resource a np:Nanopublication.
  filter not exists {
    ?resource a whyis:FRIRNanopublication.
  }
  filter not exists {
    ?resource a whyis:ArchivedNanopublication.
  }
}'''

    def process_nanopub(self, i, o, new_np):
        assertion = i.graph
        nanopub = Nanopublication(i.graph.store, i.identifier)
        quads = nanopub.serialize(format="nquads")
        i.identifier.split('/')[-1]
        fileid = self.app.nanopub_depot.create(quads, i.identifier.split('/')[-1]+'.nq', "application/n-quads")
        o.add(rdflib.RDF.type, whyis.ArchivedNanopublication)
        expressions = dict([(part.identifier, self.expression_digest(part))
                            for part in [nanopub.assertion, nanopub.provenance, nanopub.pubinfo]])
        expressions[nanopub.identifier] = sum(expressions.values())
        nanopub_expression_uri = pexp[hex(expressions[nanopub.identifier])[2:]]
        for work, expression in expressions.items():
            exp = pexp[hex(expression)[2:]]
            o.graph.add((work, frbr.realization, exp))
            o.graph.add((work, rdflib.RDF.type, frbr.Work))
            o.graph.add((exp, rdflib.RDF.type, frbr.Expression))

        with self.app.nanopub_depot.get(fileid) as stored_file:
            manifestation_id = self.manifestation_digest(stored_file)
        manifestation = pmanif[hex(manifestation_id)[2:]]
        o.graph.add((nanopub_expression_uri, frbr.embodiment, manifestation))

        o.graph.add((manifestation, rdflib.RDF.type, pv.File))
        o.graph.add((manifestation, whyis.hasFileID, rdflib.Literal(fileid)))
        o.graph.add((manifestation, dc.created, rdflib.Literal(datetime.utcnow())))
        o.graph.add((manifestation, NS.ov.hasContentType, rdflib.Literal("application/n-quads")))
        o.graph.add((manifestation, rdflib.RDF.type, NS.mediaTypes["application/n-quads"]))
        o.graph.add((NS.mediaTypes["application/n-quads"], rdflib.RDF.type, dc.FileFormat))
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (C) 2015 Intel Corporation
#
# Released under the MIT license (see COPYING.MIT)
# ./runtest.py -b build_data.json -a tag -f test.manifest

import sys
import os
import time
import unittest
import inspect
from optparse import OptionParser
from oeqa.oetest import oeTest
from oeqa.oetest import oeRuntimeTest
from oeqa.oetest import runTests
from oeqa.runexported import FakeTarget
from oeqa.runexported import MyDataDict
from oeqa.runexported import TestContext
from oeqa.utils.sshcontrol import SSHControl

try:
    import simplejson as json
except ImportError:
    import json

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "oeqa")))

def _check(obj, taglist):
    """ match tag inside obj """
    for tag in taglist:
        tkeyval = tag.split("=")
        tkey = tkeyval[0]
        if hasattr(obj, tkey):
            tval = True if len(tkeyval) == 1 else tkeyval[1]
            if tval == getattr(obj, tkey):
                return True
    return False

def check_tag_method(tcase, taglist):
    """ check if test method has specified tag enabled """
    if not hasattr(tcase, "_testMethodName"):
        return False
    tc_method = getattr(tcase, tcase._testMethodName)
    return _check(tc_method, taglist)

def check_tag_class(tcase, taglist):
    """ check if test class has specified tag enabled """
    if not hasattr(tcase, "_testMethodName"):
        return False
    tc_method = getattr(tcase, tcase._testMethodName)
    tc_class = tc_method.__self__.__class__
    return _check(tc_class, taglist)

def runTests_tag(tc, taglist):
    """ run whole test suite according to tclist"""
    # set the context object passed from the test class
    setattr(oeTest, "tc", tc)
    # set ps command to use
    setattr(oeRuntimeTest, "pscmd", "ps -ef" if oeTest.hasPackage("procps") else "ps")
    # prepare test suite, loader and runner
    suite = unittest.TestSuite()
    testloader = unittest.TestLoader()
    testloader.sortTestMethodsUsing = None
    for tname in tc.testslist:
        tsuite = testloader.loadTestsFromName(tname)
        for ts in tsuite:
            # it is test suite
            if hasattr(ts, "_tests") and ts._tests != []:
                if check_tag_class(ts._tests[0], taglist):
                    print "add suite"
                    suite.addTest(ts)
                else:
                    for x in ts._tests:
                        if check_tag_class(x, taglist) or\
                               check_tag_method(x, taglist):
                            suite.addTest(x)
            # it is test case
            else:
                if check_tag_class(ts, taglist) or\
                       check_tag_method(ts, taglist):
                    suite.addTest(ts)
    print("Test modules  %s" % tc.testslist)
    print("Found %s tests" % suite.countTestCases())
    runner = unittest.TextTestRunner(verbosity=2)
    result = runner.run(suite)
    return result

def main():

    usage = "usage: %prog [options]"
    parser = OptionParser(usage=usage)
    parser.add_option("-t", "--target-ip", dest="ip",
            help="The IP address of the target machine. Use this to \
            overwrite the value determined from TEST_TARGET_IP at build time")
    parser.add_option("-s", "--server-ip", dest="server_ip",
            help="The IP address of this machine. Use this to \
            overwrite the value determined from TEST_SERVER_IP at build time.")
    parser.add_option("-d", "--deploy-dir", dest="deploy_dir",
            help="Full path to the package feeds, that this \
            the contents of what used to be DEPLOY_DIR on the build machine. \
            If not specified it will use the value specified in the json if \
            that directory actually exists or it will error out.")
    parser.add_option("-l", "--log-dir", dest="log_dir",
            help="This sets the path for TEST_LOG_DIR. If not specified \
            the current dir is used. This is used for usually creating a \
            ssh log file and a scp test file.")
    parser.add_option("-f", "--test-manifest", dest="tests_list",
            help="The test list file")
    parser.add_option("-b", "--build-data", dest="build_data",
            help="The build data file.")
    parser.add_option("-a", "--tag", dest="tag",
            help="The tags to filter test case")
    parser.add_option("-r", "--pkgarch", dest="pkgarch",
            help="The package arch")
    parser.add_option("-n", "--nativearch", dest="nativearch",
            help="The native arch")

    (options, args) = parser.parse_args()

    tc = TestContext()

    #inject testcase list
    tclist = []
    if options.tests_list:
        with open(options.tests_list, "r") as f:
            tclist = filter(lambda x: not x.startswith('#'),
                              [n.strip() for n in f.readlines()])
    tc.testslist = tclist
    print tc.testslist

    #get build data from file
    if options.build_data:
        with open(options.build_data, "r") as f:
            loaded = json.load(f)
    else:
        loaded = {
              "d": {"DEPLOY_DIR": "./deploy",
                    "TUNE_PKGARCH": "i586",
                    "TUNE_NATIVE_ARCH": "x86_64"},
              "pkgmanifest": [],
              "filesdir": "oeqa/runtime/files",
              "imagefeatures": []
        }

    #inject build datastore
    d = MyDataDict()
    for key in loaded["d"].keys():
        d[key] = loaded["d"][key]

    if options.log_dir:
        d["TEST_LOG_DIR"] = os.path.abspath(options.log_dir)
    else:
        d["TEST_LOG_DIR"] = os.path.abspath(os.path.dirname(__file__))
    if options.deploy_dir:
        d["DEPLOY_DIR"] = os.path.abspath(options.deploy_dir)
    else:
        if not os.path.isdir(d["DEPLOY_DIR"]):
            raise Exception("The path to DEPLOY_DIR does not exists: %s" % d["DEPLOY_DIR"])
    if options.pkgarch:
        d["TUNE_PKGARCH"] = options.pkgarch
    if options.nativearch:
        d["TUNE_NATIVE_ARCH"] = options.nativearch
    setattr(tc, "d", d)

    #inject build package manifest
    pkgs = [pname.strip() for pname in loaded["pkgmanifest"]]
    setattr(tc, "pkgmanifest", " ".join(pkgs))

    #inject target information
    target = FakeTarget(d)
    target.ip = options.ip if options.ip else "192.168.7.2"
    target.server_ip = options.server_ip if options.server_ip else "192.168.7.1"
    setattr(tc, "target", target)

    #inject others
    for key in loaded.keys():
        if key not in ["testslist", "d", "target", "pkgmanifest"]:
            setattr(tc, key, loaded[key])

    target.exportStart()
    if options.tag:
        taglist = options.tag.split(',')
        runTests_tag(tc, taglist)
    else:
        runTests(tc)

    return 0

if __name__ == "__main__":
    try:
        ret = main()
    except Exception:
        ret = 1
        import traceback
        traceback.print_exc(5)
    sys.exit(ret)
<commit_msg>[test runner] get native arch automatically<commit_after>#!/usr/bin/env python

# Copyright (C) 2015 Intel Corporation
#
# Released under the MIT license (see COPYING.MIT)
# ./runtest.py -b build_data.json -a tag -f test.manifest

import sys
import os
import time
import unittest
import inspect
from optparse import OptionParser
from oeqa.oetest import oeTest
from oeqa.oetest import oeRuntimeTest
from oeqa.oetest import runTests
from oeqa.runexported import FakeTarget
from oeqa.runexported import MyDataDict
from oeqa.runexported import TestContext
from oeqa.utils.sshcontrol import SSHControl

try:
    import simplejson as json
except ImportError:
    import json

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "oeqa")))

def _check(obj, taglist):
    """ match tag inside obj """
    for tag in taglist:
        tkeyval = tag.split("=")
        tkey = tkeyval[0]
        if hasattr(obj, tkey):
            tval = True if len(tkeyval) == 1 else tkeyval[1]
            if tval == getattr(obj, tkey):
                return True
    return False

def check_tag_method(tcase, taglist):
    """ check if test method has specified tag enabled """
    if not hasattr(tcase, "_testMethodName"):
        return False
    tc_method = getattr(tcase, tcase._testMethodName)
    return _check(tc_method, taglist)

def check_tag_class(tcase, taglist):
    """ check if test class has specified tag enabled """
    if not hasattr(tcase, "_testMethodName"):
        return False
    tc_method = getattr(tcase, tcase._testMethodName)
    tc_class = tc_method.__self__.__class__
    return _check(tc_class, taglist)

def runTests_tag(tc, taglist):
    """ run whole test suite according to tclist"""
    # set the context object passed from the test class
    setattr(oeTest, "tc", tc)
    # set ps command to use
    setattr(oeRuntimeTest, "pscmd", "ps -ef" if oeTest.hasPackage("procps") else "ps")
    # prepare test suite, loader and runner
    suite = unittest.TestSuite()
    testloader = unittest.TestLoader()
    testloader.sortTestMethodsUsing = None
    for tname in tc.testslist:
        tsuite = testloader.loadTestsFromName(tname)
        for ts in tsuite:
            # it is test suite
            if hasattr(ts, "_tests") and ts._tests != []:
                if check_tag_class(ts._tests[0], taglist):
                    print "add suite"
                    suite.addTest(ts)
                else:
                    for x in ts._tests:
                        if check_tag_class(x, taglist) or\
                               check_tag_method(x, taglist):
                            suite.addTest(x)
            # it is test case
            else:
                if check_tag_class(ts, taglist) or\
                       check_tag_method(ts, taglist):
                    suite.addTest(ts)
    print("Test modules  %s" % tc.testslist)
    print("Found %s tests" % suite.countTestCases())
    runner = unittest.TextTestRunner(verbosity=2)
    result = runner.run(suite)
    return result

def main():

    usage = "usage: %prog [options]"
    parser = OptionParser(usage=usage)
    parser.add_option("-t", "--target-ip", dest="ip",
            help="The IP address of the target machine. Use this to \
            overwrite the value determined from TEST_TARGET_IP at build time")
    parser.add_option("-s", "--server-ip", dest="server_ip",
            help="The IP address of this machine. Use this to \
            overwrite the value determined from TEST_SERVER_IP at build time.")
    parser.add_option("-d", "--deploy-dir", dest="deploy_dir",
            help="Full path to the package feeds, that this \
            the contents of what used to be DEPLOY_DIR on the build machine. \
            If not specified it will use the value specified in the json if \
            that directory actually exists or it will error out.")
    parser.add_option("-l", "--log-dir", dest="log_dir",
            help="This sets the path for TEST_LOG_DIR. If not specified \
            the current dir is used. This is used for usually creating a \
            ssh log file and a scp test file.")
    parser.add_option("-f", "--test-manifest", dest="tests_list",
            help="The test list file")
    parser.add_option("-b", "--build-data", dest="build_data",
            help="The build data file.")
    parser.add_option("-a", "--tag", dest="tag",
            help="The tags to filter test case")
    parser.add_option("-r", "--pkgarch", dest="pkgarch",
            help="The package arch")
    parser.add_option("-n", "--nativearch", dest="nativearch",
            help="The native arch")

    (options, args) = parser.parse_args()

    tc = TestContext()

    #inject testcase list
    tclist = []
    if options.tests_list:
        with open(options.tests_list, "r") as f:
            tclist = filter(lambda x: not x.startswith('#'),
                              [n.strip() for n in f.readlines()])
    tc.testslist = tclist
    print tc.testslist

    #get build data from file
    if options.build_data:
        with open(options.build_data, "r") as f:
            loaded = json.load(f)
    else:
        loaded = {
              "d": {"DEPLOY_DIR": "./deploy",
                    "TUNE_PKGARCH": "i586"},
              "pkgmanifest": [],
              "filesdir": "oeqa/runtime/files",
              "imagefeatures": []
        }

    #inject build datastore
    d = MyDataDict()
    for key in loaded["d"].keys():
        d[key] = loaded["d"][key]

    if options.log_dir:
        d["TEST_LOG_DIR"] = os.path.abspath(options.log_dir)
    else:
        d["TEST_LOG_DIR"] = os.path.abspath(os.path.dirname(__file__))
    if options.deploy_dir:
        d["DEPLOY_DIR"] = os.path.abspath(options.deploy_dir)
    else:
        if not os.path.isdir(d["DEPLOY_DIR"]):
            raise Exception("The path to DEPLOY_DIR does not exists: %s" % d["DEPLOY_DIR"])
    if options.pkgarch:
        d["TUNE_PKGARCH"] = options.pkgarch
    navarch = os.popen("uname -m").read().strip()
    d["TUNE_NATIVE_ARCH"] = "x86_64" if not navarch else navarch
    if options.nativearch:
        d["TUNE_NATIVE_ARCH"] = options.nativearch
    setattr(tc, "d", d)

    #inject build package manifest
    pkgs = [pname.strip() for pname in loaded["pkgmanifest"]]
    setattr(tc, "pkgmanifest", " ".join(pkgs))

    #inject target information
    target = FakeTarget(d)
    target.ip = options.ip if options.ip else "192.168.7.2"
    target.server_ip = options.server_ip if options.server_ip else "192.168.7.1"
    setattr(tc, "target", target)

    #inject others
    for key in loaded.keys():
        if key not in ["testslist", "d", "target", "pkgmanifest"]:
            setattr(tc, key, loaded[key])

    target.exportStart()
    if options.tag:
        taglist = options.tag.split(',')
        runTests_tag(tc, taglist)
    else:
        runTests(tc)

    return 0

if __name__ == "__main__":
    try:
        ret = main()
    except Exception:
        ret = 1
        import traceback
        traceback.print_exc(5)
    sys.exit(ret)
<|endoftext|>