<commit_before>#!/usr/bin/env python

import datetime
import logging
import os
from urllib.parse import urljoin

from utils import utils, inspector

# http://www.rrb.gov/oig/Default.asp
archive = 1995

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
#

REPORTS_URL = "http://www.rrb.gov/oig/Library.asp"
AUDIT_REPORTS_URL = "http://www.rrb.gov/oig/reports/FY{year}reports.asp"

def run(options):
  year_range = inspector.year_range(options, archive)

  doc = utils.beautifulsoup_from_url(REPORTS_URL)

  # Pull the semiannual reports
  semiannul_results = doc.select("#AnnualManagementReports select")[0]
  for result in semiannul_results.select("option"):
    report = semiannual_report_from(result, year_range)
    if report:
      inspector.save_report(report)

  # Pull the special reports
  special_report_table = doc.find("table", attrs={"bordercolor": "#808080"})
  for index, result in enumerate(special_report_table.select("tr")):
    if not index:
      # Skip the header row
      continue
    report = report_from(result, REPORTS_URL, report_type='other', year_range=year_range)
    if report:
      inspector.save_report(report)

  # Pull the audit reports
  for year in year_range:
    if year < 2001:  # The oldest fiscal year page available
      continue
    year_url = AUDIT_REPORTS_URL.format(year=year)
    doc = utils.beautifulsoup_from_url(year_url)
    results = doc.select("#main table tr")
    if not results:
      raise inspector.NoReportsFoundError("Railroad Retirement Board (%d)" % year)
    for index, result in enumerate(results):
      if not index:
        # Skip the header row
        continue
      report = report_from(result, year_url, report_type='audit', year_range=year_range)
      if report:
        inspector.save_report(report)

saved_report_urls = set()

def report_from(result, landing_url, report_type, year_range):
  title = " ".join(result.select("td")[0].text.strip().split())

  published_on_text = result.select("td")[1].text.strip()
  try:
    published_on = datetime.datetime.strptime(published_on_text, '%m/%d/%y')
  except ValueError:
    published_on = datetime.datetime.strptime(published_on_text, '%m/%Y')

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % title)
    return

  link = result.find("a")
  if link:
    unreleased = False
    report_url = urljoin(landing_url, link.get('href'))
    report_filename = report_url.split("/")[-1]
    report_id, _ = os.path.splitext(report_filename)

    # Deduplicate using report_url, "Reinvention 2001" is posted in both the
    # special reports table and the FY2001 audit reports
    if report_url in saved_report_urls:
      return
    saved_report_urls.add(report_url)

  else:
    unreleased = True
    report_url = None
    report_id = "{}-{}".format(published_on_text.replace("/", "-"), "-".join(title.split()))[:50]

  report = {
    'inspector': 'rrb',
    'inspector_url': "http://www.rrb.gov/oig/",
    'agency': 'rrb',
    'agency_name': "Railroad Retirement Board",
    'type': report_type,
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  if unreleased:
    report['unreleased'] = unreleased
    report['landing_url'] = landing_url
  return report

def semiannual_report_from(result, year_range):
  relative_report_url = result.get('value')
  if not relative_report_url:
    # Skip the header
    return

  # The links go up an extra level, not sure why.
  relative_report_url = relative_report_url.replace("../", "", 1)
  report_url = urljoin(REPORTS_URL, relative_report_url)
  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)

  published_on_text = result.text.split(",")[0].strip()
  published_on = datetime.datetime.strptime(published_on_text, '%b %Y')
  title = "Semiannual Report - {}".format(published_on_text)

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'rrb',
    'inspector_url': "http://www.rrb.gov/oig/",
    'agency': 'rrb',
    'agency_name': "Railroad Retirement Board",
    'type': 'semiannual_report',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

utils.run(run) if (__name__ == "__main__") else None
<commit_msg>[rrb] Add date<commit_after>#!/usr/bin/env python

import datetime
import logging
import os
from urllib.parse import urljoin

from utils import utils, inspector

# http://www.rrb.gov/oig/Default.asp
archive = 1995

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
#

REPORTS_URL = "http://www.rrb.gov/oig/Library.asp"
AUDIT_REPORTS_URL = "http://www.rrb.gov/oig/reports/FY{year}reports.asp"

def run(options):
  year_range = inspector.year_range(options, archive)

  doc = utils.beautifulsoup_from_url(REPORTS_URL)

  # Pull the semiannual reports
  semiannul_results = doc.select("#AnnualManagementReports select")[0]
  for result in semiannul_results.select("option"):
    report = semiannual_report_from(result, year_range)
    if report:
      inspector.save_report(report)

  # Pull the special reports
  special_report_table = doc.find("table", attrs={"bordercolor": "#808080"})
  for index, result in enumerate(special_report_table.select("tr")):
    if not index:
      # Skip the header row
      continue
    report = report_from(result, REPORTS_URL, report_type='other', year_range=year_range)
    if report:
      inspector.save_report(report)

  # Pull the audit reports
  for year in year_range:
    if year < 2001:  # The oldest fiscal year page available
      continue
    year_url = AUDIT_REPORTS_URL.format(year=year)
    doc = utils.beautifulsoup_from_url(year_url)
    results = doc.select("#main table tr")
    if not results:
      raise inspector.NoReportsFoundError("Railroad Retirement Board (%d)" % year)
    for index, result in enumerate(results):
      if not index:
        # Skip the header row
        continue
      report = report_from(result, year_url, report_type='audit', year_range=year_range)
      if report:
        inspector.save_report(report)

saved_report_urls = set()

def report_from(result, landing_url, report_type, year_range):
  title = " ".join(result.select("td")[0].text.strip().split())

  published_on_text = result.select("td")[1].text.strip()
  try:
    published_on = datetime.datetime.strptime(published_on_text, '%m/%d/%y')
  except ValueError:
    published_on = datetime.datetime.strptime(published_on_text, '%m/%Y')

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % title)
    return

  link = result.find("a")
  if link:
    unreleased = False
    report_url = urljoin(landing_url, link.get('href'))
    report_filename = report_url.split("/")[-1]
    report_id, _ = os.path.splitext(report_filename)

    # Deduplicate using report_url, "Reinvention 2001" is posted in both the
    # special reports table and the FY2001 audit reports
    if report_url in saved_report_urls:
      return
    saved_report_urls.add(report_url)

  else:
    unreleased = True
    report_url = None
    report_id = "{}-{}".format(published_on_text.replace("/", "-"), "-".join(title.split()))[:50]

  report = {
    'inspector': 'rrb',
    'inspector_url': "http://www.rrb.gov/oig/",
    'agency': 'rrb',
    'agency_name': "Railroad Retirement Board",
    'type': report_type,
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  if unreleased:
    report['unreleased'] = unreleased
    report['landing_url'] = landing_url
  return report

def semiannual_report_from(result, year_range):
  relative_report_url = result.get('value')
  if not relative_report_url:
    # Skip the header
    return

  # The links go up an extra level, not sure why.
  relative_report_url = relative_report_url.replace("../", "", 1)
  report_url = urljoin(REPORTS_URL, relative_report_url)
  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)

  published_on_text = result.text.split(",")[0].strip()
  if published_on_text == "Open Audit Recommendations":
    published_on = datetime.datetime(2017, 3, 31)
  else:
    published_on = datetime.datetime.strptime(published_on_text, '%b %Y')
  title = "Semiannual Report - {}".format(published_on_text)

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'rrb',
    'inspector_url': "http://www.rrb.gov/oig/",
    'agency': 'rrb',
    'agency_name': "Railroad Retirement Board",
    'type': 'semiannual_report',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

utils.run(run) if (__name__ == "__main__") else None
<|endoftext|>


<commit_before>from django.test import TestCase
from django.contrib.admin.sites import AdminSite
from billjobs.admin import UserAdmin

class EmailExportTestCase(TestCase):
    """ Tests for email account export """

    def test_method_is_avaible(self):
        """ Test admin can select the action in dropdown list """
        self.assertTrue(hasattr(UserAdmin, 'export_email'))

<commit_msg>Test modelAdmin actions is present<commit_after>from django.test import TestCase
from django.contrib.admin.sites import AdminSite
from billjobs.admin import UserAdmin

class EmailExportTestCase(TestCase):
    """ Tests for email account export """

    def test_method_is_avaible(self):
        """ Test admin can select the action in dropdown list """
        self.assertTrue(hasattr(UserAdmin, 'export_email'))

    def test_method_is_model_admin_action(self):
        """ Test method is an custom action for user admin """
        self.assertTrue('export_email' in UserAdmin.actions)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import csv
import random

# Checkout the README.md file for some notes on the script.

if __name__ == "__main__":

    player_dict = {}
    teams = {
        'Sharks': { 'players': [] },
        'Dragons': { 'players': [] },
        'Raptors': { 'players': [] }
    }

    def load_players():
        with open('soccer_players.csv', 'r') as csv_file:
            csv_data = csv.DictReader(csv_file)
            for row in csv_data:
                player_dict[row['Name']] = {
                    'height': row['Height (inches)'],
                    'experience': row['Soccer Experience'],
                    'guardian': row['Guardian Name(s)']
                }

    def players_who_have_experience(do_they_have_it):
        return_list = []
        for player_name, player_data in player_dict.items():
            if player_data['experience'] == do_they_have_it:
                return_list.append(player_name)
        return return_list

    def put_players_on_team_randomly(player_list):
        while len(player_list):
            for team_key, team_data in teams.items():
                if len(player_list):
                    team_data['players'].append(player_list.pop(random.randint(0,len(player_list) - 1)))

    def output_team_file():
        with open('output/teams.txt', 'w') as teams_file:
            for team_key, team_data in teams.items():
                teams_file.write(team_key + '\n')
                for player in team_data['players']:
                    teams_file.write('{}, {}, {}'.format(player, player_dict[player]['experience'], player_dict[player]['guardian']) + '\n')
                teams_file.write('\n')

    load_players()
    put_players_on_team_randomly(players_who_have_experience('YES'))
    put_players_on_team_randomly(players_who_have_experience('NO'))
    output_team_file()

<commit_msg>Renamed `player_dict` to `players`.<commit_after>#!/usr/bin/env python

import csv
import random

# Checkout the README.md file for some notes on the script.

if __name__ == "__main__":

    players = {}
    teams = {
        'Sharks': { 'players': [] },
        'Dragons': { 'players': [] },
        'Raptors': { 'players': [] }
    }

    def load_players():
        with open('soccer_players.csv', 'r') as csv_file:
            csv_data = csv.DictReader(csv_file)
            for row in csv_data:
                players[row['Name']] = {
                    'height': row['Height (inches)'],
                    'experience': row['Soccer Experience'],
                    'guardian': row['Guardian Name(s)']
                }

    def players_who_have_experience(do_they_have_it):
        return_list = []
        for player_name, player_data in players.items():
            if player_data['experience'] == do_they_have_it:
                return_list.append(player_name)
        return return_list

    def put_players_on_team_randomly(player_list):
        while len(player_list):
            for team_key, team_data in teams.items():
                if len(player_list):
                    team_data['players'].append(player_list.pop(random.randint(0,len(player_list) - 1)))

    def output_team_file():
        with open('output/teams.txt', 'w') as teams_file:
            for team_key, team_data in teams.items():
                teams_file.write(team_key + '\n')
                for player in team_data['players']:
                    teams_file.write('{}, {}, {}'.format(player, players[player]['experience'], players[player]['guardian']) + '\n')
                teams_file.write('\n')

    load_players()
    put_players_on_team_randomly(players_who_have_experience('YES'))
    put_players_on_team_randomly(players_who_have_experience('NO'))
    output_team_file()

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from datetime import datetime
from pytz import timezone as tz
from cached_property import cached_property

from . import log
from . import api

class KanbanError(Exception):
    """ Error thrown when performing a non-valid operation """

class Converter(dict):
    _attrs_, _items_ = {}, {}

    def __init__(self, data, board):
        super().__init__(**data)
        self.board = board

    def __repr__(self):
        return '<{0.__class__.__name__} {0.id}>'.format(self)

    def __hash__(self):
        return hash(repr(self))

    def __getitem__(self, key):
        if key in self._attrs_:
            value = super().__getitem__(key)
            return getattr(self, '_' + self._attrs_[key] + '_')(value)
        elif key in self._items_:
            return self.board[self._items_[key]].get(int(self[key + 'Id']))
        else:
            return super().__getitem__(key)

    def __getattr__(self, name):
        key = name.title().replace('_', '')
        try:
            return self[key]
        except KeyError:
            raise AttributeError(name)

    def _list_(self, value):
        return [val for val in value.strip(',').split(',') if val]

    def _date_(self, value):
        return datetime.strptime(value, '%d/%m/%Y').date() if value else None

    def _datetime_(self, value):
        if value:
            time = datetime.strptime(value, '%d/%m/%Y %I:%M:%S %p')
            if self.board.timezone:
                return self.board.timezone.localize(time)
            return time
        return None

    @property
    def raw_data(self):
        return {key: self[key] for key in self.keys()}

class User(Converter):
    def __str__(self):
        return self.user_name

class CardType(Converter):
    def __str__(self):
        return self['Name']

class ClassOfService(Converter):
    def __str__(self):
        return self['Title']

class Event(Converter):
    _attrs_ = {'DateTime': 'datetime'}
    _items_ = {'User': 'Users', 'ToLane': 'Lanes', 'FromLane': 'Lanes'}

    def __repr__(self):
        return '<{0.__class__.__name__}>'.format(self)

    def _datetime_(self, value):
        time = datetime.strptime(value, '%d/%m/%Y at %I:%M:%S %p')
        if self.board.timezone:
            return self.board.timezone.localize(time)
        return time

class Card(Converter):
    _attrs_ = {'LastMove': 'datetime', 'LastActivity': 'datetime',
               'CreateDate': 'date', 'DateArchived': 'date', 'DueDate': 'date',
               'LastComment': 'date', 'StartDate': 'date', 'Tags': 'list',
               'ActualStartDate': 'datetime', 'ActualFinishDate': 'datetime'}
    _items_ = {'Type': 'CardTypes', 'AssignedUser': 'Users',
               'ClassOfService': 'ClassesOfService'}

    def __init__(self, data, lane, board):
        super().__init__(data, board)
        self.lane = lane
        self.board.cards[self.id] = self

    def __str__(self):
        return str(self.get('ExternalCardID', self.id) or self.id)

    @cached_property
    def history(self):
        events = api.get("/Card/History/{0.board.id}/{0.id}".format(self))
        return [Event(event, self.board) for event in reversed(events)]

    @cached_property
    def comments(self):
        return api.get("/Card/GetComments/{0.board.id}/{0.id}".format(self))

class Lane(Converter):
    _items_ = {'ParentLane': 'Lanes'}

    def __init__(self, data, board):
        super().__init__(data, board)
        self.cards = [Card(card_dict, self, board) for card_dict
                      in data['Cards'] if card_dict['TypeId']]

    def __str__(self):
        return self.path

    @property
    def path(self):
        titles = [self.title] + [lane.title for lane in self.ascendants]
        return '::'.join(reversed(titles))

    @property
    def top_lane(self):
        return ([self] + self.ascendants)[-1]

    @property
    def children(self):
        return [self.board.lanes[lane_id] for lane_id in self.child_lane_ids]

    @property
    def ascendants(self):
        """ Returns a list of all parent lanes sorted in ascending order """
        lanes = []
        lane = self.parent_lane
        while lane:
            lanes.append(lane)
            lane = lane.parent_lane
        return lanes

    @property
    def descendants(self):
        """ Returns a list of all child lanes sorted in descending order """
        def sublanes(lane, array):
            for child in lane.children:
                array.append(child)
                sublanes(child, array)
            return array

        return sublanes(self, [])

class Board(Converter):
    _attrs_ = {'AvailableTags': 'list'}
    _items_ = {'BacklogTopLevelLane': 'Lanes', 'ArchiveTopLevelLane': 'Lanes'}

    def __init__(self, board, timezone=None):
        if isinstance(board, int):
            log.debug('Downloading board {}'.format(board))
            board = api.get('/Boards/{}'.format(board))
        super().__init__(board, self)
        self.cards = {}
        self.timezone = tz(timezone) if timezone else None
        self.users = self._populate_('BoardUsers', User)
        self._populate_('CardTypes', CardType)
        self._populate_('ClassesOfService', ClassOfService)
        self._populate_('Lanes', Lane)
        self.lanes.update(self._populate_('Backlog', Lane))
        self.lanes.update(self._populate_('Archive', Lane))

    def __str__(self):
        return self['Title']

    def _populate_(self, key, element):
        items = {}
        for item in self[key]:
            instance = element(item, self)
            items[instance.id] = instance
        self[key] = items
        return items

    @property
    def top_level_lanes(self):
        return [self.lanes[lane_id] for lane_id in self.top_level_lane_ids]

    @property
    def archive_lanes(self):
        if self.archive_top_level_lane_id not in self.lanes:
            raise KanbanError("Archive lanes not available")
        archive_lane = self.archive_top_level_lane
        return [archive_lane] + archive_lane.descendants

    @property
    def backlog_lanes(self):
        if self.backlog_top_level_lane_id not in self.lanes:
            raise KanbanError("Backlog lanes not available")
        backlog_lane = self.backlog_top_level_lane
        return [backlog_lane] + backlog_lane.descendants

    @property
    def sorted_lanes(self):
        lanes = []
        lanes += self.backlog_lanes
        for lane in self.top_level_lanes:
            lanes += [lane] + lane.descendants
        lanes += self.archive_lanes
        return lanes

    def get_archive(self):
        archive = api.get('/Board/{0.id}/Archive'.format(self))[0]
        main_archive_lane = Lane(archive['Lane'], self)
        self.lanes[main_archive_lane.id] = main_archive_lane
        for lane_dict in archive['ChildLanes']:
            lane = Lane(lane_dict['Lane'], self)
            self.lanes[lane.id] = lane

    def get_recent_archive(self):
        archive = api.get('/Board/{0.id}/ArchiveCards'.format(self))
        return [Card(card, self.lanes.get(card['LaneId']), self)
                for card in archive if card['TypeId']]

    def get_card(self, card_id):
        url = '/Board/{}/GetCard/{}'
        card_dict = api.get(url.format(str(self.id), card_id))
        lane = self.lanes.get(card_dict['LaneId'])  # TODO: replace card in lane
        card = Card(card_dict, lane, self)
        return card
<commit_msg>Added decorators to internal methods<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from datetime import datetime
from pytz import timezone as tz
from cached_property import cached_property

from . import log
from . import api

class KanbanError(Exception):
    """ Error thrown when performing a non-valid operation """

class Converter(dict):
    _attrs_, _items_ = {}, {}

    def __init__(self, data, board):
        super().__init__(**data)
        self.board = board

    def __repr__(self):
        return '<{0.__class__.__name__} {0.id}>'.format(self)

    def __hash__(self):
        return hash(repr(self))

    def __getitem__(self, key):
        if key in self._attrs_:
            value = super().__getitem__(key)
            return getattr(self, '_' + self._attrs_[key] + '_')(value)
        elif key in self._items_:
            return self.board[self._items_[key]].get(int(self[key + 'Id']))
        else:
            return super().__getitem__(key)

    def __getattr__(self, name):
        key = name.title().replace('_', '')
        try:
            return self[key]
        except KeyError:
            raise AttributeError(name)

    @staticmethod
    def _list_(self, value):
        return [val for val in value.strip(',').split(',') if val]

    @staticmethod
    def _date_(self, value):
        return datetime.strptime(value, '%d/%m/%Y').date() if value else None

    def _datetime_(self, value):
        if value:
            time = datetime.strptime(value, '%d/%m/%Y %I:%M:%S %p')
            if self.board.timezone:
                return self.board.timezone.localize(time)
            return time
        return None

    @property
    def raw_data(self):
        return {key: self[key] for key in self.keys()}

class User(Converter):
    def __str__(self):
        return self.user_name

class CardType(Converter):
    def __str__(self):
        return self['Name']

class ClassOfService(Converter):
    def __str__(self):
        return self['Title']

class Event(Converter):
    _attrs_ = {'DateTime': 'datetime'}
    _items_ = {'User': 'Users', 'ToLane': 'Lanes', 'FromLane': 'Lanes'}

    def __repr__(self):
        return '<{0.__class__.__name__}>'.format(self)

    def _datetime_(self, value):
        time = datetime.strptime(value, '%d/%m/%Y at %I:%M:%S %p')
        if self.board.timezone:
            return self.board.timezone.localize(time)
        return time

class Card(Converter):
    _attrs_ = {'LastMove': 'datetime', 'LastActivity': 'datetime',
               'CreateDate': 'date', 'DateArchived': 'date', 'DueDate': 'date',
               'LastComment': 'date', 'StartDate': 'date', 'Tags': 'list',
               'ActualStartDate': 'datetime', 'ActualFinishDate': 'datetime'}
    _items_ = {'Type': 'CardTypes', 'AssignedUser': 'Users',
               'ClassOfService': 'ClassesOfService'}

    def __init__(self, data, lane, board):
        super().__init__(data, board)
        self.lane = lane
        self.board.cards[self.id] = self

    def __str__(self):
        return str(self.get('ExternalCardID', self.id) or self.id)

    @cached_property
    def history(self):
        events = api.get("/Card/History/{0.board.id}/{0.id}".format(self))
        return [Event(event, self.board) for event in reversed(events)]

    @cached_property
    def comments(self):
        return api.get("/Card/GetComments/{0.board.id}/{0.id}".format(self))

class Lane(Converter):
    _items_ = {'ParentLane': 'Lanes'}

    def __init__(self, data, board):
        super().__init__(data, board)
        self.cards = [Card(card_dict, self, board) for card_dict
                      in data['Cards'] if card_dict['TypeId']]

    def __str__(self):
        return self.path

    @property
    def path(self):
        titles = [self.title] + [lane.title for lane in self.ascendants]
        return '::'.join(reversed(titles))

    @property
    def top_lane(self):
        return ([self] + self.ascendants)[-1]

    @property
    def children(self):
        return [self.board.lanes[lane_id] for lane_id in self.child_lane_ids]

    @property
    def ascendants(self):
        """ Returns a list of all parent lanes sorted in ascending order """
        lanes = []
        lane = self.parent_lane
        while lane:
            lanes.append(lane)
            lane = lane.parent_lane
        return lanes

    @property
    def descendants(self):
        """ Returns a list of all child lanes sorted in descending order """
        def sublanes(lane, array):
            for child in lane.children:
                array.append(child)
                sublanes(child, array)
            return array

        return sublanes(self, [])

class Board(Converter):
    _attrs_ = {'AvailableTags': 'list'}
    _items_ = {'BacklogTopLevelLane': 'Lanes', 'ArchiveTopLevelLane': 'Lanes'}

    def __init__(self, board, timezone=None):
        if isinstance(board, int):
            log.debug('Downloading board {}'.format(board))
            board = api.get('/Boards/{}'.format(board))
        super().__init__(board, self)
        self.cards = {}
        self.timezone = tz(timezone) if timezone else None
        self.users = self._populate_('BoardUsers', User)
        self._populate_('CardTypes', CardType)
        self._populate_('ClassesOfService', ClassOfService)
        self._populate_('Lanes', Lane)
        self.lanes.update(self._populate_('Backlog', Lane))
        self.lanes.update(self._populate_('Archive', Lane))

    def __str__(self):
        return self['Title']

    def _populate_(self, key, element):
        items = {}
        for item in self[key]:
            instance = element(item, self)
            items[instance.id] = instance
        self[key] = items
        return items

    @property
    def top_level_lanes(self):
        return [self.lanes[lane_id] for lane_id in self.top_level_lane_ids]

    @property
    def archive_lanes(self):
        if self.archive_top_level_lane_id not in self.lanes:
            raise KanbanError("Archive lanes not available")
        archive_lane = self.archive_top_level_lane
        return [archive_lane] + archive_lane.descendants

    @property
    def backlog_lanes(self):
        if self.backlog_top_level_lane_id not in self.lanes:
            raise KanbanError("Backlog lanes not available")
        backlog_lane = self.backlog_top_level_lane
        return [backlog_lane] + backlog_lane.descendants

    @property
    def sorted_lanes(self):
        lanes = []
        lanes += self.backlog_lanes
        for lane in self.top_level_lanes:
            lanes += [lane] + lane.descendants
        lanes += self.archive_lanes
        return lanes

    def get_archive(self):
        archive = api.get('/Board/{0.id}/Archive'.format(self))[0]
        main_archive_lane = Lane(archive['Lane'], self)
        self.lanes[main_archive_lane.id] = main_archive_lane
        for lane_dict in archive['ChildLanes']:
            lane = Lane(lane_dict['Lane'], self)
            self.lanes[lane.id] = lane

    def get_recent_archive(self):
        archive = api.get('/Board/{0.id}/ArchiveCards'.format(self))
        return [Card(card, self.lanes.get(card['LaneId']), self)
                for card in archive if card['TypeId']]

    def get_card(self, card_id):
        url = '/Board/{}/GetCard/{}'
        card_dict = api.get(url.format(str(self.id), card_id))
        lane = self.lanes.get(card_dict['LaneId'])  # TODO: replace card in lane
        card = Card(card_dict, lane, self)
        return card
<|endoftext|>


<commit_before>from .base import LegistarScraper
from lxml.etree import tostring

class LegistarBillScraper(LegistarScraper):
    def legislation(self, search_text='', created_after=None,
                    created_before=None, num_pages=None) :
        for page in self.searchLegislation(search_text, created_after,
                                           created_before, num_pages) :
            for legislation_summary in self.parseSearchResults(page) :
                yield legislation_summary

    def searchLegislation(self, search_text='', created_after=None,
                          created_before=None, num_pages = None):
        """
        Submit a search query on the legislation search page, and return a list
        of summary results.
        """

        page = self.lxmlize(self.LEGISLATION_URL)

        page = self._advancedSearch(page)

        payload = {}

        # Enter the search parameters TODO: Each of the possible form
        # fields should be represented as keyword arguments to this
        # function. The default query string should be for the the
        # default 'Legislative text' field.
        payload['ctl00$ContentPlaceHolder1$txtText'] = search_text

        if created_after and created_before :
            payload.update(dateWithin(created_after, created_before))

        elif created_before :
            payload.update(dateBound(created_before))
            payload['ctl00$ContentPlaceHolder1$radFileCreated'] = '<'

        elif created_after :
            payload.update(dateBound(created_after))
            payload['ctl00$ContentPlaceHolder1$radFileCreated'] = '>'

        # Return up to one million search results
        payload['ctl00_ContentPlaceHolder1_lstMax_ClientState'] = '{"value":"1000000"}'
        payload['ctl00$ContentPlaceHolder1$btnSearch'] = 'Search Legislation'
        payload['ctl00$ContentPlaceHolder1$lstYearsAdvanced'] = 'All Years'

        payload.update(self.sessionSecrets(page))

        return self.pages(self.LEGISLATION_URL, payload)

    def parseSearchResults(self, page) :
        """Take a page of search results and return a sequence of data
        of tuples about the legislation, of the form

        ('Document ID', 'Document URL', 'Type', 'Status', 'Introduction Date'
        'Passed Date', 'Main Sponsor', 'Title')
        """
        table = page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridMain_ctl00']")[0]
        for legislation, headers, row in self.parseDataTable(table):
            # Do legislation search-specific stuff
            # ------------------------------------
            # First column should be the ID of the record.
            id_key = headers[0]
            try:
                legislation_id = legislation[id_key]['label']
            except TypeError:
                continue
            legislation_url = legislation[id_key]['url'].split(self.BASE_URL)[-1]
            legislation[id_key] = legislation_id
            legislation['url'] = self.BASE_URL + legislation_url.split('&Options')[0]

            yield legislation

    def _advancedSearch(self, page) :
        search_switcher = page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0]

        if 'Simple search' in search_switcher.value :
            return page
        else :
            payload = {}
            payload[search_switcher.name] = search_switcher.value
            payload.update(self.sessionSecrets(page))

            page = self.lxmlize(self.LEGISLATION_URL, payload)

            print(page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0].attrib)
            if 'Simple search' not in page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0].value :
                raise ValueError('Not on the advanced search page')

            return page

    def details(self, detail_url, div_id) :
        detail_page = self.lxmlize(detail_url)

        detail_div = detail_page.xpath(".//div[@id='%s']" % div_id)[0]

        return self.parseDetails(detail_div)

    def legDetails(self, detail_url) :
        div_id = 'ctl00_ContentPlaceHolder1_pageDetails'
        return self.details(detail_url, div_id)

    def actionDetails(self, detail_url) :
        div_id = 'ctl00_ContentPlaceHolder1_pageTop1'
        return self.details(detail_url, div_id)

    def history(self, detail_url) :
        detail_page = self.lxmlize(detail_url)

        history_table = detail_page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridLegislation_ctl00']")[0]

        history = self.parseDataTable(history_table)

        for action, _, _ in history :
            yield action

    def extractVotes(self, action_detail_url) :
        action_detail_page = self.lxmlize(action_detail_url)
        try:
            vote_table = action_detail_page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridVote_ctl00']")[0]
        except IndexError:
            self.warning("No votes found in table")
            return None, []
        votes = list(self.parseDataTable(vote_table))
        vote_list = []
        for vote, _, _ in votes :
            raw_option = vote['Vote'].lower()
            vote_list.append((self.VOTE_OPTIONS.get(raw_option, raw_option),
                              vote['Person Name']['label']))

        action_detail_div = action_detail_page.xpath(".//div[@id='ctl00_ContentPlaceHolder1_pageTop1']")[0]
        action_details = self.parseDetails(action_detail_div)
        result = action_details['Result'].lower()

        return result, vote_list

def dateWithin(created_after, created_before) :
    payload = dateBound(created_after)

    payload['ctl00$ContentPlaceHolder1$txtFileCreated2'] =\
        '{d.year}-{d.month:02}-{d.day:02}'.format(d=created_before)
    payload['ctl00$ContentPlaceHolder1$txtFileCreated2$dateInput'] =\
        '{d.month}/{d.day}/{d.year}'.format(d=created_before)

    payload['ctl00_ContentPlaceHolder1_txtFileCreated2_dateInput_ClientState'] =\
        '{{"enabled":true, "emptyMessage":"","validationText":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","valueAsString":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","minDateStr":"1980-01-01-00-00-00","maxDateStr":"2099-12-31-00-00-00", "lastSetTextBoxValue":"{d.month}/{d.day}/{d.year}"}}'.format(d=created_before)

    payload['ctl00$ContentPlaceHolder1$radFileCreated'] = 'between'

    return payload

def dateBound(creation_date) :
    payload = {}

    payload['ctl00$ContentPlaceHolder1$txtFileCreated1'] =\
        '{d.year}-{d.month:02}-{d.day:02}'.format(d=creation_date)
    payload['ctl00$ContentPlaceHolder1$txtFileCreated1$dateInput'] =\
        '{d.month}/{d.day}/{d.year}'.format(d=creation_date)

    payload['ctl00_ContentPlaceHolder1_txtFileCreated1_dateInput_ClientState'] =\
        '{{"enabled":true, "emptyMessage":"","validationText":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","valueAsString":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","minDateStr":"1980-01-01-00-00-00","maxDateStr":"2099-12-31-00-00-00", "lastSetTextBoxValue":"{d.month}/{d.day}/{d.year}"}}'.format(d=creation_date)

    return payload
<commit_msg>get text of bills<commit_after>from .base import LegistarScraper
from lxml.etree import tostring

class LegistarBillScraper(LegistarScraper):
    def legislation(self, search_text='', created_after=None,
                    created_before=None, num_pages=None) :
        for page in self.searchLegislation(search_text, created_after,
                                           created_before, num_pages) :
            for legislation_summary in self.parseSearchResults(page) :
                yield legislation_summary

    def searchLegislation(self, search_text='', created_after=None,
                          created_before=None, num_pages = None):
        """
        Submit a search query on the legislation search page, and return a list
        of summary results.
        """

        page = self.lxmlize(self.LEGISLATION_URL)

        page = self._advancedSearch(page)

        payload = {}

        # Enter the search parameters TODO: Each of the possible form
        # fields should be represented as keyword arguments to this
        # function. The default query string should be for the the
        # default 'Legislative text' field.
        payload['ctl00$ContentPlaceHolder1$txtText'] = search_text

        if created_after and created_before :
            payload.update(dateWithin(created_after, created_before))

        elif created_before :
            payload.update(dateBound(created_before))
            payload['ctl00$ContentPlaceHolder1$radFileCreated'] = '<'

        elif created_after :
            payload.update(dateBound(created_after))
            payload['ctl00$ContentPlaceHolder1$radFileCreated'] = '>'

        # Return up to one million search results
        payload['ctl00_ContentPlaceHolder1_lstMax_ClientState'] = '{"value":"1000000"}'
        payload['ctl00$ContentPlaceHolder1$btnSearch'] = 'Search Legislation'
        payload['ctl00$ContentPlaceHolder1$lstYearsAdvanced'] = 'All Years'

        payload.update(self.sessionSecrets(page))

        return self.pages(self.LEGISLATION_URL, payload)

    def parseSearchResults(self, page) :
        """Take a page of search results and return a sequence of data
        of tuples about the legislation, of the form

        ('Document ID', 'Document URL', 'Type', 'Status', 'Introduction Date'
        'Passed Date', 'Main Sponsor', 'Title')
        """
        table = page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridMain_ctl00']")[0]
        for legislation, headers, row in self.parseDataTable(table):
            # Do legislation search-specific stuff
            # ------------------------------------
            # First column should be the ID of the record.
            id_key = headers[0]
            try:
                legislation_id = legislation[id_key]['label']
            except TypeError:
                continue
            legislation_url = legislation[id_key]['url'].split(self.BASE_URL)[-1]
            legislation[id_key] = legislation_id
            legislation['url'] = self.BASE_URL + legislation_url.split('&Options')[0]

            yield legislation

    def _advancedSearch(self, page) :
        search_switcher = page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0]

        if 'Simple search' in search_switcher.value :
            return page
        else :
            payload = {}
            payload[search_switcher.name] = search_switcher.value
            payload.update(self.sessionSecrets(page))

            page = self.lxmlize(self.LEGISLATION_URL, payload)

            print(page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0].attrib)
            if 'Simple search' not in page.xpath("//input[@id='ctl00_ContentPlaceHolder1_btnSwitch']")[0].value :
                raise ValueError('Not on the advanced search page')

            return page

    def details(self, detail_url, div_id) :
        detail_page = self.lxmlize(detail_url)

        detail_div = detail_page.xpath(".//div[@id='%s']" % div_id)[0]

        return self.parseDetails(detail_div)

    def legDetails(self, detail_url) :
        div_id = 'ctl00_ContentPlaceHolder1_pageDetails'
        return self.details(detail_url, div_id)

    def actionDetails(self, detail_url) :
        div_id = 'ctl00_ContentPlaceHolder1_pageTop1'
        return self.details(detail_url, div_id)

    def history(self, detail_url) :
        detail_page = self.lxmlize(detail_url)

        history_table = detail_page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridLegislation_ctl00']")[0]

        history = self.parseDataTable(history_table)

        for action, _, _ in history :
            yield action

    def text(self, detail_url) :
        detail_page = self.lxmlize(detail_url)

        text_div = detail_page.xpath("//div[@id='ctl00_ContentPlaceHolder1_divText']/div/div")

        if len(text_div) :
            return tostring(text_div[0], pretty_print=True).decode()
        else :
            return None

    def extractVotes(self, action_detail_url) :
        action_detail_page = self.lxmlize(action_detail_url)
        try:
            vote_table = action_detail_page.xpath("//table[@id='ctl00_ContentPlaceHolder1_gridVote_ctl00']")[0]
        except IndexError:
            self.warning("No votes found in table")
            return None, []
        votes = list(self.parseDataTable(vote_table))
        vote_list = []
        for vote, _, _ in votes :
            raw_option = vote['Vote'].lower()
            vote_list.append((self.VOTE_OPTIONS.get(raw_option, raw_option),
                              vote['Person Name']['label']))

        action_detail_div = action_detail_page.xpath(".//div[@id='ctl00_ContentPlaceHolder1_pageTop1']")[0]
        action_details = self.parseDetails(action_detail_div)
        result = action_details['Result'].lower()

        return result, vote_list

def dateWithin(created_after, created_before) :
    payload = dateBound(created_after)

    payload['ctl00$ContentPlaceHolder1$txtFileCreated2'] =\
        '{d.year}-{d.month:02}-{d.day:02}'.format(d=created_before)
    payload['ctl00$ContentPlaceHolder1$txtFileCreated2$dateInput'] =\
        '{d.month}/{d.day}/{d.year}'.format(d=created_before)

    payload['ctl00_ContentPlaceHolder1_txtFileCreated2_dateInput_ClientState'] =\
        '{{"enabled":true, "emptyMessage":"","validationText":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","valueAsString":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","minDateStr":"1980-01-01-00-00-00","maxDateStr":"2099-12-31-00-00-00", "lastSetTextBoxValue":"{d.month}/{d.day}/{d.year}"}}'.format(d=created_before)

    payload['ctl00$ContentPlaceHolder1$radFileCreated'] = 'between'

    return payload

def dateBound(creation_date) :
    payload = {}

    payload['ctl00$ContentPlaceHolder1$txtFileCreated1'] =\
        '{d.year}-{d.month:02}-{d.day:02}'.format(d=creation_date)
    payload['ctl00$ContentPlaceHolder1$txtFileCreated1$dateInput'] =\
        '{d.month}/{d.day}/{d.year}'.format(d=creation_date)

    payload['ctl00_ContentPlaceHolder1_txtFileCreated1_dateInput_ClientState'] =\
        '{{"enabled":true, "emptyMessage":"","validationText":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","valueAsString":"{d.year}-{d.month:02}-{d.day:02}-00-00-00","minDateStr":"1980-01-01-00-00-00","maxDateStr":"2099-12-31-00-00-00", "lastSetTextBoxValue":"{d.month}/{d.day}/{d.year}"}}'.format(d=creation_date)

    return payload
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Common tools:
#	pip/pip3 : install packages
#	pylint/pychecker : verify syntax
#	unittest package for xUnit.
""" This module helps setup a fresh install the way I like it. """

# Imports
import apt
import argparse
import os
import shutil
import subprocess

# Packages to install follow, broken down into categories.

PROGRAMS = """ \
    aptitude synaptic \
    kubuntu-restricted-extras kubuntu-restricted-addons \
    brother-cups-wrapper-laser brother-cups-wrapper-laser1 \
    brother-lpr-drivers-laser brother-lpr-drivers-laser1 \
    k3b kid3 krita kolourpaint4 kchmviewer yakuake plasma-widget-quickaccess \
    chromium-browser firefox \
    abiword baobab dvdrip easytag chm2pdf dia \
    gimp gimp-plugin-registry \
    fuse gparted quicksynergy \
    redshift geoclue geoclue-hostip numlockx \
    samba samba-doc samba-tools wireshark \
    ubuntuone-client ubuntuone-control-panel-qt \
    vlc ffmpeg-doc ffmpeg mplayer mencoder \
    p7zip-full rar zip unzip gzip \
    virtualbox-qt wine \
    ttf-xfree86-nonfree"""

# Series of packages follow, separated to be easily modified.
KEYRINGS = """ \
    debian-keyring debian-archive-keyring gnome-keyring \
    debian-ports-archive-keyring python-gnomekeyring python-keyring \
    ubuntu-keyring"""

PROGRAMMING = """ \
    build-essential debianutils ubuntu-dev-tools \
    automake automake1.9-doc ant ant-doc checkinstall checkbox \
    dkms docbook make-doc lynx kdiff3 kdiff3-doc patch rpm2cpio rpm \
    codeblocks kdevelop qt-sdk \
    colormake colordiff colorgcc \
    vim vim-doc vim-gtk vim-rails vim-syntax-go vim-syntax-gtk vim-doc \
    flex flex-doc bison bison-doc graphviz exuberant-ctags \
    clisp clisp-doc clisp-dev clisp-module-gdbm \
    erlang erlang-eunit \
    gcc gcc-doc gcc-4.7-source libcunit1 gdb gdb-doc cgdb xxgdb \
    libboost-all-dev libglm-dev libglew-dev libglfw-dev \
    libncursesw5-dev \
    gfortran \
    ghc ghc-doc ghc-haddock ghc-prof haskell-debian-utils \
    haskell-devscripts haskell-doc cabal-install \
    junit junit-doc maven openjdk-7-doc openjdk-7-jdk openjdk-7-dbg \
    openjdk-7-source openjdk-7-demo icedtea-7-plugin \
    lua5.2 lua5.2-doc luadoc \
    perl perl-doc perl-modules libpadwalker-perl \
    php5 php5-mysql phpunit php5-dev \
    python python-doc python3-doc jython jython-doc \
    ruby1.9.1-full shunit2 \
    bzr bzr-builddeb bzr-doc python-bzrlib bzrtools git git-gui git-doc \
    mercurial subversion cvs"""

CABAL = "buildwrapper scion-browser hoogle terminfo happy hlint"

class NotSudo(Exception):
    """ Throw this if we aren't sudo but need to be. """
    pass

# Functions

def install_packages():
    """ Installs required packages on this system. """
    if os.getuid() != 0:
        raise NotSudo('This stage must be run as root.')

    packages = '{} {} {}'.format(PROGRAMS, PROGRAMMING, KEYRINGS)
    packages = packages.split()

    cache = apt.Cache()
    print("One moment while we update cache.")
    cache.update()
    cache.open(None)
    print("Update done.")

    for pack in packages:
        package = cache[pack]
        if not package.is_installed:
            package.mark_install()

    cache.commit()
    cache.close()

def setup_config():
    """ Setup the dev environment, stuff goes in ~ and ~/.optSoftware. """
    src = './vim_and_bash_config/{}'
    dst = os.path.expanduser('~')

    # Copy config files for vim and bash into home.
    for fil in ['.vimrc', '.bash_aliases']:
        os.symlink(src, dst+'/'+fil)
    shutil.copytree(src.format('.vim'), dst + '/.vim')

def install_cabal():
    """ Installs locally some haskell packages for Eclipse Haskell plugin. """
    cmd = ['cabal', 'update']
    subprocess.call(cmd)

    cmd = 'cabal install {}'.format(CABAL)
    cmd = cmd.split()
    subprocess.call(cmd)

def take_choice(choice):
    """ Simple wrapper function to select right action. """
    choice = int(choice)
    if choice == 1:
        install_packages()
    elif choice == 2:
        setup_config()
    elif choice == 3:
        install_cabal()

if __name__ == '__main__':
    DESC = """ This program sets up a vanilla Ubuntu install.
    Pass a number to determine step.
    1 -> Install most packages.
    2 -> Setup vim, bash_aliases and development environment.
    3 -> Setup cabal for haskell development.
    """
    PARSER = argparse.ArgumentParser(description=DESC)
    PARSER.add_argument('choice', action='store', help='the stage')

    ARGS = PARSER.parse_args()  # Default parses argv[1:]

    try:
        take_choice(ARGS.choice)
    except IOError as exc:
        print('Failed to install: {}'.format(exc))
    except NotSudo as exc:
        print("Rerun this part of script with sudo.")
<commit_msg>Added protection checks to setup. Made it use sym links and init NeoBundle.<commit_after>#!/usr/bin/env python3
# Common tools:
#	pip/pip3 : install packages
#	pylint/pychecker : verify syntax
#	unittest package for xUnit.
""" This module helps setup a fresh install the way I like it. """

# Imports
import apt
import argparse
import os
import shutil
import subprocess

# Packages to install follow, broken down into categories.

PROGRAMS = """ \
    aptitude synaptic \
    kubuntu-restricted-extras kubuntu-restricted-addons \
    brother-cups-wrapper-laser brother-cups-wrapper-laser1 \
    brother-lpr-drivers-laser brother-lpr-drivers-laser1 \
    k3b kid3 krita kolourpaint4 kchmviewer yakuake plasma-widget-quickaccess \
    chromium-browser firefox \
    abiword baobab dvdrip easytag chm2pdf dia \
    gimp gimp-plugin-registry \
    fuse gparted quicksynergy \
    redshift geoclue geoclue-hostip numlockx \
    samba samba-doc samba-tools wireshark \
    ubuntuone-client ubuntuone-control-panel-qt \
    vlc ffmpeg-doc ffmpeg mplayer mencoder \
    p7zip-full rar zip unzip gzip \
    virtualbox-qt wine \
    ttf-xfree86-nonfree"""

# Series of packages follow, separated to be easily modified.
KEYRINGS = """ \
    debian-keyring debian-archive-keyring gnome-keyring \
    debian-ports-archive-keyring python-gnomekeyring python-keyring \
    ubuntu-keyring"""

PROGRAMMING = """ \
    build-essential debianutils ubuntu-dev-tools \
    automake automake1.9-doc ant ant-doc checkinstall checkbox \
    dkms docbook make-doc lynx kdiff3 kdiff3-doc patch rpm2cpio rpm \
    codeblocks kdevelop qt-sdk \
    colormake colordiff colorgcc \
    vim vim-doc vim-gtk vim-rails vim-syntax-go vim-syntax-gtk vim-doc \
    flex flex-doc bison bison-doc graphviz exuberant-ctags \
    clisp clisp-doc clisp-dev clisp-module-gdbm \
    erlang erlang-eunit \
    gcc gcc-doc gcc-4.7-source libcunit1 gdb gdb-doc cgdb xxgdb \
    libboost-all-dev libglm-dev libglew-dev libglfw-dev \
    libncursesw5-dev \
    gfortran \
    ghc ghc-doc ghc-haddock ghc-prof haskell-debian-utils \
    haskell-devscripts haskell-doc cabal-install \
    junit junit-doc maven openjdk-7-doc openjdk-7-jdk openjdk-7-dbg \
    openjdk-7-source openjdk-7-demo icedtea-7-plugin \
    lua5.2 lua5.2-doc luadoc \
    perl perl-doc perl-modules libpadwalker-perl \
    php5 php5-mysql phpunit php5-dev \
    python python-doc python3-doc jython jython-doc \
    ruby1.9.1-full shunit2 \
    bzr bzr-builddeb bzr-doc python-bzrlib bzrtools git git-gui git-doc \
    mercurial subversion cvs"""

CABAL = "buildwrapper scion-browser hoogle terminfo happy hlint"

class NotSudo(Exception):
    """ Throw this if we aren't sudo but need to be. """
    pass

# Functions

def install_packages():
    """ Installs required packages on this system. """
    if os.getuid() != 0:
        raise NotSudo('This stage must be run as root.')

    packages = '{} {} {}'.format(PROGRAMS, PROGRAMMING, KEYRINGS)
    packages = packages.split()

    cache = apt.Cache()
    print("One moment while we update cache.")
    cache.update()
    cache.open(None)
    print("Update done.")

    for pack in packages:
        package = cache[pack]
        if not package.is_installed:
            package.mark_install()

    cache.commit()
    cache.close()

def setup_config():
    """ Setup the dev environment, stuff goes in the user's home folder. """
    src = './vim_and_bash_config/{}'
    dst = os.path.expanduser('~')

    # Link to config files, copy .vim folder.
    for fil in ['.vimrc', '.bash_aliases']:
        dfile = dst + '/' + fil
        if not os.path.exists(dfile):
            os.symlink(src, dfile)

    # Copy vim folder if not there.
    ddir = dst + '/.vim'
    if not os.path.exists(ddir):
        shutil.copytree(src.format('.vim'), ddir)

    # Init NeoBundle.
    ddir = dst + '/.vim/bundle'
    if not os.path.exists(ddir):
        os.mkdir(ddir)
        cmd = ['git', 'clone', 'https://github.com/Shougo/neobundle.vim',
               '~/.vim/bundle/neobundle.vim']
        subprocess.call(cmd)

def install_cabal():
    """ Installs locally some haskell packages for Eclipse Haskell plugin. """
    cmd = ['cabal', 'update']
    subprocess.call(cmd)

    cmd = 'cabal install {}'.format(CABAL)
    cmd = cmd.split()
    subprocess.call(cmd)

def take_choice(choice):
    """ Simple wrapper function to select right action. """
    choice = int(choice)
    if choice == 1:
        install_packages()
    elif choice == 2:
        setup_config()
    elif choice == 3:
        install_cabal()

if __name__ == '__main__':
    DESC = """ This program sets up a vanilla Ubuntu install.
    Pass a number to determine step.
    1 -> Install most packages.
    2 -> Setup vim, bash_aliases and development environment.
    3 -> Setup cabal for haskell development.
    """
    PARSER = argparse.ArgumentParser(description=DESC)
    PARSER.add_argument('choice', action='store', help='the stage')

    ARGS = PARSER.parse_args()  # Default parses argv[1:]

    try:
        take_choice(ARGS.choice)
    except IOError as exc:
        print('Failed to install: {}'.format(exc))
    except NotSudo as exc:
        print("Rerun this part of script with sudo.")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# OpenCraft -- tools to aid developing and hosting free software projects
# Copyright (C) 2015-2016 OpenCraft <contact@opencraft.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
"""
Worker tasks for instance hosting & management
"""

# Imports #####################################################################

from datetime import datetime
import logging

from django.db.models import F
from huey.contrib.djhuey import crontab, db_task, db_periodic_task

from instance.models.load_balancer import LoadBalancingServer
from instance.models.openedx_appserver import OpenEdXAppServer
from instance.models.openedx_instance import OpenEdXInstance
from instance.utils import sufficient_time_passed
from pr_watch import github

# Logging #####################################################################

logger = logging.getLogger(__name__)

# Tasks #######################################################################

@db_task()
def spawn_appserver(
        instance_ref_id,
        mark_active_on_success=False,
        deactivate_old_appservers=False,
        num_attempts=1,
        success_tag=None,
        failure_tag=None):
    """
    Create a new AppServer for an existing instance.

    instance_ref_id should be the ID of an InstanceReference (instance.ref.pk)

    Optionally mark the new AppServer as active when the provisioning completes.
    Optionally deactivate old AppServers when the provisioning completes.
    Optionally retry up to 'num_attempts' times.
    Optionally tag the instance with 'success_tag' when the deployment succeeds,
    or failure_tag if it fails.
    """
    for i in range(1, num_attempts + 1):
        logger.info('Retrieving instance: ID=%s', instance_ref_id)
        # Fetch the instance inside the loop, in case it has been updated
        instance = OpenEdXInstance.objects.get(ref_set__pk=instance_ref_id)

        instance.logger.info('Spawning new AppServer, attempt %d of %d', i, num_attempts)
        appserver_id = instance.spawn_appserver()
        if appserver_id:
            if failure_tag:
                instance.tags.remove(failure_tag)
            if success_tag:
                instance.tags.add(success_tag)
            if mark_active_on_success and make_appserver_active(
                    appserver_id,
                    active=mark_active_on_success,
                    deactivate_others=deactivate_old_appservers
            ):
                break
    else:
        if failure_tag:
            instance.tags.add(failure_tag)
        if success_tag:
            instance.tags.remove(success_tag)

@db_task()
def make_appserver_active(appserver_id, active=True, deactivate_others=False):
    """
    Mark an AppServer as active or inactive.

    :returns
        True if the appserver was made active,
        False if the corresponding server was not healthy and thus the appserver was not made active.
    """
    appserver = OpenEdXAppServer.objects.get(pk=appserver_id)
    if not appserver.server.status.is_healthy_state:
        appserver.server.update_status()
        if not appserver.server.status.is_healthy_state:
            logger.info('Not %s %s: ID=%s since the server is not Ready.', "Activating" if active else "Deactivating",
                        appserver, appserver_id)
            return False

    logger.info('%s %s: ID=%s', "Activating" if active else "Deactivating", appserver, appserver_id)
    appserver.make_active(active=active)
    if active and deactivate_others:
        other_appservers = appserver.instance.appserver_set.filter(_is_active=True).exclude(pk=appserver_id)
        for appserver_to_deactivate in other_appservers:
            logger.info('Deactivating %s [%s]', appserver_to_deactivate, appserver_to_deactivate.id)
            appserver_to_deactivate.make_active(active=False)

    return True

@db_task()
def shut_down_obsolete_pr_sandboxes():
    """
    Shut down instances whose PRs got merged (more than) one week ago.
    """
    for instance in OpenEdXInstance.objects.filter(watchedpullrequest__isnull=False):
        pr = github.get_pr_info_by_number(
            instance.watchedpullrequest.target_fork_name,
            instance.watchedpullrequest.github_pr_number
        )
        if pr['state'] == 'closed' and not instance.ref.is_archived:
            closed_at = github.parse_date(pr['closed_at'])
            now = datetime.now()
            if sufficient_time_passed(closed_at, now, 7):
                instance.logger.info("Shutting down obsolete sandbox instance")
                instance.archive()

@db_task()
def terminate_obsolete_appservers_all_instances():
    """
    Terminate obsolete app servers for all instances.
    """
    for instance in OpenEdXInstance.objects.all():
        instance.logger.info("Terminating obsolete appservers for instance")
        instance.terminate_obsolete_appservers()

@db_periodic_task(crontab(day='*/1', hour='1', minute='0'))
def clean_up():
    """
    Clean up obsolete VMs.

    This task runs once per day.
    """
    shut_down_obsolete_pr_sandboxes()
    terminate_obsolete_appservers_all_instances()

@db_periodic_task(crontab())
def reconfigure_dirty_load_balancers():
    """
    Any load balancers that are dirty need to be reconfigured.

    This task runs every minute.
    """
    logger.info('Reconfiguring all dirty load balancers')
    for load_balancer in LoadBalancingServer.objects.filter(
            configuration_version__gt=F('deployed_configuration_version')
    ):
        load_balancer.reconfigure(mark_dirty=False)
<commit_msg>Catch any exception if an appserver fails to spawn.<commit_after># -*- coding: utf-8 -*-
#
# OpenCraft -- tools to aid developing and hosting free software projects
# Copyright (C) 2015-2016 OpenCraft <contact@opencraft.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
"""
Worker tasks for instance hosting & management
"""

# Imports #####################################################################

from datetime import datetime
import logging

from django.db.models import F
from huey.contrib.djhuey import crontab, db_task, db_periodic_task

from instance.models.load_balancer import LoadBalancingServer
from instance.models.openedx_appserver import OpenEdXAppServer
from instance.models.openedx_instance import OpenEdXInstance
from instance.utils import sufficient_time_passed
from pr_watch import github

# Logging #####################################################################

logger = logging.getLogger(__name__)

# Tasks #######################################################################

@db_task()
def spawn_appserver(
        instance_ref_id,
        mark_active_on_success=False,
        deactivate_old_appservers=False,
        num_attempts=1,
        success_tag=None,
        failure_tag=None):
    """
    Create a new AppServer for an existing instance.

    instance_ref_id should be the ID of an InstanceReference (instance.ref.pk)

    Optionally mark the new AppServer as active when the provisioning completes.
    Optionally deactivate old AppServers when the provisioning completes.
    Optionally retry up to 'num_attempts' times.
    Optionally tag the instance with 'success_tag' when the deployment succeeds,
    or failure_tag if it fails.
    """
    for i in range(1, num_attempts + 1):
        logger.info('Retrieving instance: ID=%s', instance_ref_id)
        # Fetch the instance inside the loop, in case it has been updated
        instance = OpenEdXInstance.objects.get(ref_set__pk=instance_ref_id)

        instance.logger.info('Spawning new AppServer, attempt %d of %d', i, num_attempts)
        try:
            appserver_id = instance.spawn_appserver()
        except Exception:  # pylint: disable=broad-except
            appserver_id = None
        if appserver_id:
            if failure_tag:
                instance.tags.remove(failure_tag)
            if success_tag:
                instance.tags.add(success_tag)
            if mark_active_on_success and make_appserver_active(
                    appserver_id,
                    active=mark_active_on_success,
                    deactivate_others=deactivate_old_appservers
            ):
                break
    else:
        if failure_tag:
            instance.tags.add(failure_tag)
        if success_tag:
            instance.tags.remove(success_tag)

@db_task()
def make_appserver_active(appserver_id, active=True, deactivate_others=False):
    """
    Mark an AppServer as active or inactive.

    :returns
        True if the appserver was made active,
        False if the corresponding server was not healthy and thus the appserver was not made active.
    """
    appserver = OpenEdXAppServer.objects.get(pk=appserver_id)
    if not appserver.server.status.is_healthy_state:
        appserver.server.update_status()
        if not appserver.server.status.is_healthy_state:
            logger.info('Not %s %s: ID=%s since the server is not Ready.', "Activating" if active else "Deactivating",
                        appserver, appserver_id)
            return False

    logger.info('%s %s: ID=%s', "Activating" if active else "Deactivating", appserver, appserver_id)
    appserver.make_active(active=active)
    if active and deactivate_others:
        other_appservers = appserver.instance.appserver_set.filter(_is_active=True).exclude(pk=appserver_id)
        for appserver_to_deactivate in other_appservers:
            logger.info('Deactivating %s [%s]', appserver_to_deactivate, appserver_to_deactivate.id)
            appserver_to_deactivate.make_active(active=False)

    return True

@db_task()
def shut_down_obsolete_pr_sandboxes():
    """
    Shut down instances whose PRs got merged (more than) one week ago.
    """
    for instance in OpenEdXInstance.objects.filter(watchedpullrequest__isnull=False):
        pr = github.get_pr_info_by_number(
            instance.watchedpullrequest.target_fork_name,
            instance.watchedpullrequest.github_pr_number
        )
        if pr['state'] == 'closed' and not instance.ref.is_archived:
            closed_at = github.parse_date(pr['closed_at'])
            now = datetime.now()
            if sufficient_time_passed(closed_at, now, 7):
                instance.logger.info("Shutting down obsolete sandbox instance")
                instance.archive()

@db_task()
def terminate_obsolete_appservers_all_instances():
    """
    Terminate obsolete app servers for all instances.
    """
    for instance in OpenEdXInstance.objects.all():
        instance.logger.info("Terminating obsolete appservers for instance")
        instance.terminate_obsolete_appservers()

@db_periodic_task(crontab(day='*/1', hour='1', minute='0'))
def clean_up():
    """
    Clean up obsolete VMs.

    This task runs once per day.
    """
    shut_down_obsolete_pr_sandboxes()
    terminate_obsolete_appservers_all_instances()

@db_periodic_task(crontab())
def reconfigure_dirty_load_balancers():
    """
    Any load balancers that are dirty need to be reconfigured.

    This task runs every minute.
    """
    logger.info('Reconfiguring all dirty load balancers')
    for load_balancer in LoadBalancingServer.objects.filter(
            configuration_version__gt=F('deployed_configuration_version')
    ):
        load_balancer.reconfigure(mark_dirty=False)
<|endoftext|>


<commit_before>from .runner import run

class Context(object):
    """
    Context-aware API wrapper & state-passing object.

    Stores various bits of configuration state internally and uses them to set
    default kwarg values for various API calls, such as `.run`.

    See method call docstrings for additional details.
    """
    def __init__(self, run=None):
        self.config = {
            'run': run or {}
        }

    def run(self, *args, **kwargs):
        """
        Wrapper for `.run`.

        To set default `.run` keyword argument values, instantiate `.Context`
        with the ``run`` kwarg set to a dict.

        E.g. to create a `.Context` whose `.Context.run` method always defaults
        to ``warn=True``::

            ctx = Context(run={'warn': True})
            ctx.run('command') # behaves like invoke.run('command', warn=True)

        """
        options = dict(self.config['run'])
        options.update(kwargs)
        return run(*args, **options)
<commit_msg>Reword docstring a tad<commit_after>from .runner import run

class Context(object):
    """
    Context-aware API wrapper & state-passing object.

    Stores various bits of configuration state and uses them to set default
    kwarg values for its wrappers to API calls such as `.Context.run`.

    See method call docstrings for additional details.
    """
    def __init__(self, run=None):
        self.config = {
            'run': run or {}
        }

    def run(self, *args, **kwargs):
        """
        Wrapper for `.run`.

        To set default `.run` keyword argument values, instantiate `.Context`
        with the ``run`` kwarg set to a dict.

        E.g. to create a `.Context` whose `.Context.run` method always defaults
        to ``warn=True``::

            ctx = Context(run={'warn': True})
            ctx.run('command') # behaves like invoke.run('command', warn=True)

        """
        options = dict(self.config['run'])
        options.update(kwargs)
        return run(*args, **options)
<|endoftext|>


<commit_before># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:

import numpy as np
import nibabel as nib
import os
import cPickle
from scipy.io import savemat, loadmat
import pandas as pd

pjoin = os.path.join
class IOFactory(object):
    """
    Make a factory for congruent read/write data
    Usage:
        >>>factory = iofiles.IOFactory()
        >>>factory.createfactory('.', 'data.csv')
    """
    def createfactory(self, filepath, filename):
        """
        Create your factory
        ----------------------------------------
        Input:
            filepath: filepath as reading/writing
            filename: filenames
        Output:
            A class

        Note:
            What support now is .csv, .pkl, .mat and .nifti
        """
        _comp_file = pjoin(filepath, filename)
        if _comp_file.endswith('csv'):
            return _CSV(_comp_file)
        elif _comp_file.endswith('txt'):
            return _TXT(_comp_file)
        elif _comp_file.endswith('pkl'):
            return _PKL(_comp_file)
        elif _comp_file.endswith('mat'):
            return _MAT(_comp_file)
        elif _comp_file.endswith('gz') | _comp_file.endswith('nii'):
            return _NIFTI(_comp_file)
        else:
            return None

class _CSV(object):
    def __init__(self, _comp_file):
	    self._comp_file = _comp_file

    def save_csv(self, data, labels = None):
        """
        Save a np array into a csv file.
        ---------------------------------------------
        Parameters:
            data: raw data
            labels: Data names. Labels as a list.
        """
        if isinstance(data, np.ndarray):
            if data.ndim == 1:
                data = np.expand_dims(data,axis=1)
            try:
                f = open(self._comp_file, 'w')
            except IOError:
                print('Can not save file' + self._comp_file)
            else:
                if isinstance(labels, list):
                    labels = [str(item) for item in labels]
                    labels = ','.join(labels)
                    f.write(labels + '\n')
                for line in data:
                    line_str = [str(item) for item in line]
                    line_str = ','.join(line_str)
                    f.write(line_str + '\n')
                f.close()
        else:
            raise ValueError, 'Input must be a numpy array.'

    def read_csv(self):
        """
        Read data from .csv
        ----------------------------------
        Parameters:
            outdata: a directory, with key and its data
        """
        pddata = pd.read_csv(self._comp_file)
        outdata = {}
        for key in pddata.keys():
            outdata[key] = pddata[key].get_values()
        return outdata

class _TXT(object):
    def __init__(self, _comp_file):
        self._comp_file = _comp_file

    def save_txt(self, data):

        """
        Save data to .txt
        ------------------------
        Parameters:
            data: raw data
        """
        np.savetxt(self._comp_file, data)

    def load_txt(self):
        """
        Load .txt data
        ------------------------
        """
        return np.loadtxt(self._comp_file)

class _PKL(object):
    def __init__(self, _comp_file):
        self._comp_file = _comp_file

    def save_pkl(self, data):
        """
        Save data to .pkl
        ----------------------------
        Parameters:
            data: raw data
        """
        output_class = open(self._comp_file, 'wb')
        cPickle.dump(data, output_class)
        output_class.close()

    def load_pkl(self):
        """
        Load data from .pkl
        ------------------------------
        Parameters:
            filename: file name
            path: path of pointed pickle file
        Return:
            data
        """
        pkl_file = open(self._comp_file, 'rb')
        data = cPickle.load(pkl_file)
        pkl_file.close()
        return data

class _MAT(object):
    def __init__(self, _comp_file):
        self._comp_file = _comp_file

    def save_mat(self, data):
        """
        Save data to .mat
        ---------------------------------------
        Parameters:
            data: raw data dictionary, note that data must be a dictionary data
        """
        savemat(self._comp_file, data)

    def load_mat(self):
        """
        Load data from .mat
        ---------------------------------------
        Parameters:
            outdata: output data
        """
        return loadmat(self._comp_file)

class _NIFTI(object):
    def __init__(self, _comp_file):
        self._comp_file = _comp_file

    def save_nifti(self, data, header):
        """
        Save nifti data
	Parameters:
            data: saving data
        """
        img = nib.Nifti1Image(data, None, header)
        nib.save(img, self._comp_file)

    def load_nifti(self, datatype = 'data'):
        """
        Load nifti data.
        Parameters:
        -----------------------------------
        datatype: data type to load.
                  By default, 'data', nifti image values
                  'affine', affine matrix
                  'header', header
                  'shape', matrix shapes
        """
        img = nib.load(self._comp_file)
        if datatype == 'data':
            outdata = img.get_data()
        elif datatype == 'affine':
            outdata = img.get_affine()
        elif datatype == 'header':
            outdata = img.get_header()
        elif datatype == 'shape':
            outdata = img.get_shape()
        else:
            raise Exception('Wrong datatype input')
        return outdata

<commit_msg>fix raise error to suit higher version python<commit_after># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:

import numpy as np
import nibabel as nib
import os
import cPickle
from scipy.io import savemat, loadmat
import pandas as pd

pjoin = os.path.join
class IOFactory(object):
    """
    Make a factory for congruent read/write data
    Usage:
        >>>factory = iofiles.IOFactory()
        >>>factory.createfactory('.', 'data.csv')
    """
    def createfactory(self, filepath, filename):
        """
        Create your factory
        ----------------------------------------
        Input:
            filepath: filepath as reading/writing
            filename: filenames
        Output:
            A class

        Note:
            What support now is .csv, .pkl, .mat and .nifti
        """
        _comp_file = pjoin(filepath, filename)
        if _comp_file.endswith('csv'):
            return _CSV(_comp_file)
        elif _comp_file.endswith('txt'):
            return _TXT(_comp_file)
        elif _comp_file.endswith('pkl'):
            return _PKL(_comp_file)
        elif _comp_file.endswith('mat'):
            return _MAT(_comp_file)
        elif _comp_file.endswith('gz') | _comp_file.endswith('nii'):
            return _NIFTI(_comp_file)
        else:
            return None

class _CSV(object):
    def __init__(self, _comp_file):
	    self._comp_file = _comp_file

    def save_csv(self, data, labels = None):
        """
        Save a np array into a csv file.
        ---------------------------------------------
        Parameters:
            data: raw data
            labels: Data names. Labels as a list.
        """
        if isinstance(data, np.ndarray):
            if data.ndim == 1:
                data = np.expand_dims(data,axis=1)
            try:
                f = open(self._comp_file, 'w')
            except IOError:
                print('Can not save file' + self._comp_file)
            else:
                if isinstance(labels, list):
                    labels = [str(item) for item in labels]
                    labels = ','.join(labels)
                    f.write(labels + '\n')
                for line in data:
                    line_str = [str(item) for item in line]
                    line_str = ','.join(line_str)
                    f.write(line_str + '\n')
                f.close()
        else:
            raise Exception('Input must be a numpy array.')

    def read_csv(self):
        """
        Read data from .csv
        ----------------------------------
        Parameters:
            outdata: a directory, with key and its data
        """
        pddata = pd.read_csv(self._comp_file)
        outdata = {}
        for key in pddata.keys():
            outdata[key] = pddata[key].get_values()
        return outdata

class _TXT(object):
    def __init__(self, _comp_file):
        self._comp_file = _comp_file

    def save_txt(self, data):

        """
        Save data to .txt
        ------------------------
        Parameters:
            data: raw data
        """
        np.savetxt(self._comp_file, data)

    def load_txt(self):
        """
        Load .txt data
        ------------------------
        """
        return np.loadtxt(self._comp_file)

class _PKL(object):
    def __init__(self, _comp_file):
        self._comp_file = _comp_file

    def save_pkl(self, data):
        """
        Save data to .pkl
        ----------------------------
        Parameters:
            data: raw data
        """
        output_class = open(self._comp_file, 'wb')
        cPickle.dump(data, output_class)
        output_class.close()

    def load_pkl(self):
        """
        Load data from .pkl
        ------------------------------
        Parameters:
            filename: file name
            path: path of pointed pickle file
        Return:
            data
        """
        pkl_file = open(self._comp_file, 'rb')
        data = cPickle.load(pkl_file)
        pkl_file.close()
        return data

class _MAT(object):
    def __init__(self, _comp_file):
        self._comp_file = _comp_file

    def save_mat(self, data):
        """
        Save data to .mat
        ---------------------------------------
        Parameters:
            data: raw data dictionary, note that data must be a dictionary data
        """
        savemat(self._comp_file, data)

    def load_mat(self):
        """
        Load data from .mat
        ---------------------------------------
        Parameters:
            outdata: output data
        """
        return loadmat(self._comp_file)

class _NIFTI(object):
    def __init__(self, _comp_file):
        self._comp_file = _comp_file

    def save_nifti(self, data, header):
        """
        Save nifti data
	Parameters:
            data: saving data
        """
        img = nib.Nifti1Image(data, None, header)
        nib.save(img, self._comp_file)

    def load_nifti(self, datatype = 'data'):
        """
        Load nifti data.
        Parameters:
        -----------------------------------
        datatype: data type to load.
                  By default, 'data', nifti image values
                  'affine', affine matrix
                  'header', header
                  'shape', matrix shapes
        """
        img = nib.load(self._comp_file)
        if datatype == 'data':
            outdata = img.get_data()
        elif datatype == 'affine':
            outdata = img.get_affine()
        elif datatype == 'header':
            outdata = img.get_header()
        elif datatype == 'shape':
            outdata = img.get_shape()
        else:
            raise Exception('Wrong datatype input')
        return outdata

<|endoftext|>


<commit_before>from urllib.request import urlopen
from urllib.parse import urlencode
import json
import logging
from datetime import datetime, date as Date
from pytz import timezone
import sys
import re
from typing import List, Iterator, Tuple
from . import env

LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)

URL = 'http://www.elliottbaycrossfit.com/api/v1/wods?'

def _is_announcement_line(line: str) -> bool:
    """
    Is a line of text considered an announcement line?

    Examples:

    HAPPY BIRTHDAY <person>!!!!
    NO CLASSES <>

    :param line: line to check
    :return: if the line matches an expected pattern
    """
    return bool(line) and (line.upper() == line or line.endswith('!!'))

def _split_announcement_and_strength(strength_raw: str) -> Tuple[List[str], List[str]]:
    """
    Split the strength section into 2 sections: announcements and the actual
    strength training workout.

    Rohan sometimes starts the description of the strength training
    section with an announcement, like wishing someone happy birthday
    or announcing that a class is canceled.

    :param strength_raw: The raw strength text from the API
    :return: 2-tuple with the strength and announcements split by line
    """
    strength_raw = strength_raw.strip()
    if not strength_raw:
        return [], []
    announcement = []
    lines = strength_raw.splitlines(False)
    while _is_announcement_line(lines[0]):
        announcement.append(lines.pop(0))
    return announcement, lines

class WOD(object):
    def __init__(self, wod_attributes: dict):
        self.announcement_lines, self.strength_lines = _split_announcement_and_strength(
            wod_attributes.get('strength', '')
        )
        self.conditioning_lines = wod_attributes.get('conditioning', '').splitlines(False)
        self.image = wod_attributes.get('image', None)
        self.datetime = _safe_datetime(wod_attributes.get('date'))
        self.date = None
        if self.datetime:
            self.date = self.datetime.date()
        self.publish_datetime = _safe_datetime(wod_attributes.get('publishDate'))

    def announcement_ssml(self) -> str:
        if self.announcement_lines:
            ssml_chunks = ['
Announcement:']
            for line in self.announcement_lines:
                line = line.strip()
                if line:
                    ssml_chunks.append('<s>{}</s>'.format(_clean_illegal_ssml_chars(line)))
                else:
                    ssml_chunks.append('<break time="500ms"/>')
            ssml_chunks.append('
')
            return ''.join(ssml_chunks)
        return ''

    def announcement_pprint(self) -> str:
        if self.announcement_lines:
            return 'Announcement:\n' + '\n'.join(self.announcement_lines)
        return ''

    def strength_ssml(self) -> str:
        if self.strength_lines:
            return _convert_ssml(self.strength_lines, 'Strength Section:')
        return ''

    def strength_pprint(self) -> str:
        if self.strength_lines:
            return 'Strength:\n' + '\n'.join(self.strength_lines)
        return ''

    def conditioning_ssml(self) -> str:
        if self.conditioning_lines:
            return _convert_ssml(self.conditioning_lines, 'Conditioning:')
        return ''

    def conditioning_pprint(self) -> str:
        if self.conditioning_lines:
            return 'Conditioning:\n' + '\n'.join(self.conditioning_lines)
        return ''

    def full_ssml(self) -> str:
        return self.announcement_ssml() + self.strength_ssml() + self.conditioning_ssml()

    def pprint(self) -> str:
        return '\n'.join([
            self.announcement_pprint(),
            self.strength_pprint(),
            self.conditioning_pprint()
        ])

    def as_wod_attributes(self) -> dict:
        return {
            'strength': self.strength_raw,
            'conditioning': self.conditioning_raw,
            'image': self.image,
            'date': self.datetime.strftime(EBCF_API_TSTAMP_FMT),
            'publishDate': self.publish_datetime.strftime(EBCF_API_TSTAMP_FMT)
        }

class APIParseError(ValueError):
    """Thrown when the underlying expectations of the API stop working."""

def _urlencode_multilevel(obj: dict) -> str:
    """
    EBCF uses PHP-style query args that support nested dictionaries.

    E.g. we need to typically pass the following args to the API:
        filter[simple][date]:2017-06-01T00:00:00.000Z
        filter[simple][enabled]:True

    and these need to be encoded.
    """
    flattened_params = {}

    def _flatten_obj(obj, parent_key):
        sub_params = {}
        if isinstance(obj, dict):
            for child_key in obj:
                encode_key = '{}[{}]'.format(parent_key, child_key)
                sub_params[encode_key] = obj[child_key]
        elif isinstance(obj, list):
            for i, val in enumerate(obj):
                encode_key = '{}[{}]'.format(parent_key, i)
                sub_params[encode_key] = obj[child_key]
        else:
            flattened_params[parent_key] = obj
        for k, v in sub_params.items():
            _flatten_obj(v, k)

    if isinstance(obj, dict):
        for k, v in obj.items():
            _flatten_obj(v, k)

    return urlencode(flattened_params)

def _call_api(params: dict) -> dict:
    LOG.debug('EBCF API params: %s', params)
    query_url = URL + _urlencode_multilevel(params)
    LOG.debug('HTTP GET %s', query_url)
    with urlopen(query_url) as f:
        return json.load(f)

def _parse_wod_response(api_response: dict) -> Iterator[WOD]:
    LOG.debug('EBCF API response: %s', api_response)
    wod_list = api_response.get('data', [])
    for wod_data in wod_list:
        try:
            yield WOD(wod_data['attributes'])
        except KeyError:
            continue

EBCF_RANGE_STRF_FMT = '%Y-%m-%dT%H:%M:%S%z'

def get_wods_by_range(start_date: datetime, end_date: datetime) -> List[WOD]:
    """
    Gets the WOD by publishDate range.

    :param start_date: Start day
    :param end_date: End day
    :return: WOD
    :rtype: WOD
    """
    params = {'filter': {'simple': {
        'publishDate': {
            '$gt': start_date.strftime(EBCF_RANGE_STRF_FMT),
            '$lt': end_date.strftime(EBCF_RANGE_STRF_FMT)
        },
        'enabled': True
    }}}
    return list(_parse_wod_response(_call_api(params)))

def get_wod(date: Date) -> WOD:
    """
    gets the WOD for a specific day.

    :param datetime.date date: the date
    :returns: wod data or None if not found
    :rtype: WOD
    """
    params = {'filter': {'simple': {
        'date': date.strftime('%Y-%m-%d') + 'T00:00:00.000Z',
        'enabled': True
    }}}
    for wod in _parse_wod_response(_call_api(params)):
        if wod.date == date:
            return wod

_ALIASES = {
    r'OH': r'OH',
    r'DB': r'DB',
    r'KB': r'KB',
    r'EMOM': r'every minute on the minute',
    r'E(\d)MOM': r'every \1 minutes on the minute',
    r'HSPU': r'hand stand push ups',
    r'#': r'#',
    r'(\d+)"': r'\1"',
    r'\'': '\'',
    r'&': 'and',
    r'(\d+) [Ss]ec\.? ': r'\1 second '
}

def _inject_aliases(text: str) -> str:
    for key, replacement in _ALIASES.items():
        text = re.sub(key, replacement, text)
    return text

def _fix_sets(text: str) -> str:
    return re.sub(r'(\d+)x(\d+)', r'\1 sets of \2', text)

def _fix_rx(text: str) -> str:
    return re.sub(r'(\d+[#"\'])/(\d+[#"\'])', r'<prosody rate="fast">\1 male, \2 female</prosody>', text)

def _clean_illegal_ssml_chars(text: str) -> str:
    return text.replace('&', 'and')

def _massage_for_tts(text: str) -> str:
    text = _fix_sets(text)
    text = _fix_rx(text)
    text = _inject_aliases(text)
    return text

def _convert_ssml(lines: List[str], section: str) -> str:
    section = '
%s
' % section
    new_lines = [
        '<s>{}</s>'.format(_massage_for_tts(l))
        for l in lines
    ]
    return section + ''.join(new_lines)

EBCF_API_TSTAMP_FMT = '%Y-%m-%dT%H:%M:%S.000Z'

def _safe_datetime(datestr: str) -> datetime:
    """Tries to convert a timestamp into a datetime object, without crashing.

    :param datestr: date string
    :returns: datetime object set to UTC or None
    """
    if not str:
        return None
    try:
        return env.UTC.localize(datetime.strptime(datestr, EBCF_API_TSTAMP_FMT))
    except ValueError:
        return None

def _test(argv: List[str]) -> None:
    logging.basicConfig(format='%(levelname)s %(filename)s-%(funcName)s-%(lineno)d: %(message)s', level=logging.DEBUG)
    try:
        date = datetime.strptime(argv[1], '%Y-%m-%d').date()
    except IndexError:
        print('Must give me a date in format: YYYY-MM-DD')
        sys.exit(1)
    wod = get_wod(date)
    if wod:
        print(wod.pprint())
        print('SSML:')
        print(wod.full_ssml())
    else:
        print(wod)

if __name__ == '__main__':
    _test(sys.argv)

<commit_msg>_split_announcement_and_strength fix empty line issue<commit_after>from urllib.request import urlopen
from urllib.parse import urlencode
import json
import logging
from datetime import datetime, date as Date
from pytz import timezone
import sys
import re
from typing import List, Iterator, Tuple
from . import env

LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)

URL = 'http://www.elliottbaycrossfit.com/api/v1/wods?'

def _is_announcement_line(line: str) -> bool:
    """
    Is a line of text considered an announcement line?

    Examples:

    HAPPY BIRTHDAY <person>!!!!
    NO CLASSES <>

    :param line: line to check
    :return: if the line matches an expected pattern
    """
    return bool(line) and (line.upper() == line or line.endswith('!!'))

def _split_announcement_and_strength(strength_raw: str) -> Tuple[List[str], List[str]]:
    """
    Split the strength section into 2 sections: announcements and the actual
    strength training workout.

    Rohan sometimes starts the description of the strength training
    section with an announcement, like wishing someone happy birthday
    or announcing that a class is canceled.

    :param strength_raw: The raw strength text from the API
    :return: 2-tuple with the strength and announcements split by line
    """
    strength_raw = strength_raw.strip()
    if not strength_raw:
        return [], []
    announcement = []
    lines = [l.strip() for l in strength_raw.splitlines(False)]
    # while we have lines to parse (whole strength can be an announcement)
    # and the line is an announcement line or empty line (multiple announcements
    # can be split by an empty line)
    while lines and (_is_announcement_line(lines[0]) or not lines[0]):
        announcement.append(lines.pop(0))
    return announcement, lines

class WOD(object):
    def __init__(self, wod_attributes: dict):
        self.announcement_lines, self.strength_lines = _split_announcement_and_strength(
            wod_attributes.get('strength', '')
        )
        self.conditioning_lines = wod_attributes.get('conditioning', '').splitlines(False)
        self.image = wod_attributes.get('image', None)
        self.datetime = _safe_datetime(wod_attributes.get('date'))
        self.date = None
        if self.datetime:
            self.date = self.datetime.date()
        self.publish_datetime = _safe_datetime(wod_attributes.get('publishDate'))

    def announcement_ssml(self) -> str:
        if self.announcement_lines:
            ssml_chunks = ['
Announcement:']
            for line in self.announcement_lines:
                line = line.strip()
                if line:
                    ssml_chunks.append('<s>{}</s>'.format(_clean_illegal_ssml_chars(line)))
                else:
                    ssml_chunks.append('<break time="500ms"/>')
            ssml_chunks.append('
')
            return ''.join(ssml_chunks)
        return ''

    def announcement_pprint(self) -> str:
        if self.announcement_lines:
            return 'Announcement:\n' + '\n'.join(self.announcement_lines)
        return ''

    def strength_ssml(self) -> str:
        if self.strength_lines:
            return _convert_ssml(self.strength_lines, 'Strength Section:')
        return ''

    def strength_pprint(self) -> str:
        if self.strength_lines:
            return 'Strength:\n' + '\n'.join(self.strength_lines)
        return ''

    def conditioning_ssml(self) -> str:
        if self.conditioning_lines:
            return _convert_ssml(self.conditioning_lines, 'Conditioning:')
        return ''

    def conditioning_pprint(self) -> str:
        if self.conditioning_lines:
            return 'Conditioning:\n' + '\n'.join(self.conditioning_lines)
        return ''

    def full_ssml(self) -> str:
        return self.announcement_ssml() + self.strength_ssml() + self.conditioning_ssml()

    def pprint(self) -> str:
        return '\n'.join([
            self.announcement_pprint(),
            self.strength_pprint(),
            self.conditioning_pprint()
        ])

    def as_wod_attributes(self) -> dict:
        return {
            'strength': self.strength_raw,
            'conditioning': self.conditioning_raw,
            'image': self.image,
            'date': self.datetime.strftime(EBCF_API_TSTAMP_FMT),
            'publishDate': self.publish_datetime.strftime(EBCF_API_TSTAMP_FMT)
        }

class APIParseError(ValueError):
    """Thrown when the underlying expectations of the API stop working."""

def _urlencode_multilevel(obj: dict) -> str:
    """
    EBCF uses PHP-style query args that support nested dictionaries.

    E.g. we need to typically pass the following args to the API:
        filter[simple][date]:2017-06-01T00:00:00.000Z
        filter[simple][enabled]:True

    and these need to be encoded.
    """
    flattened_params = {}

    def _flatten_obj(obj, parent_key):
        sub_params = {}
        if isinstance(obj, dict):
            for child_key in obj:
                encode_key = '{}[{}]'.format(parent_key, child_key)
                sub_params[encode_key] = obj[child_key]
        elif isinstance(obj, list):
            for i, val in enumerate(obj):
                encode_key = '{}[{}]'.format(parent_key, i)
                sub_params[encode_key] = obj[child_key]
        else:
            flattened_params[parent_key] = obj
        for k, v in sub_params.items():
            _flatten_obj(v, k)

    if isinstance(obj, dict):
        for k, v in obj.items():
            _flatten_obj(v, k)

    return urlencode(flattened_params)

def _call_api(params: dict) -> dict:
    LOG.debug('EBCF API params: %s', params)
    query_url = URL + _urlencode_multilevel(params)
    LOG.debug('HTTP GET %s', query_url)
    with urlopen(query_url) as f:
        return json.load(f)

def _parse_wod_response(api_response: dict) -> Iterator[WOD]:
    LOG.debug('EBCF API response: %s', api_response)
    wod_list = api_response.get('data', [])
    for wod_data in wod_list:
        try:
            yield WOD(wod_data['attributes'])
        except KeyError:
            continue

EBCF_RANGE_STRF_FMT = '%Y-%m-%dT%H:%M:%S%z'

def get_wods_by_range(start_date: datetime, end_date: datetime) -> List[WOD]:
    """
    Gets the WOD by publishDate range.

    :param start_date: Start day
    :param end_date: End day
    :return: WOD
    :rtype: WOD
    """
    params = {'filter': {'simple': {
        'publishDate': {
            '$gt': start_date.strftime(EBCF_RANGE_STRF_FMT),
            '$lt': end_date.strftime(EBCF_RANGE_STRF_FMT)
        },
        'enabled': True
    }}}
    return list(_parse_wod_response(_call_api(params)))

def get_wod(date: Date) -> WOD:
    """
    gets the WOD for a specific day.

    :param datetime.date date: the date
    :returns: wod data or None if not found
    :rtype: WOD
    """
    params = {'filter': {'simple': {
        'date': date.strftime('%Y-%m-%d') + 'T00:00:00.000Z',
        'enabled': True
    }}}
    for wod in _parse_wod_response(_call_api(params)):
        if wod.date == date:
            return wod

_ALIASES = {
    r'OH': r'OH',
    r'DB': r'DB',
    r'KB': r'KB',
    r'EMOM': r'every minute on the minute',
    r'E(\d)MOM': r'every \1 minutes on the minute',
    r'HSPU': r'hand stand push ups',
    r'#': r'#',
    r'(\d+)"': r'\1"',
    r'\'': '\'',
    r'&': 'and',
    r'(\d+) [Ss]ec\.? ': r'\1 second '
}

def _inject_aliases(text: str) -> str:
    for key, replacement in _ALIASES.items():
        text = re.sub(key, replacement, text)
    return text

def _fix_sets(text: str) -> str:
    return re.sub(r'(\d+)x(\d+)', r'\1 sets of \2', text)

def _fix_rx(text: str) -> str:
    return re.sub(r'(\d+[#"\'])/(\d+[#"\'])', r'<prosody rate="fast">\1 male, \2 female</prosody>', text)

def _clean_illegal_ssml_chars(text: str) -> str:
    return text.replace('&', 'and')

def _massage_for_tts(text: str) -> str:
    text = _fix_sets(text)
    text = _fix_rx(text)
    text = _inject_aliases(text)
    return text

def _convert_ssml(lines: List[str], section: str) -> str:
    section = '
%s
' % section
    new_lines = [
        '<s>{}</s>'.format(_massage_for_tts(l))
        for l in lines
    ]
    return section + ''.join(new_lines)

EBCF_API_TSTAMP_FMT = '%Y-%m-%dT%H:%M:%S.000Z'

def _safe_datetime(datestr: str) -> datetime:
    """Tries to convert a timestamp into a datetime object, without crashing.

    :param datestr: date string
    :returns: datetime object set to UTC or None
    """
    if not str:
        return None
    try:
        return env.UTC.localize(datetime.strptime(datestr, EBCF_API_TSTAMP_FMT))
    except ValueError:
        return None

def _test(argv: List[str]) -> None:
    logging.basicConfig(format='%(levelname)s %(filename)s-%(funcName)s-%(lineno)d: %(message)s', level=logging.DEBUG)
    try:
        date = datetime.strptime(argv[1], '%Y-%m-%d').date()
    except IndexError:
        print('Must give me a date in format: YYYY-MM-DD')
        sys.exit(1)
    wod = get_wod(date)
    if wod:
        print(wod.pprint())
        print('SSML:')
        print(wod.full_ssml())
    else:
        print(wod)

if __name__ == '__main__':
    _test(sys.argv)

<|endoftext|>