<commit_before>"""
Binary Tree implementations, recursive and iterative traversals
@author: Lucas Nemeth

This file may be distributed and/or modified under the terms of the
GNU General Public License version 2 as published by the Free Software
Foundation.
"""
import weakref
from utils import Queue, Stack

def copy_node(origin, destination):
    if not (origin is None or destination is None):
        destination.key = origin.key
        destination.item = origin.item
        destination.left = origin.left
        destination.right = origin.right

class BinaryTree(object):
    '''A generic Binary Tree implementation (en.wikipedia.org/wiki/Binary_tree)
    Used as super-class to other binary trees
    '''
    def __init__(self, key=None, item=None, left=None, right=None):
        self.key = key
        self.item = item
        self.left = left
        self.right = right
        self._parent = None

    @property
    def parent(self):
        if self._parent:
            return self._parent()

    @property
    def left(self):
        return self._left

    @left.setter
    def left(self, val):
        self._left = val
        if not self._left is None:
            self._left._parent = weakref.ref(self)

    @property
    def right(self):
        return self._right

    @right.setter
    def right(self, val):
        self._right = val
        if not self._right is None:
            self._right._parent = weakref.ref(self)

    def get_key(self):
        """Returns the key value of the binary tree"""
        return self.key

    def __str__(self):
        return '{} : {}'.format(self.key, self.item)

    def is_leaf(self):
        return (self.left is None) and (self.right is None)

    def get_heigth(self):
        """recursive get Heigth method"""
        hl = self.left.getHeigth()
        hr = self.right.getHeigth()
        #returns the highest heigth
        if (hl < hr):
            # + 1 --> counting self
            return hl + 1
        else:
            # + 1 --> counting self
            return hr + 1

    def in_order(self):
        '''recursive in-order traversal.
        Returns a list.
        '''
        l = [self]
        if self.left:
            l = self.left.in_order() + l
        if self.right:
            l += self.right.in_order()
        return l

    def pre_order(self):
        '''recursive pre-order traversal.
        Returns a list.
        '''
        l = [self]
        if self.left:
            l += self.left.pre_order()
        if self.right:
            l += self.right.pre_order()
        return l

    def post_order(self):
        """recursive post-order traversal.
        Returns a list."""
        l = []
        if self.left:
            l = self.left.post_order()
        if self.right:
            l = l + self.right.post_order()
        l = l + [self]
        return l

    def breadth_first_search(self):
        """iterative Breadth First Search.
        Returns a list."""
        l = []
        queue = Queue()
        #enqueue itself
        queue.enqueue(self)
        while not queue.is_empty() :
            #dequeue and gets acess to top of queue
            p = queue.dequeue()
            #append to output list
            l.append(p)
            #gets right child, if it's not empty, enqueue
            child_right = p.right
            if child_right:
                queue.enqueue(child_right)
            #gets right child, if it's not empty, enqueue
            child_left = p.left
            if child_left:
                queue.enqueue(child_left)
        return l

    def in_order_iterative(self):
        '''iterative in order method. returns a list'''
        stack = Stack()
        tree = self
        l = []
        while tree or not stack.is_empty():
            if tree:
                stack.push(tree)
                tree = tree.left
            else:
                tree = stack.pop()
                l.append(tree)
                tree = tree.right
        return l

    def pre_order_iterative(self):
        """Iterative preoder method"""
        stack = Stack()
        tree = self
        queue = Queue()
        stack.push(tree)
        l = []
        while not stack.is_empty():
            tree = stack.pop()
            queue.enqueue(tree)
            if tree.right:
                stack.push(tree.right)
            if tree.left:
                stack.push(tree.left)
        while not queue.is_empty():
            l.append(queue.dequeue())
        return l

    def post_order_iterative(self):
        """Iterative post order method
        it's the same method of the preorder
        iterative, but reversed, using an
        stack in the place of the queue"""
        stack = Stack()
        output = Stack()
        tree = self
        stack.push(tree)
        l = []
        while not stack.is_empty():
            tree = stack.pop()
            output.push(tree)
            if tree.left:
                stack.push(tree.left)
            if tree.right:
                stack.push(tree.right)
        while not output.is_empty():
            l.append(output.pop())
        return l

class BinarySearchTree(BinaryTree):

    def search(self, key):
        tree = self
        while tree:
            if key == tree.get_key():
                return tree
            elif key < tree.get_key():
                tree = tree.left
            else:
                tree = tree.right

    def insert(self, key, item):
        tree = self
        aux = tree
        while tree:
            aux = tree
            if key < tree.get_key():
                tree = tree.left
            else:
                tree = tree.right
        newtree = BinarySearchTree(key,item)
        if key <  aux.get_key():
            aux.left = newtree
        else:
            aux.right = newtree

    def remove_root(self):
        #The tree only have a left child
        if self.right is None and not self.left is None:
            copy_node(self.left, self)
        #The tree only have a right child
        if self.left is None and not self.right is None:
            copy_node(self.right, self)
        #The tree haves the two children
        if not self.left is None and not self.right is None:
            node = self.left
            while node:
                tree = node
                node = node.right
            old_left = self.left
            old_right = self.right
            parent = tree.parent
            copy_node(tree, self)
            remove_node(tree)
            self.left = old_left
            self.right = old_right

def remove_node(node):
    parent = node.parent
    if parent:
        if parent.right == node:
            parent.right = None
        else:
            parent.left = None
    else:
        self.key = None
        self.item = None

def remove_search_tree(tree, key):
    node = tree.search(key)
    if not node.right and not node.left:
        remove_node(node)
    else:
        node.remove_root()
<commit_msg>correct license<commit_after>"""
Binary Tree implementations, recursive and iterative traversals
@author: Lucas Nemeth
"""

import weakref
from utils import Queue, Stack

def copy_node(origin, destination):
    if not (origin is None or destination is None):
        destination.key = origin.key
        destination.item = origin.item
        destination.left = origin.left
        destination.right = origin.right

class BinaryTree(object):
    '''A generic Binary Tree implementation (en.wikipedia.org/wiki/Binary_tree)
    Used as super-class to other binary trees
    '''
    def __init__(self, key=None, item=None, left=None, right=None):
        self.key = key
        self.item = item
        self.left = left
        self.right = right
        self._parent = None

    @property
    def parent(self):
        if self._parent:
            return self._parent()

    @property
    def left(self):
        return self._left

    @left.setter
    def left(self, val):
        self._left = val
        if not self._left is None:
            self._left._parent = weakref.ref(self)

    @property
    def right(self):
        return self._right

    @right.setter
    def right(self, val):
        self._right = val
        if not self._right is None:
            self._right._parent = weakref.ref(self)

    def get_key(self):
        """Returns the key value of the binary tree"""
        return self.key

    def __str__(self):
        return '{} : {}'.format(self.key, self.item)

    def is_leaf(self):
        return (self.left is None) and (self.right is None)

    def get_heigth(self):
        """recursive get Heigth method"""
        hl = self.left.getHeigth()
        hr = self.right.getHeigth()
        #returns the highest heigth
        if (hl < hr):
            # + 1 --> counting self
            return hl + 1
        else:
            # + 1 --> counting self
            return hr + 1

    def in_order(self):
        '''recursive in-order traversal.
        Returns a list.
        '''
        l = [self]
        if self.left:
            l = self.left.in_order() + l
        if self.right:
            l += self.right.in_order()
        return l

    def pre_order(self):
        '''recursive pre-order traversal.
        Returns a list.
        '''
        l = [self]
        if self.left:
            l += self.left.pre_order()
        if self.right:
            l += self.right.pre_order()
        return l

    def post_order(self):
        """recursive post-order traversal.
        Returns a list."""
        l = []
        if self.left:
            l = self.left.post_order()
        if self.right:
            l = l + self.right.post_order()
        l = l + [self]
        return l

    def breadth_first_search(self):
        """iterative Breadth First Search.
        Returns a list."""
        l = []
        queue = Queue()
        #enqueue itself
        queue.enqueue(self)
        while not queue.is_empty() :
            #dequeue and gets acess to top of queue
            p = queue.dequeue()
            #append to output list
            l.append(p)
            #gets right child, if it's not empty, enqueue
            child_right = p.right
            if child_right:
                queue.enqueue(child_right)
            #gets right child, if it's not empty, enqueue
            child_left = p.left
            if child_left:
                queue.enqueue(child_left)
        return l

    def in_order_iterative(self):
        '''iterative in order method. returns a list'''
        stack = Stack()
        tree = self
        l = []
        while tree or not stack.is_empty():
            if tree:
                stack.push(tree)
                tree = tree.left
            else:
                tree = stack.pop()
                l.append(tree)
                tree = tree.right
        return l

    def pre_order_iterative(self):
        """Iterative preoder method"""
        stack = Stack()
        tree = self
        queue = Queue()
        stack.push(tree)
        l = []
        while not stack.is_empty():
            tree = stack.pop()
            queue.enqueue(tree)
            if tree.right:
                stack.push(tree.right)
            if tree.left:
                stack.push(tree.left)
        while not queue.is_empty():
            l.append(queue.dequeue())
        return l

    def post_order_iterative(self):
        """Iterative post order method
        it's the same method of the preorder
        iterative, but reversed, using an
        stack in the place of the queue"""
        stack = Stack()
        output = Stack()
        tree = self
        stack.push(tree)
        l = []
        while not stack.is_empty():
            tree = stack.pop()
            output.push(tree)
            if tree.left:
                stack.push(tree.left)
            if tree.right:
                stack.push(tree.right)
        while not output.is_empty():
            l.append(output.pop())
        return l

class BinarySearchTree(BinaryTree):

    def search(self, key):
        tree = self
        while tree:
            if key == tree.get_key():
                return tree
            elif key < tree.get_key():
                tree = tree.left
            else:
                tree = tree.right

    def insert(self, key, item):
        tree = self
        aux = tree
        while tree:
            aux = tree
            if key < tree.get_key():
                tree = tree.left
            else:
                tree = tree.right
        newtree = BinarySearchTree(key,item)
        if key <  aux.get_key():
            aux.left = newtree
        else:
            aux.right = newtree

    def remove_root(self):
        #The tree only have a left child
        if self.right is None and not self.left is None:
            copy_node(self.left, self)
        #The tree only have a right child
        if self.left is None and not self.right is None:
            copy_node(self.right, self)
        #The tree haves the two children
        if not self.left is None and not self.right is None:
            node = self.left
            while node:
                tree = node
                node = node.right
            old_left = self.left
            old_right = self.right
            parent = tree.parent
            copy_node(tree, self)
            remove_node(tree)
            self.left = old_left
            self.right = old_right

def remove_node(node):
    parent = node.parent
    if parent:
        if parent.right == node:
            parent.right = None
        else:
            parent.left = None
    else:
        self.key = None
        self.item = None

def remove_search_tree(tree, key):
    node = tree.search(key)
    if not node.right and not node.left:
        remove_node(node)
    else:
        node.remove_root()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin

from utils import utils, inspector

# http://www.si.edu/OIG
archive = 2003

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
#

# The general strategy for this report is a bit complicated. First we go through
# the RSS feed and add reports. This gives us a bunch of reports with definitive
# published dates. Unfortunately, this feed is only updated sporadically.
# Next, we use the main audits page since this also gives us accurate published
# dates for the newer reports that are listed. Lastly, we fall back to the
# archives page, which unfortunately only lists the year the report was
# published.
#
# Since a report can be listed on multiple pages, it is important to use
# consistent report ids across pages. We only grab reports the first time a
# a given report id is seen.

RSS_URL = "http://www.si.edu/Content/OIG/Misc/OIG-RSS.xml"
RECENT_AUDITS_URL = "http://www.si.edu/OIG/Audits"
AUDIT_ARCHIVE_URL = "http://www.si.edu/oig/Archive"
OTHER_REPORTS_URl = "http://www.si.edu/OIG/ReportsToCongress"

RSS_BROKEN_LINKS = {
  "http://www.si.edu/Content/OIG/Misc/Peer_Review_09-21-2011.pdf":
    "http://www.si.edu/Content/OIG/Misc/Peer_Review_09-21-11.pdf",
  "http://www.si.edu/oig/RecoveryAct.htm":
    "http://www.si.edu/OIG/Recovery",
  "http://www.si.edu/oig/AuditReports/UnderstandingAudits.pdf":
    "http://www.si.edu/Content/OIG/Misc/UnderstandingAudits.pdf",
  "http://www.si.edu/oig/AuditReports/A-0907-FSA-Oversight.pdf":
    "http://www.si.edu/Content/OIG/Audits/2010/A-09-07.pdf",
  "http://www.si.edu/oig/ARRA_Reports/M-10--04-1.pdf":
    "http://www.si.edu/Content/OIG/Audits/M-10-04-1.pdf",
  "http://www.si.edu/oig/AuditReports/SIIG_Testimony_121009.pdf":
    "http://www.si.edu/Content/OIG/Testimony/SIIG_Testimony_121009.pdf",
  "http://www.si.edu/oig/AuditReports/IBA-0902.pdf":
    "http://www.si.edu/Content/OIG/Audits/2009/IBA-09-02.pdf",
  "http://www.si.edu/oig/AuditReports/IBA-0808.pdf":
    "http://www.si.edu/Content/OIG/Audits/2009/IBA-08-08.pdf",
  "http://www.si.edu/oig/AuditReports/A-08-05-FSA-Oversight-Letter.pdf":
    "http://www.si.edu/Content/OIG/Audits/2009/A-08-05.pdf",
}

report_ids_seen = set()

def run(options):
  year_range = inspector.year_range(options, archive)

  # # Pull the RSS feed
  doc = utils.beautifulsoup_from_url(RSS_URL)
  results = doc.select("item")
  if not results:
    raise inspector.NoReportsFoundError("Smithsonian (RSS)")
  for result in results:
    report = rss_report_from(result, year_range)
    if report:
      inspector.save_report(report)

  # # Pull the recent audit reports.
  doc = utils.beautifulsoup_from_url(RECENT_AUDITS_URL)
  results = doc.select("div.block > a")
  if not results:
    raise inspector.NoReportsFoundError("Smithsonian (recent audit reports)")
  for result in results:
    report = report_from(result, year_range)
    if report:
      inspector.save_report(report)

  # Pull the archive audit reports
  doc = utils.beautifulsoup_from_url(AUDIT_ARCHIVE_URL)
  results = doc.select("div.block a")
  if not results:
    raise inspector.NoReportsFoundError("Smithsonian (audit archive)")
  for result in results:
    report = report_from(result, year_range)
    if report:
      inspector.save_report(report)

  # Pull the other reports
  doc = utils.beautifulsoup_from_url(OTHER_REPORTS_URl)
  results = doc.select("div.block > a")
  if not results:
    raise inspector.NoReportsFoundError("Smithsonian (other)")
  for result in results:
    report = report_from(result, year_range)
    if report:
      inspector.save_report(report)

def report_type_from_url(report_url):
  if 'Audit' in report_url or 'Announcements' in report_url:
    return 'audit'
  elif 'Semiannual' in report_url:
    return 'semiannual_report'
  elif 'Testimony' in report_url:
    return 'testimony'
  elif 'Peer_Review' in report_url:
    return 'peer_review'
  elif 'PressRelease' in report_url:
    return 'press'
  else:
    return 'other'

def rss_report_from(result, year_range):
  report_url = result.find("link").next_sibling.strip()
  if report_url.rstrip("/") == 'http://www.si.edu/oig':
    # This is the default url the IG uses for announcements of things like
    # a website redesign or changes to the RSS feed.
    return

  if report_url == "http://www.si.edu/oig/OIGStratPlan.pdf":
    # This strategic plan is no longer on the website, but it is reproduced in
    # multiple semiannual reports, so we skip it here.
    return

  if report_url in RSS_BROKEN_LINKS:
    report_url = RSS_BROKEN_LINKS[report_url]
  else:
    report_url = report_url.replace("/OIG/SAR/Semiannual_Reports/", "/OIG/SAR/")
    report_url = report_url.replace("/oig/Semiannual_Reports/", "/Content/OIG/SAR/")
    report_url = report_url.replace("/oig/AuditReports/", "/Content/OIG/Audits/")
    report_url = report_url.replace("/oig/ARRA_Reports/", "/Content/OIG/Audits/")

  file_type = None
  if not report_url.endswith(".pdf"):
    file_type = "html"

  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)

  if report_id in report_ids_seen:
    return
  report_ids_seen.add(report_id)

  title = result.find("title").text
  report_type = report_type_from_url(report_url)
  published_on_text = result.find("pubdate").text
  try:
    published_on = datetime.datetime.strptime(published_on_text, '%a, %d %b %Y %H:%M:%S %z').date()
  except ValueError:
    published_on = datetime.datetime.strptime(published_on_text, '%a, %d %B %Y %H:%M:%S %z').date()

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'smithsonian',
    'inspector_url': 'http://www.si.edu/OIG',
    'agency': 'smithsonian',
    'agency_name': 'Smithsonian Institution',
    'type': report_type,
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  if file_type:
    report['file_type'] = file_type
  return report

def report_from(result, year_range):
  report_url = urljoin(RECENT_AUDITS_URL, result.get('href'))
  # Strip extra path adjustments
  report_url = report_url.replace("../", "")

  summary = None
  if not report_url.endswith(".pdf"):
    # Some reports link to other page which link to the full report
    report_page = utils.beautifulsoup_from_url(report_url)
    relative_report_url = report_page.select("div.block a")[0].get('href')
    report_url = urljoin(report_url, relative_report_url)
    # Strip extra path adjustments
    report_url = report_url.replace("../", "")

    summary = "\n".join(paragraph.text for paragraph in report_page.select("div.grid_12 p"))

  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)

  if report_id in report_ids_seen:
    return
  report_ids_seen.add(report_id)

  title = result.text
  report_type = report_type_from_url(report_url)

  estimated_date = False
  published_on = None
  try:
    published_on_text = "/".join(re.search('(\w+) (\d+), (\d+)', title).groups())
    published_on = datetime.datetime.strptime(published_on_text, '%B/%d/%Y')
  except AttributeError:
    pass

  if not published_on:
    month_year_match = MONTH_YEAR_RE.search(result.text)
    if month_year_match:
      date_text = ' '.join(month_year_match.group(0).split())
      published_on = datetime.datetime.strptime(date_text, '%B %Y')
      estimated_date = True

  if not published_on:
    try:
      # For reports where we can only find the year, set them to Nov 1st of
      # that year
      published_on_year = int(result.find_previous("h2").text)
      published_on = datetime.datetime(published_on_year, 11, 1)
      estimated_date = True
    except AttributeError:
      pass

  if not published_on:
    result_text = str(result.text)
    if result_text.find('CNCS') != -1 and \
                result_text.find('peer review report') != -1:
      published_on = datetime.datetime(2013, 2, 27)

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'smithsonian',
    'inspector_url': 'http://www.si.edu/OIG',
    'agency': 'smithsonian',
    'agency_name': 'Smithsonian Institution',
    'type': report_type,
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  if summary:
    report['summary'] = summary
  if estimated_date:
    report['estimated_date'] = estimated_date
  return report

MONTH_YEAR_RE = re.compile('(?:January|February|March|April|May|June|July|' \
                           'August|September|October|November|December)\s+' \
                           '[0-9]{4}')

utils.run(run) if (__name__ == "__main__") else None
<commit_msg>[smithsonian] Upgrade to HTTPS<commit_after>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin

from utils import utils, inspector

# https://www.si.edu/OIG
archive = 2003

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
#

# The general strategy for this report is a bit complicated. First we go through
# the RSS feed and add reports. This gives us a bunch of reports with definitive
# published dates. Unfortunately, this feed is only updated sporadically.
# Next, we use the main audits page since this also gives us accurate published
# dates for the newer reports that are listed. Lastly, we fall back to the
# archives page, which unfortunately only lists the year the report was
# published.
#
# Since a report can be listed on multiple pages, it is important to use
# consistent report ids across pages. We only grab reports the first time a
# a given report id is seen.

RSS_URL = "https://www.si.edu/Content/OIG/Misc/OIG-RSS.xml"
RECENT_AUDITS_URL = "https://www.si.edu/OIG/Audits"
AUDIT_ARCHIVE_URL = "https://www.si.edu/oig/Archive"
OTHER_REPORTS_URl = "https://www.si.edu/OIG/ReportsToCongress"

RSS_BROKEN_LINKS = {
  "http://www.si.edu/Content/OIG/Misc/Peer_Review_09-21-2011.pdf":
    "https://www.si.edu/Content/OIG/Misc/Peer_Review_09-21-11.pdf",
  "http://www.si.edu/oig/RecoveryAct.htm":
    "https://www.si.edu/OIG/Recovery",
  "http://www.si.edu/oig/AuditReports/UnderstandingAudits.pdf":
    "https://www.si.edu/Content/OIG/Misc/UnderstandingAudits.pdf",
  "http://www.si.edu/oig/AuditReports/A-0907-FSA-Oversight.pdf":
    "https://www.si.edu/Content/OIG/Audits/2010/A-09-07.pdf",
  "http://www.si.edu/oig/ARRA_Reports/M-10--04-1.pdf":
    "https://www.si.edu/Content/OIG/Audits/M-10-04-1.pdf",
  "http://www.si.edu/oig/AuditReports/SIIG_Testimony_121009.pdf":
    "https://www.si.edu/Content/OIG/Testimony/SIIG_Testimony_121009.pdf",
  "http://www.si.edu/oig/AuditReports/IBA-0902.pdf":
    "https://www.si.edu/Content/OIG/Audits/2009/IBA-09-02.pdf",
  "http://www.si.edu/oig/AuditReports/IBA-0808.pdf":
    "https://www.si.edu/Content/OIG/Audits/2009/IBA-08-08.pdf",
  "http://www.si.edu/oig/AuditReports/A-08-05-FSA-Oversight-Letter.pdf":
    "https://www.si.edu/Content/OIG/Audits/2009/A-08-05.pdf",
}

report_ids_seen = set()

def run(options):
  year_range = inspector.year_range(options, archive)

  # # Pull the RSS feed
  doc = utils.beautifulsoup_from_url(RSS_URL)
  results = doc.select("item")
  if not results:
    raise inspector.NoReportsFoundError("Smithsonian (RSS)")
  for result in results:
    report = rss_report_from(result, year_range)
    if report:
      inspector.save_report(report)

  # # Pull the recent audit reports.
  doc = utils.beautifulsoup_from_url(RECENT_AUDITS_URL)
  results = doc.select("div.block > a")
  if not results:
    raise inspector.NoReportsFoundError("Smithsonian (recent audit reports)")
  for result in results:
    report = report_from(result, year_range)
    if report:
      inspector.save_report(report)

  # Pull the archive audit reports
  doc = utils.beautifulsoup_from_url(AUDIT_ARCHIVE_URL)
  results = doc.select("div.block a")
  if not results:
    raise inspector.NoReportsFoundError("Smithsonian (audit archive)")
  for result in results:
    report = report_from(result, year_range)
    if report:
      inspector.save_report(report)

  # Pull the other reports
  doc = utils.beautifulsoup_from_url(OTHER_REPORTS_URl)
  results = doc.select("div.block > a")
  if not results:
    raise inspector.NoReportsFoundError("Smithsonian (other)")
  for result in results:
    report = report_from(result, year_range)
    if report:
      inspector.save_report(report)

def report_type_from_url(report_url):
  if 'Audit' in report_url or 'Announcements' in report_url:
    return 'audit'
  elif 'Semiannual' in report_url:
    return 'semiannual_report'
  elif 'Testimony' in report_url:
    return 'testimony'
  elif 'Peer_Review' in report_url:
    return 'peer_review'
  elif 'PressRelease' in report_url:
    return 'press'
  else:
    return 'other'

def rss_report_from(result, year_range):
  report_url = result.find("link").next_sibling.strip()
  if report_url.rstrip("/") == 'http://www.si.edu/oig':
    # This is the default url the IG uses for announcements of things like
    # a website redesign or changes to the RSS feed.
    return

  if report_url == "http://www.si.edu/oig/OIGStratPlan.pdf":
    # This strategic plan is no longer on the website, but it is reproduced in
    # multiple semiannual reports, so we skip it here.
    return

  if report_url in RSS_BROKEN_LINKS:
    report_url = RSS_BROKEN_LINKS[report_url]
  else:
    report_url = report_url.replace("/OIG/SAR/Semiannual_Reports/", "/OIG/SAR/")
    report_url = report_url.replace("/oig/Semiannual_Reports/", "/Content/OIG/SAR/")
    report_url = report_url.replace("/oig/AuditReports/", "/Content/OIG/Audits/")
    report_url = report_url.replace("/oig/ARRA_Reports/", "/Content/OIG/Audits/")

  file_type = None
  if not report_url.endswith(".pdf"):
    file_type = "html"

  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)

  if report_id in report_ids_seen:
    return
  report_ids_seen.add(report_id)

  title = result.find("title").text
  report_type = report_type_from_url(report_url)
  published_on_text = result.find("pubdate").text
  try:
    published_on = datetime.datetime.strptime(published_on_text, '%a, %d %b %Y %H:%M:%S %z').date()
  except ValueError:
    published_on = datetime.datetime.strptime(published_on_text, '%a, %d %B %Y %H:%M:%S %z').date()

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'smithsonian',
    'inspector_url': 'https://www.si.edu/OIG',
    'agency': 'smithsonian',
    'agency_name': 'Smithsonian Institution',
    'type': report_type,
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  if file_type:
    report['file_type'] = file_type
  return report

def report_from(result, year_range):
  report_url = urljoin(RECENT_AUDITS_URL, result.get('href'))
  # Strip extra path adjustments
  report_url = report_url.replace("../", "")

  summary = None
  if not report_url.endswith(".pdf"):
    # Some reports link to other page which link to the full report
    report_page = utils.beautifulsoup_from_url(report_url)
    relative_report_url = report_page.select("div.block a")[0].get('href')
    report_url = urljoin(report_url, relative_report_url)
    # Strip extra path adjustments
    report_url = report_url.replace("../", "")

    summary = "\n".join(paragraph.text for paragraph in report_page.select("div.grid_12 p"))

  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)

  if report_id in report_ids_seen:
    return
  report_ids_seen.add(report_id)

  title = result.text
  report_type = report_type_from_url(report_url)

  estimated_date = False
  published_on = None
  try:
    published_on_text = "/".join(re.search('(\w+) (\d+), (\d+)', title).groups())
    published_on = datetime.datetime.strptime(published_on_text, '%B/%d/%Y')
  except AttributeError:
    pass

  if not published_on:
    month_year_match = MONTH_YEAR_RE.search(result.text)
    if month_year_match:
      date_text = ' '.join(month_year_match.group(0).split())
      published_on = datetime.datetime.strptime(date_text, '%B %Y')
      estimated_date = True

  if not published_on:
    try:
      # For reports where we can only find the year, set them to Nov 1st of
      # that year
      published_on_year = int(result.find_previous("h2").text)
      published_on = datetime.datetime(published_on_year, 11, 1)
      estimated_date = True
    except AttributeError:
      pass

  if not published_on:
    result_text = str(result.text)
    if result_text.find('CNCS') != -1 and \
                result_text.find('peer review report') != -1:
      published_on = datetime.datetime(2013, 2, 27)

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'smithsonian',
    'inspector_url': 'https://www.si.edu/OIG',
    'agency': 'smithsonian',
    'agency_name': 'Smithsonian Institution',
    'type': report_type,
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  if summary:
    report['summary'] = summary
  if estimated_date:
    report['estimated_date'] = estimated_date
  return report

MONTH_YEAR_RE = re.compile('(?:January|February|March|April|May|June|July|' \
                           'August|September|October|November|December)\s+' \
                           '[0-9]{4}')

utils.run(run) if (__name__ == "__main__") else None
<|endoftext|>


<commit_before>from django.db import models
import django.contrib.auth.models as auth

class User(auth.User):
    """Model for representing users.

    It has few fields that aren't in the standard authentication user
    table, and are needed for the forum to work, like footers.
    """
    display_name = models.CharField(max_length=30, null=True)
    footer = models.TextField(null=True)

class Thread(models.Model):
    """Model for representing threads."""
    title = models.CharField(max_length=100)
    views = models.PositiveIntegerField(default=0)
    sticky = models.BooleanField()
    closed = models.BooleanField()
<commit_msg>Add model for posts.<commit_after>from django.db import models
import django.contrib.auth.models as auth

class User(auth.User):
    """Model for representing users.

    It has few fields that aren't in the standard authentication user
    table, and are needed for the forum to work, like footers.
    """
    display_name = models.CharField(max_length=30, null=True)
    footer = models.TextField(null=True)

class Thread(models.Model):
    """Model for representing threads."""
    title = models.CharField(max_length=100)
    views = models.PositiveIntegerField(default=0)
    sticky = models.BooleanField()
    closed = models.BooleanField()

class Post(models.Model):
    thread = models.ForeignKey(Thread)
<|endoftext|>


<commit_before>from django.http import HttpResponse
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import IsAuthenticatedOrReadOnly
from rest_framework.response import Response
from rest_framework import status
from rest_framework.parsers import JSONParser
from .models import Post
from .serializers import PostSerializer

@api_view(['GET', 'POST'])
@permission_classes((IsAuthenticatedOrReadOnly,))
def topic_list(request):
    if request.method == 'GET':
        topics = Post.objects.all().filter(is_topic=True)
        serializer = PostSerializer(topics, many=True)
        return Response(serializer.data)

    elif request.method == 'POST':
        data = JSONParser().parse(request)
        data["is_topic"] = True
        data["author_id"] = request.user.id
        serializer = PostSerializer(data=data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors,
                        status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET', 'PUT', 'DELETE'])
@permission_classes((IsAuthenticatedOrReadOnly,))
def topic_detail(request, pk):
    try:
        topic = Post.objects.get(pk=pk, is_topic=True)
    except Post.DoesNotExist:
        return Response(status=status.HTTP_404_NOT_FOUND)

    if request.method == 'GET':
        serializer = PostSerializer(topic)
        # Todo: Add replies
        return Response(serializer.data)

    elif request.method == 'PUT':
        # Could be made better, but it'll do.
        request.data["is_topic"] = True
        request.data["author_id"] = request.user.id
        serializer = PostSerializer(topic, data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        topic.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

<commit_msg>Added setting of edit time when PUT is used.<commit_after>from django.http import HttpResponse
from django.utils import timezone

from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import IsAuthenticatedOrReadOnly
from rest_framework.response import Response
from rest_framework import status
from rest_framework.parsers import JSONParser

from .models import Post
from .serializers import PostSerializer

@api_view(['GET', 'POST'])
@permission_classes((IsAuthenticatedOrReadOnly,))
def topic_list(request):
    if request.method == 'GET':
        topics = Post.objects.all().filter(is_topic=True)
        serializer = PostSerializer(topics, many=True)
        return Response(serializer.data)

    elif request.method == 'POST':
        data = JSONParser().parse(request)
        data["is_topic"] = True
        data["author_id"] = request.user.id
        serializer = PostSerializer(data=data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors,
                        status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET', 'PUT', 'DELETE'])
@permission_classes((IsAuthenticatedOrReadOnly,))
def topic_detail(request, pk):
    try:
        topic = Post.objects.get(pk=pk, is_topic=True)
    except Post.DoesNotExist:
        return Response(status=status.HTTP_404_NOT_FOUND)

    if request.method == 'GET':
        serializer = PostSerializer(topic)
        # Todo: Add replies
        return Response(serializer.data)

    elif request.method == 'PUT':
        # Could be made better, but it'll do.
        request.data["is_topic"] = True
        request.data["author_id"] = request.user.id
        request.data["edit_date"] = timezone.now
        serializer = PostSerializer(topic, data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        topic.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

<|endoftext|>


<commit_before>from time import time
from threading import Lock

from satella.instrumentation.basecounter import InstrumentationCounter
from satella.instrumentation.exceptions import CounterExists, NoData
from satella.instrumentation.snapshots import CountersSnapshot
from satella.threads import Monitor

class InstrumentationManager(Monitor):
    """
    Class used to manage instrumentation counters.

    There can exist many L{RootInstrumentation} classes, each with own set of instrumentation
    counters. You could use it if you have multiple output targets, or incredibly large
    set of counters that are specific to a given task. Still, you could prefix instrumentation
    counters with a string to differ them, but that task could be tedious and make your
    app's instrumentation buggy.

    Not threadsafe.
    """
    def __init__(self, namespace):
        Monitor.__init__(self)
        self.counters = {}  #: dict(name => InstrumentationCounter)
        self.namespace = namespace

    @Monitor.protect
    def add_counter(self, counter):
        """
        @param counter: A counter to register for this instrumentation manager
        @type counter: descendant of L{InstrumentationCounter}
        """
        if counter.name in self.counters:
            raise CounterExists, 'Counter already exists'

        self.counters[counter.name] = counter
        counter._on_added(self)

    @Monitor.protect
    def get_snapshot(self):
        """Snapshots current counter state.

        @return: L{satella.instrumentation.snapshots.CountersSnapshot}
        """
        cdata = {}

        for name, counter in self.counters.iteritems():
            try:
                cdata[name] = counter.get_current()
            except NoData:
                cdata[name] = NoData

        return CountersSnapshot(cdata, time(), self)<commit_msg>docstring for InstrumentationManager fixed<commit_after>from time import time
from threading import Lock

from satella.instrumentation.basecounter import InstrumentationCounter
from satella.instrumentation.exceptions import CounterExists, NoData
from satella.instrumentation.snapshots import CountersSnapshot
from satella.threads import Monitor

class InstrumentationManager(Monitor):
    """
    Class used to manage instrumentation counters.

    There can exist many L{InstrumentationManager} classes, each with own set
    of instrumentation counters - with different namespace names. You could use
    it if you have multiple output targets, or incredibly large set of counters
    that are specific to a given task. Still, you could prefix instrumentation
    counters with a string to differ them, but that task could be tedious and
    make your app's instrumentation buggy.

    This class is threadsafe.
    """
    def __init__(self, namespace):
        Monitor.__init__(self)
        self.counters = {}  #: dict(name => InstrumentationCounter)
        self.namespace = namespace

    @Monitor.protect
    def add_counter(self, counter):
        """
        @param counter: A counter to register for this instrumentation manager
        @type counter: descendant of L{InstrumentationCounter}
        """
        if counter.name in self.counters:
            raise CounterExists, 'Counter already exists'

        self.counters[counter.name] = counter
        counter._on_added(self)

    @Monitor.protect
    def get_snapshot(self):
        """Snapshots current counter state.

        @return: L{satella.instrumentation.snapshots.CountersSnapshot}
        """
        cdata = {}

        for name, counter in self.counters.iteritems():
            try:
                cdata[name] = counter.get_current()
            except NoData:
                cdata[name] = NoData

        return CountersSnapshot(cdata, time(), self)<|endoftext|>


<commit_before>#!/usr/bin/env python

def ath(array):
    hash = {}
    for s in array:
        hash[s] = True
    return hash

knownHash = ath([

# Bug 467008, bug 467647
"nsDocument",

# bug 391976
"nsMathMLContainerFrame",
"nsMathMLmtableOuterFrame",
"nsMathMLmtdInnerFrame",
"nsMathMLmactionFrame",

# Bug 398462
"nsBaseAppShell",
"nsRunnable",

# Bug 403199
"nsSimpleNestedURI"

])

# Things that are known to leak AND entrain smaller objects.
# If one of these leaks, leaks of small objects will not be reported.
knownLargeHash = ath([

# Bug 397206
"BackstagePass",

# Bug 102229 or bug 419562
"nsDNSService",

# Bug 424418
"nsRDFResource",

# Bug 417630 and friends
"nsJVMManager"
])

# Large items that
# - should be reported even if things in knownLargeHash leak
# - should quell the reporting of smaller objects
otherLargeHash = ath([
"nsGlobalWindow",
"nsDocShell"
])

def amiss(logPrefix):
    currentFile = file(logPrefix + "-out", "r")
    sawLeakStats = False

    for line in currentFile:
        line = line.rstrip("\n")
        if (line == "== BloatView: ALL (cumulative) LEAK STATISTICS"):
            sawLeakStats = True
        # This line appears only if there are leaks
        if (line.endswith("Mean       StdDev")):
            break
    else:
        if sawLeakStats:
            #print "No leaks :)"
            pass
        else:
            print "Didn't see leak stats"
        currentFile.close()
        return False

    smallLeaks = ""
    largeKnownLeaks = ""
    largeOtherLeaks = ""

    for line in currentFile:
        line = line.strip("\x07").rstrip("\n").lstrip(" ")
        if (line == ""):
            break
        a = line.split(" ")[1]
        if a == "TOTAL":
            continue
        # print "Leaked at least one: " + a
        if a in knownLargeHash:
            largeKnownLeaks += "*** Leaked large object " + a + " (known)\n"
        if a in otherLargeHash:
            largeOtherLeaks += "*** Leaked large object " + a + "\n"
        if not a in knownHash:
            smallLeaks += a + "\n"

    if largeOtherLeaks != "":
        print "Leaked large objects:"
        print largeOtherLeaks
        # print "Also leaked 'known' large objects:"
        # print largeKnownLeaks
        currentFile.close()
        return True
    elif largeKnownLeaks != "":
        # print "(Known large leaks, and no other large leaks, so all leaks were ignored)"
        # print largeKnownLeaks
        currentFile.close()
        return False
    elif smallLeaks != "":
        print "Leaked:"
        print smallLeaks
        currentFile.close()
        return True
    else:
        # print "(Only known small leaks)"
        currentFile.close()
        return False

# print "detect_leaks is ready"
<commit_msg>Add bug 467648 (nsHTMLDNSPrefetch leak) to known leak list<commit_after>#!/usr/bin/env python

def ath(array):
    hash = {}
    for s in array:
        hash[s] = True
    return hash

knownHash = ath([

# Bug 467008, bug 467647
"nsDocument",

# bug 391976
"nsMathMLContainerFrame",
"nsMathMLmtableOuterFrame",
"nsMathMLmtdInnerFrame",
"nsMathMLmactionFrame",

# Bug 398462
"nsBaseAppShell",
"nsRunnable",

# Bug 403199
"nsSimpleNestedURI"

])

# Things that are known to leak AND entrain smaller objects.
# If one of these leaks, leaks of small objects will not be reported.
knownLargeHash = ath([

# Bug 397206
"BackstagePass",

# Bug 102229 or bug 419562
"nsDNSService",

# Bug 467648
"nsHTMLDNSPrefetch",

# Bug 424418
"nsRDFResource",

# Bug 417630 and friends
"nsJVMManager"
])

# Large items that
# - should be reported even if things in knownLargeHash leak
# - should quell the reporting of smaller objects
otherLargeHash = ath([
"nsGlobalWindow",
"nsDocShell"
])

def amiss(logPrefix):
    currentFile = file(logPrefix + "-out", "r")
    sawLeakStats = False

    for line in currentFile:
        line = line.rstrip("\n")
        if (line == "== BloatView: ALL (cumulative) LEAK STATISTICS"):
            sawLeakStats = True
        # This line appears only if there are leaks
        if (line.endswith("Mean       StdDev")):
            break
    else:
        if sawLeakStats:
            #print "No leaks :)"
            pass
        else:
            print "Didn't see leak stats"
        currentFile.close()
        return False

    smallLeaks = ""
    largeKnownLeaks = ""
    largeOtherLeaks = ""

    for line in currentFile:
        line = line.strip("\x07").rstrip("\n").lstrip(" ")
        if (line == ""):
            break
        a = line.split(" ")[1]
        if a == "TOTAL":
            continue
        # print "Leaked at least one: " + a
        if a in knownLargeHash:
            largeKnownLeaks += "*** Leaked large object " + a + " (known)\n"
        if a in otherLargeHash:
            largeOtherLeaks += "*** Leaked large object " + a + "\n"
        if not a in knownHash:
            smallLeaks += a + "\n"

    if largeOtherLeaks != "":
        print "Leaked large objects:"
        print largeOtherLeaks
        # print "Also leaked 'known' large objects:"
        # print largeKnownLeaks
        currentFile.close()
        return True
    elif largeKnownLeaks != "":
        # print "(Known large leaks, and no other large leaks, so all leaks were ignored)"
        # print largeKnownLeaks
        currentFile.close()
        return False
    elif smallLeaks != "":
        print "Leaked:"
        print smallLeaks
        currentFile.close()
        return True
    else:
        # print "(Only known small leaks)"
        currentFile.close()
        return False

# print "detect_leaks is ready"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import json
import logging
import couchdb
import datetime
from openprocurement.auction.databridge import ResourceFeeder
import pytest
from gevent import spawn
from openprocurement.auction import core as core_module
from openprocurement.auction.chronograph import AuctionsChronograph
from openprocurement.auction.databridge import AuctionsDataBridge
from openprocurement.auction.helpers.chronograph import \
    MIN_AUCTION_START_TIME_RESERV
from openprocurement.auction.tests.utils import get_tenders_dummy
from openprocurement.auction.worker.auction import Auction
from openprocurement.auction.tests.utils import update_auctionPeriod, \
    AUCTION_DATA
from openprocurement.auction.tests.utils import worker_defaults, \
    test_chronograph_config, worker_defaults_file_path, test_bridge_config
import yaml
import openprocurement.auction.helpers.couch as couch_module
import openprocurement.auction.chronograph as chrono_module
from openprocurement.auction.tests.utils import DummyTrue, iterview_wrappper
import pytest
from webtest import TestApp
from openprocurement.auction.auctions_server import auctions_server as frontend
import openprocurement.auction.auctions_server as auctions_server_module
from mock import MagicMock, NonCallableMock
from couchdb import Server
from memoize import Memoizer
from mock import sentinel

DEFAULT = sentinel.DEFAULT

LOGGER = logging.getLogger('Log For Tests')

test_log_config = {
     'version': 1,
     'disable_existing_loggers': False,
     'formatters': {'simpleFormatter': {'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s'}},
     'handlers': {'journal': {'class': 'ExtendedJournalHandler.ExtendedJournalHandler', 'formatter': 'simpleFormatter', 'SYSLOG_IDENTIFIER': 'AUCTIONS_LOG_FOR_TESTS', 'level': 'DEBUG'}},
     'loggers': {'Log For Tests': {'handlers': ['journal'], 'propagate': False, 'level': 'DEBUG'},
                 '': {'handlers': ['journal'], 'propagate': False, 'level': 'DEBUG'}}
     }

logging.config.dictConfig(test_log_config)

@pytest.fixture(scope='function')
def db(request):
    server = couchdb.Server("http://" + worker_defaults['COUCH_DATABASE'].split('/')[2])
    name = worker_defaults['COUCH_DATABASE'].split('/')[3]

    documents = getattr(request, 'param', None)

    def delete():
        del server[name]

    if name in server:
        delete()

    data_base = server.create(name)

    if documents:
        for doc in documents:
            data_base.save(doc)

    request.addfinalizer(delete)

    return data_base

@pytest.fixture(scope='function')
def chronograph(request, mocker):
    logging.config.dictConfig(test_chronograph_config)

    # We use 'dummy_true' variable instead of real True and mock iterview
    # with iterview_wrapper function to tear down the test gracefully.
    # Without these steps iterview from previous test running continue working
    # while next test have already been launched.
    dummy_true = DummyTrue()
    couch_module.TRUE = dummy_true
    mocker.patch.object(chrono_module, 'iterview',
                        side_effect=iterview_wrappper, autospec=True)

    chrono = AuctionsChronograph(test_chronograph_config)
    chrono_thread = spawn(chrono.run)

    def delete_chronograph():
        chrono.scheduler.execution_stopped = True
        dummy_true.ind = False
        chrono_thread.join(0.15)
        chrono.scheduler.shutdown(True, True)

    request.addfinalizer(delete_chronograph)

    return chrono_thread

@pytest.fixture(scope="function")
def auction(request):
    defaults = {'time': MIN_AUCTION_START_TIME_RESERV,
                'delta_t': datetime.timedelta(seconds=10)}

    params = getattr(request, 'param', defaults)
    for key in defaults.keys():
        params[key] = defaults[key] if params.get(key, 'default') == 'default'\
            else params[key]

    with update_auctionPeriod(
            AUCTION_DATA['simple']['path'],
            auction_type='simple',
            time_shift=params['time']+params['delta_t']) \
            as updated_doc, open(updated_doc, 'r') as auction_updated_data:
        auction_inst = Auction(
            tender_id=AUCTION_DATA['simple']['data']['data']['tenderID'],
            worker_defaults=yaml.load(open(worker_defaults_file_path)),
            auction_data=json.load(auction_updated_data),
            lot_id=False)

    return auction_inst

@pytest.fixture(scope='function')
def bridge(request, mocker):
    params = getattr(request, 'param', {})
    tenders = params.get('tenders', [])
    bridge_config = params.get('bridge_config', test_bridge_config)

    mock_resource_items = \
        mocker.patch.object(ResourceFeeder, 'get_resource_items',
                            side_effect=get_tenders_dummy(tenders),
                            autospec=True)

    mock_do_until_success = \
        mocker.patch.object(core_module, 'do_until_success', autospec=True)

    bridge_inst = AuctionsDataBridge(bridge_config)
    thread = spawn(bridge_inst.run)

    return {'bridge': bridge_inst,
            'bridge_thread': thread,
            'bridge_config': bridge_config,
            'tenders': tenders,
            'mock_resource_items': mock_resource_items,
            'mock_do_until_success': mock_do_until_success}

@pytest.fixture(scope='function')
def send(mocker):
    mock_send = mocker.patch.object(auctions_server_module, 'send')
    return mock_send

@pytest.fixture(scope='function')
def response(mocker):
    mock_response = mocker.patch.object(auctions_server_module, 'Response',
                                        return_value='Response Message')
    return mock_response

@pytest.fixture(scope='function')
def mock_auctions_server(request, mocker):
    params = getattr(request, 'param', {})

    server_config_redis = params.get('server_config_redis', DEFAULT)
    connection_limit = params.get('connection_limit', DEFAULT)
    get_mapping = params.get('get_mapping', DEFAULT)
    proxy_path = params.get('proxy_path', DEFAULT)
    event_sources_pool = params.get('event_sources_pool', DEFAULT)
    proxy_connection_pool = params.get('proxy_connection_pool', DEFAULT)
    stream_proxy = params.get('stream_proxy', DEFAULT)
    db = params.get('db', DEFAULT)
    request_headers = params.get('request_headers', [])
    request_url = params.get('request_url', DEFAULT)
    redirect_url = params.get('redirect_url', DEFAULT)
    abort = params.get('abort', DEFAULT)

    class AuctionsServerAttributesContainer(object):
        logger = NotImplemented
        proxy_mappings = NotImplemented
        config = NotImplemented
        event_sources_pool = NotImplemented
        proxy_connection_pool = NotImplemented
        get_mapping = NotImplemented
        db = NotImplemented
        request_headers = NotImplemented

    attr = AuctionsServerAttributesContainer()

    class Config(object):
        __getitem__ = NotImplemented

    attr_conf = Config()

    def config_getitem(item):
        if item == 'REDIS':
            return server_config_redis
        elif item == 'event_source_connection_limit':
            return connection_limit
        else:
            raise KeyError

    mock_path_info = MagicMock()

    def environ_setitem(item, value):
        if item == 'PATH_INFO':
            mock_path_info(value)
            return value
        else:
            raise KeyError

    mocker.patch.object(auctions_server_module, 'get_mapping', get_mapping)

    patch_request = mocker.patch.object(auctions_server_module, 'request')
    patch_request.environ.__setitem__.side_effect = environ_setitem
    patch_request.headers = request_headers
    patch_request.url = request_url

    patch_redirect = mocker.patch.object(auctions_server_module, 'redirect',
                                         return_value=redirect_url)
    patch_abort = mocker.patch.object(auctions_server_module, 'abort',
                                      return_value=abort)

    patch_StreamProxy = \
        mocker.patch.object(auctions_server_module, 'StreamProxy',
                            return_value=stream_proxy)

    auctions_server = NonCallableMock(spec_set=attr)

    logger = MagicMock(spec_set=frontend.logger)
    proxy_mappings = MagicMock(spec_set=Memoizer({}))
    proxy_mappings.get.return_value = proxy_path
    config = MagicMock(spec_set=attr_conf)
    config.__getitem__.side_effect = config_getitem

    auctions_server.logger = logger
    auctions_server.proxy_mappings = proxy_mappings
    auctions_server.config = config
    auctions_server.event_sources_pool = event_sources_pool
    auctions_server.proxy_connection_pool = proxy_connection_pool
    auctions_server.db = db

    mocker.patch.object(auctions_server_module, 'auctions_server',
                        auctions_server)

    return {'server': auctions_server,
            'proxy_mappings': proxy_mappings,
            'mock_path_info': mock_path_info,
            'patch_StreamProxy': patch_StreamProxy,
            'patch_redirect': patch_redirect,
            'patch_abort': patch_abort}

@pytest.fixture(scope='function')
def auctions_server(request):
    params = getattr(request, 'param', {})
    server_config = params.get('server_config', {})

    logger = MagicMock(spec_set=frontend.logger)
    logger.name = server_config.get('logger_name', 'some-logger')
    frontend.logger_name = logger.name
    frontend._logger = logger

    for key in ('limit_replications_func', 'limit_replications_progress'):
        frontend.config.pop(key, None)

    for key in ('limit_replications_func', 'limit_replications_progress'):
        if key in server_config:
            frontend.config[key] = server_config[key]

    frontend.couch_server = MagicMock(spec_set=Server)
    frontend.config['TIMEZONE'] = 'some_time_zone'

    if 'couch_tasks' in params:
        frontend.couch_server.tasks.return_value = params['couch_tasks']

    test_app = TestApp(frontend)
    return {'app': frontend, 'test_app': test_app}

@pytest.yield_fixture(scope="function")
def log_for_test(request):
    LOGGER.debug('-------- Test Start ---------')
    LOGGER.debug('Current module: {0}'.format(request.module.__name__))
    LOGGER.debug('Current test class: {0}'.format(request.cls.__name__))
    LOGGER.debug('Current test function: {0}'
                 .format(request.function.__name__))
    yield LOGGER
    LOGGER.debug('-------- Test End ---------')
<commit_msg>Improve request object patch<commit_after># -*- coding: utf-8 -*-
import json
import logging
import couchdb
import datetime
from openprocurement.auction.databridge import ResourceFeeder
from gevent import spawn
from openprocurement.auction import core as core_module
from openprocurement.auction.chronograph import AuctionsChronograph
from openprocurement.auction.databridge import AuctionsDataBridge
from openprocurement.auction.helpers.chronograph import \
    MIN_AUCTION_START_TIME_RESERV
from openprocurement.auction.tests.utils import get_tenders_dummy
from openprocurement.auction.worker.auction import Auction
from openprocurement.auction.tests.utils import update_auctionPeriod, \
    AUCTION_DATA
from openprocurement.auction.tests.utils import worker_defaults, \
    test_chronograph_config, worker_defaults_file_path, test_bridge_config
import yaml
import openprocurement.auction.helpers.couch as couch_module
import openprocurement.auction.chronograph as chrono_module
from openprocurement.auction.tests.utils import DummyTrue, iterview_wrappper
import pytest
from webtest import TestApp
from openprocurement.auction.auctions_server import auctions_server as frontend
import openprocurement.auction.auctions_server as auctions_server_module
from mock import MagicMock, NonCallableMock
from couchdb import Server
from memoize import Memoizer
from mock import sentinel

DEFAULT = sentinel.DEFAULT

LOGGER = logging.getLogger('Log For Tests')

test_log_config = {
     'version': 1,
     'disable_existing_loggers': False,
     'formatters': {'simpleFormatter': {'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s'}},
     'handlers': {'journal': {'class': 'ExtendedJournalHandler.ExtendedJournalHandler', 'formatter': 'simpleFormatter', 'SYSLOG_IDENTIFIER': 'AUCTIONS_LOG_FOR_TESTS', 'level': 'DEBUG'}},
     'loggers': {'Log For Tests': {'handlers': ['journal'], 'propagate': False, 'level': 'DEBUG'},
                 '': {'handlers': ['journal'], 'propagate': False, 'level': 'DEBUG'}}
     }

logging.config.dictConfig(test_log_config)

@pytest.fixture(scope='function')
def db(request):
    server = couchdb.Server("http://" + worker_defaults['COUCH_DATABASE'].split('/')[2])
    name = worker_defaults['COUCH_DATABASE'].split('/')[3]

    documents = getattr(request, 'param', None)

    def delete():
        del server[name]

    if name in server:
        delete()

    data_base = server.create(name)

    if documents:
        for doc in documents:
            data_base.save(doc)

    request.addfinalizer(delete)

    return data_base

@pytest.fixture(scope='function')
def chronograph(request, mocker):
    logging.config.dictConfig(test_chronograph_config)

    # We use 'dummy_true' variable instead of real True and mock iterview
    # with iterview_wrapper function to tear down the test gracefully.
    # Without these steps iterview from previous test running continue working
    # while next test have already been launched.
    dummy_true = DummyTrue()
    couch_module.TRUE = dummy_true
    mocker.patch.object(chrono_module, 'iterview',
                        side_effect=iterview_wrappper, autospec=True)

    chrono = AuctionsChronograph(test_chronograph_config)
    chrono_thread = spawn(chrono.run)

    def delete_chronograph():
        chrono.scheduler.execution_stopped = True
        dummy_true.ind = False
        chrono_thread.join(0.15)
        chrono.scheduler.shutdown(True, True)

    request.addfinalizer(delete_chronograph)

    return chrono_thread

@pytest.fixture(scope="function")
def auction(request):
    defaults = {'time': MIN_AUCTION_START_TIME_RESERV,
                'delta_t': datetime.timedelta(seconds=10)}

    params = getattr(request, 'param', defaults)
    for key in defaults.keys():
        params[key] = defaults[key] if params.get(key, 'default') == 'default'\
            else params[key]

    with update_auctionPeriod(
            AUCTION_DATA['simple']['path'],
            auction_type='simple',
            time_shift=params['time']+params['delta_t']) \
            as updated_doc, open(updated_doc, 'r') as auction_updated_data:
        auction_inst = Auction(
            tender_id=AUCTION_DATA['simple']['data']['data']['tenderID'],
            worker_defaults=yaml.load(open(worker_defaults_file_path)),
            auction_data=json.load(auction_updated_data),
            lot_id=False)

    return auction_inst

@pytest.fixture(scope='function')
def bridge(request, mocker):
    params = getattr(request, 'param', {})
    tenders = params.get('tenders', [])
    bridge_config = params.get('bridge_config', test_bridge_config)

    mock_resource_items = \
        mocker.patch.object(ResourceFeeder, 'get_resource_items',
                            side_effect=get_tenders_dummy(tenders),
                            autospec=True)

    mock_do_until_success = \
        mocker.patch.object(core_module, 'do_until_success', autospec=True)

    bridge_inst = AuctionsDataBridge(bridge_config)
    thread = spawn(bridge_inst.run)

    return {'bridge': bridge_inst,
            'bridge_thread': thread,
            'bridge_config': bridge_config,
            'tenders': tenders,
            'mock_resource_items': mock_resource_items,
            'mock_do_until_success': mock_do_until_success}

@pytest.fixture(scope='function')
def send(mocker):
    mock_send = mocker.patch.object(auctions_server_module, 'send')
    return mock_send

@pytest.fixture(scope='function')
def response(mocker):
    mock_response = mocker.patch.object(auctions_server_module, 'Response',
                                        return_value='Response Message')
    return mock_response

@pytest.fixture(scope='function')
def mock_auctions_server(request, mocker):
    params = getattr(request, 'param', {})

    server_config_redis = params.get('server_config_redis', DEFAULT)
    connection_limit = params.get('connection_limit', DEFAULT)
    get_mapping = params.get('get_mapping', DEFAULT)
    proxy_path = params.get('proxy_path', DEFAULT)
    event_sources_pool = params.get('event_sources_pool', DEFAULT)
    proxy_connection_pool = params.get('proxy_connection_pool', DEFAULT)
    stream_proxy = params.get('stream_proxy', DEFAULT)
    db = params.get('db', DEFAULT)
    request_headers = params.get('request_headers', [])
    request_url = params.get('request_url', DEFAULT)
    redirect_url = params.get('redirect_url', DEFAULT)
    abort = params.get('abort', DEFAULT)

    class AuctionsServerAttributesContainer(object):
        logger = NotImplemented
        proxy_mappings = NotImplemented
        config = NotImplemented
        event_sources_pool = NotImplemented
        proxy_connection_pool = NotImplemented
        get_mapping = NotImplemented
        db = NotImplemented
        request_headers = NotImplemented

    class Request(object):
        headers = NotImplemented
        environ = NotImplemented
        url = NotImplemented

    class Config(object):
        __getitem__ = NotImplemented

    def config_getitem(item):
        if item == 'REDIS':
            return server_config_redis
        elif item == 'event_source_connection_limit':
            return connection_limit
        else:
            raise KeyError

    mock_path_info = MagicMock()

    def environ_setitem(item, value):
        if item == 'PATH_INFO':
            mock_path_info(value)
            return value
        else:
            raise KeyError

    mocker.patch.object(auctions_server_module, 'get_mapping', get_mapping)

    patch_request = mocker.patch.object(auctions_server_module, 'request',
                                        spec_set=Request)
    patch_request.environ.__setitem__.side_effect = environ_setitem
    patch_request.headers = request_headers
    patch_request.url = request_url

    patch_redirect = mocker.patch.object(auctions_server_module, 'redirect',
                                         return_value=redirect_url)
    patch_abort = mocker.patch.object(auctions_server_module, 'abort',
                                      return_value=abort)

    patch_StreamProxy = \
        mocker.patch.object(auctions_server_module, 'StreamProxy',
                            return_value=stream_proxy)

    auctions_server = NonCallableMock(spec_set=
                                      AuctionsServerAttributesContainer)

    logger = MagicMock(spec_set=frontend.logger)
    proxy_mappings = MagicMock(spec_set=Memoizer({}))
    proxy_mappings.get.return_value = proxy_path
    config = MagicMock(spec_set=Config)
    config.__getitem__.side_effect = config_getitem

    auctions_server.logger = logger
    auctions_server.proxy_mappings = proxy_mappings
    auctions_server.config = config
    auctions_server.event_sources_pool = event_sources_pool
    auctions_server.proxy_connection_pool = proxy_connection_pool
    auctions_server.db = db

    mocker.patch.object(auctions_server_module, 'auctions_server',
                        auctions_server)

    return {'server': auctions_server,
            'proxy_mappings': proxy_mappings,
            'mock_path_info': mock_path_info,
            'patch_StreamProxy': patch_StreamProxy,
            'patch_redirect': patch_redirect,
            'patch_abort': patch_abort}

@pytest.fixture(scope='function')
def auctions_server(request):
    params = getattr(request, 'param', {})
    server_config = params.get('server_config', {})

    logger = MagicMock(spec_set=frontend.logger)
    logger.name = server_config.get('logger_name', 'some-logger')
    frontend.logger_name = logger.name
    frontend._logger = logger

    for key in ('limit_replications_func', 'limit_replications_progress'):
        frontend.config.pop(key, None)

    for key in ('limit_replications_func', 'limit_replications_progress'):
        if key in server_config:
            frontend.config[key] = server_config[key]

    frontend.couch_server = MagicMock(spec_set=Server)
    frontend.config['TIMEZONE'] = 'some_time_zone'

    if 'couch_tasks' in params:
        frontend.couch_server.tasks.return_value = params['couch_tasks']

    test_app = TestApp(frontend)
    return {'app': frontend, 'test_app': test_app}

@pytest.yield_fixture(scope="function")
def log_for_test(request):
    LOGGER.debug('-------- Test Start ---------')
    LOGGER.debug('Current module: {0}'.format(request.module.__name__))
    LOGGER.debug('Current test class: {0}'.format(request.cls.__name__))
    LOGGER.debug('Current test function: {0}'
                 .format(request.function.__name__))
    yield LOGGER
    LOGGER.debug('-------- Test End ---------')
<|endoftext|>


<commit_before>""" Test schema IO

:Author: Jonathan Karr <karr@mssm.edu>
:Date: 2016-11-23
:Copyright: 2016, Karr Lab
:License: MIT
"""
from wc_utils.schema import core, utils
from wc_utils.schema.io import ExcelIo
import os
import sys
import tempfile
import unittest

class Root(core.Model):
    id = core.StringAttribute(primary=True, unique=True)
    name = core.StringAttribute()

    class Meta(core.Model.Meta):
        attribute_order = ('id', 'name', )
        tabular_orientation = core.TabularOrientation['column']

class Node(core.Model):
    id = core.SlugAttribute(primary=True)
    root = core.ManyToOneAttribute(Root, related_name='nodes')
    val1 = core.FloatAttribute()
    val2 = core.FloatAttribute()

    class Meta(core.Model.Meta):
        attribute_order = ('id', 'root', 'val1', 'val2', )

class OneToManyRowAttribute(core.OneToManyAttribute):
    pass

class OneToManyInlineAttribute(core.OneToManyAttribute):

    def serialize(self, value):
        return ', '.join([obj.id for obj in value])

    def deserialize(self, value, objects):
        if value:
            return (set((OneToManyInline(x) for x in value.split(', '))), None)
        else:
            return (set(), None)

class Leaf(core.Model):
    id = core.StringAttribute(primary=True)
    nodes = core.ManyToManyAttribute(Node, related_name='leaves')
    val1 = core.FloatAttribute()
    val2 = core.FloatAttribute()
    onetomany_rows = OneToManyRowAttribute('OneToManyRow', related_name='leaf', related_none=False)
    #onetomany_inlines = OneToManyInlineAttribute('OneToManyInline', related_name='leaf', related_none=False)

    class Meta(core.Model.Meta):
        attribute_order = (
            'id', 'nodes', 'val1', 'val2',
            'onetomany_rows',  # 'onetomany_inlines'
        )

class OneToManyRow(core.Model):
    id = core.SlugAttribute(primary=True)

    class Meta(core.Model.Meta):
        attribute_order = ('id',)

class OneToManyInline(core.Model):
    id = core.SlugAttribute(primary=False)

    class Meta(core.Model.Meta):
        attribute_order = ('id',)
        tabular_orientation = core.TabularOrientation['inline']

class TestIo(unittest.TestCase):

    def setUp(self):
        _, self.filename = tempfile.mkstemp(suffix='.xlsx')

    def tearDown(self):
        print(self.filename)
        #if os.path.isfile(self.filename): todo
        #    os.remove(self.filename)

    def test_write_read(self):
        root = Root(id='root', name=u'\u20ac')
        nodes = [
            Node(root=root, id='node_0', val1=1, val2=2),
            #Node(root=root, id='node_1', val1=3, val2=4),
            #Node(root=root, id='node_2', val1=5, val2=6),
        ]
        leaves = [
            Leaf(nodes=[nodes[0]], id='leaf_0_0', val1=7, val2=8),
            #Leaf(nodes=[nodes[0]], id='leaf_0_1', val1=9, val2=10),
            #Leaf(nodes=[nodes[1]], id='leaf_1_0', val1=11, val2=12),
            #Leaf(nodes=[nodes[1]], id='leaf_1_1', val1=13, val2=14),
            #Leaf(nodes=[nodes[2]], id='leaf_2_0', val1=15, val2=16),
            #Leaf(nodes=[nodes[2]], id='leaf_2_1', val1=17, val2=18),
        ]
        leaves[0].onetomany_rows = [OneToManyRow(id='row_0_0')]  # , OneToManyRow(id='row_0_1')]
        #leaves[1].onetomany_rows = [OneToManyRow(id='row_1_0'), OneToManyRow(id='row_1_1')]
        #leaves[2].onetomany_rows = [OneToManyRow(id='row_2_0'), OneToManyRow(id='row_2_1')]
        #leaves[3].onetomany_rows = [OneToManyRow(id='row_3_0'), OneToManyRow(id='row_3_1')]
        #leaves[4].onetomany_rows = [OneToManyRow(id='row_4_0'), OneToManyRow(id='row_4_1')]
        #leaves[5].onetomany_rows = [OneToManyRow(id='row_5_0'), OneToManyRow(id='row_5_1')]

        """
        leaves[0].onetomany_inlines = [OneToManyInline(id='inline_0_0'), OneToManyInline(id='inline_0_1')]
        leaves[1].onetomany_inlines = [OneToManyInline(id='inline_1_0'), OneToManyInline(id='inline_1_1')]
        leaves[2].onetomany_inlines = [OneToManyInline(id='inline_2_0'), OneToManyInline(id='inline_2_1')]
        leaves[3].onetomany_inlines = [OneToManyInline(id='inline_3_0'), OneToManyInline(id='inline_3_1')]
        leaves[4].onetomany_inlines = [OneToManyInline(id='inline_4_0'), OneToManyInline(id='inline_4_1')]
        leaves[5].onetomany_inlines = [OneToManyInline(id='inline_5_0'), OneToManyInline(id='inline_5_1')]
        """

        # test integrity of relationships
        for leaf in leaves:
            for row in leaf.onetomany_rows:
                self.assertEqual(row.leaf, leaf)

        # write/read to/from Excel
        objects = set((root, )) | root.get_related()
        objects = utils.group_objects_by_model(objects)

        ExcelIo.write(self.filename, set((root,)), [Root, Node, Leaf, ])
        objects2 = ExcelIo.read(self.filename, set((Root, Node, Leaf, OneToManyRow)))

        # test objects saved and loaded correctly
        self.assertEqual(len(objects2), len(objects))
        for model in objects.keys():
            self.assertEqual(len(objects2[model]), len(objects[model]))

        root2 = objects2[Root].pop()
        self.assertEqual(set([x.id for x in root2.nodes]), set([x.id for x in root.nodes]))
        self.assertEqual(root2, root)

        # unicode
        self.assertEqual(root2.name, u'\u20ac')
<commit_msg>debugging test<commit_after>""" Test schema IO

:Author: Jonathan Karr <karr@mssm.edu>
:Date: 2016-11-23
:Copyright: 2016, Karr Lab
:License: MIT
"""
from wc_utils.schema import core, utils
from wc_utils.schema.io import ExcelIo
import os
import sys
import tempfile
import unittest

class Root(core.Model):
    id = core.StringAttribute(primary=True, unique=True)
    name = core.StringAttribute()

    class Meta(core.Model.Meta):
        attribute_order = ('id', 'name', )
        tabular_orientation = core.TabularOrientation['column']

class Node(core.Model):
    id = core.SlugAttribute(primary=True)
    root = core.ManyToOneAttribute(Root, related_name='nodes')
    val1 = core.FloatAttribute()
    val2 = core.FloatAttribute()

    class Meta(core.Model.Meta):
        attribute_order = ('id', 'root', 'val1', 'val2', )

class OneToManyRowAttribute(core.OneToManyAttribute):
    pass

class OneToManyInlineAttribute(core.OneToManyAttribute):

    def serialize(self, value):
        return ', '.join([obj.id for obj in value])

    def deserialize(self, value, objects):
        if value:
            objs = []
            for id in value.split(', '):
                obj = OneToManyInline(id=id)
                if OneToManyInline not in objects:
                    objects[OneToManyInline] = {}
                objects[OneToManyInline][id] = obj
                objs.append(obj)

            return (objs, None)
        else:
            return (set(), None)

class Leaf(core.Model):
    id = core.StringAttribute(primary=True)
    nodes = core.ManyToManyAttribute(Node, related_name='leaves')
    val1 = core.FloatAttribute()
    val2 = core.FloatAttribute()
    onetomany_rows = OneToManyRowAttribute('OneToManyRow', related_name='leaf', related_none=False)
    onetomany_inlines = OneToManyInlineAttribute('OneToManyInline', related_name='leaf', related_none=False)

    class Meta(core.Model.Meta):
        attribute_order = (
            'id', 'nodes', 'val1', 'val2',
            'onetomany_rows', 'onetomany_inlines'
        )

class OneToManyRow(core.Model):
    id = core.SlugAttribute(primary=True)

    class Meta(core.Model.Meta):
        attribute_order = ('id',)

class OneToManyInline(core.Model):
    id = core.SlugAttribute(primary=False)

    class Meta(core.Model.Meta):
        attribute_order = ('id',)
        tabular_orientation = core.TabularOrientation['inline']

class TestIo(unittest.TestCase):

    def setUp(self):
        _, self.filename = tempfile.mkstemp(suffix='.xlsx')

    def tearDown(self):
        if os.path.isfile(self.filename):
            os.remove(self.filename)

    def test_write_read(self):
        root = Root(id='root', name=u'\u20ac')
        nodes = [
            Node(root=root, id='node_0', val1=1, val2=2),
            Node(root=root, id='node_1', val1=3, val2=4),
            Node(root=root, id='node_2', val1=5, val2=6),
        ]
        leaves = [
            Leaf(nodes=[nodes[0]], id='leaf_0_0', val1=7, val2=8),
            Leaf(nodes=[nodes[0]], id='leaf_0_1', val1=9, val2=10),
            Leaf(nodes=[nodes[1]], id='leaf_1_0', val1=11, val2=12),
            Leaf(nodes=[nodes[1]], id='leaf_1_1', val1=13, val2=14),
            Leaf(nodes=[nodes[2]], id='leaf_2_0', val1=15, val2=16),
            Leaf(nodes=[nodes[2]], id='leaf_2_1', val1=17, val2=18),
        ]
        leaves[0].onetomany_rows = [OneToManyRow(id='row_0_0'), OneToManyRow(id='row_0_1')]
        leaves[1].onetomany_rows = [OneToManyRow(id='row_1_0'), OneToManyRow(id='row_1_1')]
        leaves[2].onetomany_rows = [OneToManyRow(id='row_2_0'), OneToManyRow(id='row_2_1')]
        leaves[3].onetomany_rows = [OneToManyRow(id='row_3_0'), OneToManyRow(id='row_3_1')]
        leaves[4].onetomany_rows = [OneToManyRow(id='row_4_0'), OneToManyRow(id='row_4_1')]
        leaves[5].onetomany_rows = [OneToManyRow(id='row_5_0'), OneToManyRow(id='row_5_1')]

        leaves[0].onetomany_inlines = [OneToManyInline(id='inline_0_0'), OneToManyInline(id='inline_0_1')]
        leaves[1].onetomany_inlines = [OneToManyInline(id='inline_1_0'), OneToManyInline(id='inline_1_1')]
        leaves[2].onetomany_inlines = [OneToManyInline(id='inline_2_0'), OneToManyInline(id='inline_2_1')]
        leaves[3].onetomany_inlines = [OneToManyInline(id='inline_3_0'), OneToManyInline(id='inline_3_1')]
        leaves[4].onetomany_inlines = [OneToManyInline(id='inline_4_0'), OneToManyInline(id='inline_4_1')]
        leaves[5].onetomany_inlines = [OneToManyInline(id='inline_5_0'), OneToManyInline(id='inline_5_1')]

        # test integrity of relationships
        for leaf in leaves:
            for row in leaf.onetomany_rows:
                self.assertEqual(row.leaf, leaf)

        # write/read to/from Excel
        objects = set((root, )) | root.get_related()
        objects = utils.group_objects_by_model(objects)

        ExcelIo.write(self.filename, set((root,)), [Root, Node, Leaf, ])
        objects2 = ExcelIo.read(self.filename, set((Root, Node, Leaf, OneToManyRow)))

        # test objects saved and loaded correctly
        self.assertEqual(len(objects2), len(objects))
        for model in objects.keys():
            self.assertEqual(len(objects2[model]), len(objects[model]))

        root2 = objects2[Root].pop()
        self.assertEqual(set([x.id for x in root2.nodes]), set([x.id for x in root.nodes]))
        self.assertEqual(root2, root)

        # unicode
        self.assertEqual(root2.name, u'\u20ac')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
skipif_markers
--------------

Contains pytest skipif markers to be used in the suite.
"""

import pytest
import os

try:
    os.environ[u'TRAVIS']
except KeyError:
    travis = False
else:
    travis = True

try:
    os.environ[u'DISABLE_NETWORK_TESTS']
except KeyError:
    no_network = False
else:
    no_network = True

# For some reason pytest incorrectly uses the first reason text regardless of
# which condition matches. Using a unified message for now
# travis_reason = 'Works locally with tox but fails on Travis.'
# no_network_reason = 'Needs a network connection to GitHub.'
reason = (
    'Fails on Travis or else there is no network connection to '
    'GitHub/Bitbucket.'
)

skipif_travis = pytest.mark.skipif(travis, reason=reason)
skipif_no_network = pytest.mark.skipif(no_network, reason=reason)
<commit_msg>Revert skipif markers to use correct reasons (bug fixed in pytest)<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
skipif_markers
--------------

Contains pytest skipif markers to be used in the suite.
"""

import pytest
import os

try:
    os.environ[u'TRAVIS']
except KeyError:
    travis = False
else:
    travis = True

try:
    os.environ[u'DISABLE_NETWORK_TESTS']
except KeyError:
    no_network = False
else:
    no_network = True

skipif_travis = pytest.mark.skipif(
    travis, reason='Works locally with tox but fails on Travis.'
)

skipif_no_network = pytest.mark.skipif(
    no_network, reason='Needs a network connection to GitHub/Bitbucket.'
)
<|endoftext|>