<commit_before>from django.test import TestCase
from django.core.urlresolvers import reverse
import json

class TestDataAPI(TestCase):
    fixtures = ['data.json', 'hierarchy.json']

    def test_basic_api_functionality(self):
        """
            Testing basic API functionality.
        """
        url = reverse('api_dispatch_list',
                      kwargs={'resource_name': 'hierarchy',
                      'api_name': 'api'})
        response = self.client.get(url)
        self.assertEqual("application/json", response["Content-Type"])
        self.assertEqual(response.status_code, 200)
        json_item = json.loads(response.content)
        self.assertIn("meta", json_item)
        self.assertIn("objects", json_item)

    def test_post_data(self):
        """
            Testing basic API functionality.
        """
        url = reverse('api_dispatch_list',
                      kwargs={'resource_name': 'data/headteacher',
                      'api_name': 'api'})
        response = self.client.post(url,
                                    format="json",
                                    data={"name_first": "head",
                                    "name_last": "teacher",
                                    "year": "2012-10-12T10:00:00Z",
                                    "date_of_birth": "2012-10-12T10:00:00Z",
                                    })
        print json.loads(response.content)
        self.assertEqual("application/json", response["Content-Type"])
        self.assertEqual(response.status_code, 200)
        json_item = json.loads(response.content)
        self.assertIn("meta", json_item)
        self.assertIn("objects", json_item)<commit_msg>create unittests for API<commit_after>from django.test import TestCase
from tastypie.test import ResourceTestCase
from django.core.urlresolvers import reverse
import json
from data.models import HeadTeacher
import datetime

class TestHeadteacherAPI(ResourceTestCase):
    fixtures = ['hierarchy.json']

    def test_basic_api_functionality(self):
        """
            Testing basic API functionality.
        """
        url = reverse('api_dispatch_list',
                      kwargs={'resource_name': 'school',
                      'api_name': 'v1'})
        response = self.client.get(url)
        self.assertEqual("application/json", response["Content-Type"])
        self.assertEqual(response.status_code, 200)
        json_item = json.loads(response.content)
        self.assertIn("meta", json_item)
        self.assertIn("objects", json_item)

    def test_post_headteacher_json_data(self):
        """
            Testing basic API functionality.
        """
        url = reverse('api_dispatch_list',
                      kwargs={'resource_name': 'data/headteacher',
                      'api_name': 'api'})
        response = self.api_client.post(url,
                                    format="json",
                                    data={"first_name": "test_first_name",
                                    "last_name": "test_last_name",
                                    "date_of_birth": "2012-10-12T10:00:00Z",
                                    "gender": "male",
                                    "msisdn": "0123456789",
                                    "emis": "/api/v1/school/EMIS/4813/"
                                    })

        json_item = json.loads(response.content)
        self.assertEqual("test_first_name", json_item["first_name"])
        self.assertEqual("test_last_name", json_item["last_name"])

        self.assertEqual("2012-10-12T10:00:00", json_item["date_of_birth"])
        self.assertEqual("male", json_item["gender"])
        self.assertEqual("0123456789", json_item["msisdn"])
        self.assertEqual(4813, json_item["emis"]["EMIS"])
        self.assertEqual("Musungu", json_item["emis"]["name"])

        headteacher = HeadTeacher.objects.get(pk=1)
        self.assertEqual("test_first_name", headteacher.first_name)
        self.assertEqual("test_last_name", headteacher.last_name)
        self.assertEqual(2, headteacher.emis_id)
        self.assertEqual("male", headteacher.gender)
        self.assertEqual(False, headteacher.is_zonal_head)
        self.assertEqual( datetime.date(2012, 10, 12), headteacher.date_of_birth)
        self.assertIsNotNone(headteacher.created_at)
        self.assertEqual("Musungu", headteacher.emis.name)
        self.assertEqual(4813, headteacher.emis.EMIS)

# class TestSchoolDataAPI(ResourceTestCase):
#     fixtures = ['hierarchy.json']

#     def test_basic_api_functionality(self):
#         """
#             Testing basic schooldata API functionality.
#         """
#         url = reverse('api_dispatch_list',
#                       kwargs={'resource_name': 'hierarchy',
#                       'api_name': 'v1'})
#         response = self.client.get(url)
#         self.assertEqual("application/json", response["Content-Type"])
#         self.assertEqual(response.status_code, 200)
#         json_item = json.loads(response.content)
#         self.assertIn("meta", json_item)
#         self.assertIn("objects", json_item)

#     def test_post_headteacher_json_data(self):
#         """
#             Testing posting school data.
#         """
#         url = reverse('api_dispatch_list',
#                       kwargs={'resource_name': 'data/headteacher',
#                       'api_name': 'api'})
#         response = self.api_client.post(url,
#                                     format="json",
#                                     data={"first_name": "test_first_name",
#                                     "last_name": "test_last_name",
#                                     "date_of_birth": "2012-10-12T10:00:00Z",
#                                     "gender": "male",
#                                     "msisdn": "0123456789",
#                                     "emis": "/api/v1/hierarchy/EMIS/4813/"
#                                     })

#         json_item = json.loads(response.content)
#         self.assertEqual("test_first_name", json_item["first_name"])
#         self.assertEqual("test_last_name", json_item["last_name"])

#         self.assertEqual("2012-10-12T10:00:00", json_item["date_of_birth"])
#         self.assertEqual("male", json_item["gender"])
#         self.assertEqual("0123456789", json_item["msisdn"])
#         self.assertEqual(4813, json_item["emis"]["EMIS"])
#         self.assertEqual("Musungu", json_item["emis"]["name"])

#         headteacher = HeadTeacher.objects.get(pk=1)
#         self.assertEqual("test_first_name", headteacher.first_name)
#         self.assertEqual("test_last_name", headteacher.last_name)
#         self.assertEqual(2, headteacher.emis_id)
#         self.assertEqual("male", headteacher.gender)
#         self.assertEqual(False, headteacher.is_zonal_head)
#         self.assertEqual( datetime.date(2012, 10, 12), headteacher.date_of_birth)
#         self.assertIsNotNone(headteacher.created_at)
#         self.assertEqual("Musungu", headteacher.emis.name)
#         self.assertEqual(4813, headteacher.emis.EMIS)<|endoftext|>


<commit_before>#!/usr/bin/env python

print ("Hello World!")
<commit_msg>Updating the test commit file<commit_after>#!/usr/bin/env python

print ("Hello World!")

print ("change to this file")

<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Tests for Howdoi."""
import json
import os
import re
import time
import unittest
from cachelib import NullCache

from howdoi import howdoi
from pyquery import PyQuery as pq

class HowdoiTestCase(unittest.TestCase):
    def call_howdoi(self, query):
        return howdoi.howdoi(query)

    def _get_result_mock(self, url):
        file_name = howdoi._format_url_to_filename(url)
        file_path = os.path.join(howdoi.HTML_CACHE_PATH, file_name)
        try:
            f = open(file_path, 'r', encoding="utf8")
            cached_file_content = f.read()
            f.close()
            return cached_file_content
        except FileNotFoundError:
            result = self.original_get_result(url)
            f = open(file_path, 'w+')
            f.write(result)
            f.close()
            return result

    def setUp(self):
        self.original_get_result = howdoi._get_result
        howdoi._get_result = self._get_result_mock

        # ensure no cache is used during testing.
        howdoi.cache = NullCache()

        self.queries = ['format date bash',
                        'print stack trace python',
                        'convert mp4 to animated gif',
                        'create tar archive',
                        'cat']
        self.help_queries = howdoi.SUPPORTED_HELP_QUERIES
        self.pt_queries = ['abrir arquivo em python',
                           'enviar email em django',
                           'hello world em c']
        self.bad_queries = ['moe',
                            'mel']

    def assertValidResponse(self, res):
        self.assertTrue(len(res) > 0)

    def tearDown(self):
        keys_to_remove = ['HOWDOI_URL', 'HOWDO_SEARCH_ENGINE']
        for key in keys_to_remove:
            if key in os.environ:
                del os.environ[key]

    def test_get_link_at_pos(self):
        self.assertEqual(howdoi.get_link_at_pos(['/questions/42/'], 1),
                         '/questions/42/')
        self.assertEqual(howdoi.get_link_at_pos(['/questions/42/'], 2),
                         '/questions/42/')
        self.assertEqual(howdoi.get_link_at_pos(['/howdoi', '/questions/42/'], 1),
                         '/howdoi')
        self.assertEqual(howdoi.get_link_at_pos(['/howdoi', '/questions/42/'], 2),
                         '/questions/42/')
        self.assertEqual(howdoi.get_link_at_pos(['/questions/42/', '/questions/142/'], 1),
                         '/questions/42/')

    def test_answers(self):
        for query in self.queries:
            self.assertValidResponse(self.call_howdoi(query))
        for query in self.bad_queries:
            self.assertValidResponse(self.call_howdoi(query))

        os.environ['HOWDOI_URL'] = 'pt.stackoverflow.com'
        for query in self.pt_queries:
            self.assertValidResponse(self.call_howdoi(query))

    def test_answers_bing(self):
        os.environ['HOWDOI_SEARCH_ENGINE'] = 'bing'
        for query in self.queries:
            self.assertValidResponse(self.call_howdoi(query))
        for query in self.bad_queries:
            self.assertValidResponse(self.call_howdoi(query))

        os.environ['HOWDOI_URL'] = 'pt.stackoverflow.com'
        for query in self.pt_queries:
            self.assertValidResponse(self.call_howdoi(query))

        os.environ['HOWDOI_SEARCH_ENGINE'] = ''

    def test_answers_duckduckgo(self):
        os.environ['HOWDOI_SEARCH_ENGINE'] = 'duckduckgo'
        for query in self.queries:
            self.assertValidResponse(self.call_howdoi(query))
        for query in self.bad_queries:
            self.assertValidResponse(self.call_howdoi(query))

        os.environ['HOWDOI_URL'] = 'pt.stackoverflow.com'
        for query in self.pt_queries:
            self.assertValidResponse(self.call_howdoi(query))

        os.environ['HOWDOI_SEARCH_ENGINE'] = ''

    def test_answer_links_using_l_option(self):
        for query in self.queries:
            response = self.call_howdoi(query + ' -l')
            self.assertNotEqual(re.match('http.?://.*questions/\d.*', response, re.DOTALL), None)

    def test_answer_links_using_all_option(self):
        for query in self.queries:
            response = self.call_howdoi(query + ' -a')
            self.assertNotEqual(re.match('.*http.?://.*questions/\d.*', response, re.DOTALL), None)

    def test_position(self):
        query = self.queries[0]
        first_answer = self.call_howdoi(query)
        not_first_answer = self.call_howdoi(query + ' -p5')
        self.assertNotEqual(first_answer, not_first_answer)

    def test_all_text(self):
        query = self.queries[0]
        first_answer = self.call_howdoi(query)
        second_answer = self.call_howdoi(query + ' -a')
        self.assertNotEqual(first_answer, second_answer)
        self.assertNotEqual(re.match('.*Answer from http.?://.*', second_answer, re.DOTALL), None)

    def test_json_output(self):
        query = self.queries[0]
        txt_answer = self.call_howdoi(query)
        json_answer = self.call_howdoi(query + ' -j')
        link_answer = self.call_howdoi(query + ' -l')
        json_answer = json.loads(json_answer)[0]
        self.assertEqual(json_answer["answer"], txt_answer)
        self.assertEqual(json_answer["link"], link_answer)
        self.assertEqual(json_answer["position"], 1)

    def test_multiple_answers(self):
        query = self.queries[0]
        first_answer = self.call_howdoi(query)
        second_answer = self.call_howdoi(query + ' -n3')
        self.assertNotEqual(first_answer, second_answer)

    def test_unicode_answer(self):
        assert self.call_howdoi('make a log scale d3')
        assert self.call_howdoi('python unittest -n3')
        assert self.call_howdoi('parse html regex -a')
        assert self.call_howdoi('delete remote git branch -a')

    def test_colorize(self):
        query = self.queries[0]
        normal = self.call_howdoi(query)
        colorized = self.call_howdoi('-c ' + query)
        self.assertTrue(normal.find('[39;') is -1)
        self.assertTrue(colorized.find('[39;') is not -1)

    def test_get_text_without_links(self):
        html = '''\n
The halting problem is basically a\n  formal way of asking if you can tell\n  whether or not an arbitrary program\n  will eventually halt.
\n  \n
In other words, can you write a\n  program called a halting oracle,\n  HaltingOracle(program, input), which\n  returns true if program(input) would\n  eventually halt, and which returns\n  false if it wouldn't?
\n  \n
The answer is: no, you can't.
\n'''
        paragraph = pq(html)
        expected_output = '''The halting problem is basically a\n  formal way of asking if you can tell\n  whether or not an arbitrary program\n  will eventually halt.\n\n  \n  \nIn other words, can you write a\n  program called a halting oracle,\n  HaltingOracle(program, input), which\n  returns true if program(input) would\n  eventually halt, and which returns\n  false if it wouldn't?\n\n  \n  \nThe answer is: no, you can't.\n\n'''
        actual_output = howdoi.get_text(paragraph)
        self.assertEqual(actual_output, expected_output)

    def test_get_text_with_one_link(self):
        html = '
It\'s a protocol-relative URL (typically HTTP or HTTPS). So if I\'m on http://example.org and I link (or include an image, script, etc.) to //example.com/1.png, it goes to http://example.com/1.png. If I\'m on https://example.org, it goes to https://example.com/1.png.
'
        paragraph = pq(html)
        expected_output = "It's a [protocol-relative URL](http://paulirish.com/2010/the-protocol-relative-url/) (typically HTTP or HTTPS). So if I'm on http://example.org and I link (or include an image, script, etc.) to //example.com/1.png, it goes to http://example.com/1.png. If I'm on https://example.org, it goes to https://example.com/1.png."
        actual_output = howdoi.get_text(paragraph)
        self.assertEqual(actual_output, expected_output)

    def test_get_text_with_multiple_links_test_one(self):
        html = 'Here\'s a quote from wikipedia\'s manual of style section on links (but see also their comprehensive page on External Links)'
        paragraph = pq(html)
        expected_output = "Here's a quote from [wikipedia's manual of style](http://en.wikipedia.org/wiki/Wikipedia:Manual_of_Style#Links) section on links (but see also [their comprehensive page on External Links](http://en.wikipedia.org/wiki/Wikipedia:External_links))"
        actual_output = howdoi.get_text(paragraph)
        self.assertEqual(actual_output, expected_output)

    def test_get_text_with_multiple_links_test_two(self):
        html = 'For example, if I were to reference apple.com as the subject of a sentence - or to talk about Apple\'s website as the topic of conversation. This being different to perhaps recommendations for reading our article about Apple\'s website.'
        paragraph = pq(html)
        expected_output = "For example, if I were to reference [apple.com](http://www.apple.com/) as the subject of a sentence - or to talk about [Apple's website](http://www.apple.com/) as the topic of conversation. This being different to perhaps recommendations for reading [our article about Apple's website](https://ux.stackexchange.com/q/14872/6046)."
        actual_output = howdoi.get_text(paragraph)
        self.assertEqual(actual_output, expected_output)

    def test_get_text_with_link_but_with_copy_duplicating_the_href(self):
        html = 'https://github.com/jquery/jquery/blob/56136897f241db22560b58c3518578ca1453d5c7/src/manipulation.js#L451'
        paragraph = pq(html)
        expected_output = 'https://github.com/jquery/jquery/blob/56136897f241db22560b58c3518578ca1453d5c7/src/manipulation.js#L451'
        actual_output = howdoi.get_text(paragraph)
        self.assertEqual(actual_output, expected_output)

    def test_get_text_with_a_link_but_copy_is_within_nested_div(self):
        html = 'If the function is from a source file available on the filesystem, then inspect.getsource(foo) might be of help:'
        paragraph = pq(html)
        expected_output = 'If the function is from a source file available on the filesystem, then [inspect.getsource(foo)](https://docs.python.org/3/library/inspect.html#inspect.getsource) might be of help:'
        actual_output = howdoi.get_text(paragraph)
        self.assertEqual(actual_output, expected_output)

    def test_get_questions(self):
        links = ['https://stackoverflow.com/questions/tagged/cat', 'http://rads.stackoverflow.com/amzn/click/B007KAZ166',
                 'https://stackoverflow.com/questions/40108569/how-to-get-the-last-line-of-a-file-using-cat-command']
        expected_output = [
            'https://stackoverflow.com/questions/40108569/how-to-get-the-last-line-of-a-file-using-cat-command']
        actual_output = howdoi._get_questions(links)
        self.assertSequenceEqual(actual_output, expected_output)

    def test_help_queries(self):
        help_queries = self.help_queries

        for query in help_queries:
            output = self.call_howdoi(query)
            self.assertTrue(output)
            self.assertIn('few popular howdoi commands', output)
            self.assertIn('retrieve n number of answers', output)
            self.assertIn(
                'Specify the search engine you want to use e.g google,bing',
                output
            )

    def test_format_url_to_filename(self):
        url = 'https://stackoverflow.com/questions/tagged/cat'
        INVALID_FILENAME_CHARACTERS = ['/', '\\', '%']
        filename = howdoi._format_url_to_filename(url, 'html')
        self.assertTrue(filename)
        self.assertTrue(filename.endswith('html'))
        for invalid_character in INVALID_FILENAME_CHARACTERS:
            self.assertNotIn(invalid_character, filename)

    def test_help_queries_are_properly_validated(self):
        help_queries = self.help_queries
        for query in help_queries:
            is_valid_help_query = howdoi._is_help_query(query)
            self.assertTrue(is_valid_help_query)
        bad_help_queries = [self.queries[0],
                            self.bad_queries[0], 'use how do i']

        for query in bad_help_queries:
            self.assertFalse(howdoi._is_help_query(query))

class HowdoiTestCaseEnvProxies(unittest.TestCase):

    def setUp(self):
        self.temp_get_proxies = howdoi.getproxies

    def tearDown(self):
        howdoi.getproxies = self.temp_get_proxies

    def test_get_proxies1(self):
        def getproxies1():
            proxies = {'http': 'wwwproxy.company.com',
                       'https': 'wwwproxy.company.com',
                       'ftp': 'ftpproxy.company.com'}
            return proxies

        howdoi.getproxies = getproxies1
        filtered_proxies = howdoi.get_proxies()
        self.assertTrue('http://' in filtered_proxies['http'])
        self.assertTrue('http://' in filtered_proxies['https'])
        self.assertTrue('ftp' not in filtered_proxies.keys())

if __name__ == '__main__':
    unittest.main()
<commit_msg>refactored cache to use gzip instead of raw html<commit_after>#!/usr/bin/env python

"""Tests for Howdoi."""
import gzip
import json
import os
import re
import time
import unittest

from cachelib import NullCache
from pyquery import PyQuery as pq

from howdoi import howdoi

class HowdoiTestCase(unittest.TestCase):
    def call_howdoi(self, query):
        return howdoi.howdoi(query)

    def _get_result_mock(self, url):
        file_name = howdoi._format_url_to_filename(url, 'html.gz')
        file_path = os.path.join(howdoi.HTML_CACHE_PATH, file_name)
        try:
            with gzip.open(file_path, 'rb') as f:
                cached_page_content = str(f.read(),encoding='utf-8')
                return cached_page_content

        except FileNotFoundError:
            page_content = self.original_get_result(url)
            with gzip.open(file_path, 'wb') as f:
                f.write(page_content)
                return page_content

    def setUp(self):
        self.original_get_result = howdoi._get_result
        howdoi._get_result = self._get_result_mock

        # ensure no cache is used during testing.
        howdoi.cache = NullCache()

        self.queries = ['format date bash',
                        'print stack trace python',
                        'convert mp4 to animated gif',
                        'create tar archive',
                        'cat']
        self.help_queries = howdoi.SUPPORTED_HELP_QUERIES
        self.pt_queries = ['abrir arquivo em python',
                           'enviar email em django',
                           'hello world em c']
        self.bad_queries = ['moe',
                            'mel']

    def assertValidResponse(self, res):
        self.assertTrue(len(res) > 0)

    def tearDown(self):
        keys_to_remove = ['HOWDOI_URL', 'HOWDO_SEARCH_ENGINE']
        for key in keys_to_remove:
            if key in os.environ:
                del os.environ[key]

    def test_get_link_at_pos(self):
        self.assertEqual(howdoi.get_link_at_pos(['/questions/42/'], 1),
                         '/questions/42/')
        self.assertEqual(howdoi.get_link_at_pos(['/questions/42/'], 2),
                         '/questions/42/')
        self.assertEqual(howdoi.get_link_at_pos(['/howdoi', '/questions/42/'], 1),
                         '/howdoi')
        self.assertEqual(howdoi.get_link_at_pos(['/howdoi', '/questions/42/'], 2),
                         '/questions/42/')
        self.assertEqual(howdoi.get_link_at_pos(['/questions/42/', '/questions/142/'], 1),
                         '/questions/42/')

    def test_answers(self):
        for query in self.queries:
            self.assertValidResponse(self.call_howdoi(query))
        for query in self.bad_queries:
            self.assertValidResponse(self.call_howdoi(query))

        os.environ['HOWDOI_URL'] = 'pt.stackoverflow.com'
        for query in self.pt_queries:
            self.assertValidResponse(self.call_howdoi(query))

    def test_answers_bing(self):
        os.environ['HOWDOI_SEARCH_ENGINE'] = 'bing'
        for query in self.queries:
            self.assertValidResponse(self.call_howdoi(query))
        for query in self.bad_queries:
            self.assertValidResponse(self.call_howdoi(query))

        os.environ['HOWDOI_URL'] = 'pt.stackoverflow.com'
        for query in self.pt_queries:
            self.assertValidResponse(self.call_howdoi(query))

        os.environ['HOWDOI_SEARCH_ENGINE'] = ''

    def test_answers_duckduckgo(self):
        os.environ['HOWDOI_SEARCH_ENGINE'] = 'duckduckgo'
        for query in self.queries:
            self.assertValidResponse(self.call_howdoi(query))
        for query in self.bad_queries:
            self.assertValidResponse(self.call_howdoi(query))

        os.environ['HOWDOI_URL'] = 'pt.stackoverflow.com'
        for query in self.pt_queries:
            self.assertValidResponse(self.call_howdoi(query))

        os.environ['HOWDOI_SEARCH_ENGINE'] = ''

    def test_answer_links_using_l_option(self):
        for query in self.queries:
            response = self.call_howdoi(query + ' -l')
            self.assertNotEqual(re.match('http.?://.*questions/\d.*', response, re.DOTALL), None)

    def test_answer_links_using_all_option(self):
        for query in self.queries:
            response = self.call_howdoi(query + ' -a')
            self.assertNotEqual(re.match('.*http.?://.*questions/\d.*', response, re.DOTALL), None)

    def test_position(self):
        query = self.queries[0]
        first_answer = self.call_howdoi(query)
        not_first_answer = self.call_howdoi(query + ' -p5')
        self.assertNotEqual(first_answer, not_first_answer)

    def test_all_text(self):
        query = self.queries[0]
        first_answer = self.call_howdoi(query)
        second_answer = self.call_howdoi(query + ' -a')
        self.assertNotEqual(first_answer, second_answer)
        self.assertNotEqual(re.match('.*Answer from http.?://.*', second_answer, re.DOTALL), None)

    def test_json_output(self):
        query = self.queries[0]
        txt_answer = self.call_howdoi(query)
        json_answer = self.call_howdoi(query + ' -j')
        link_answer = self.call_howdoi(query + ' -l')
        json_answer = json.loads(json_answer)[0]
        self.assertEqual(json_answer["answer"], txt_answer)
        self.assertEqual(json_answer["link"], link_answer)
        self.assertEqual(json_answer["position"], 1)

    def test_multiple_answers(self):
        query = self.queries[0]
        first_answer = self.call_howdoi(query)
        second_answer = self.call_howdoi(query + ' -n3')
        self.assertNotEqual(first_answer, second_answer)

    def test_unicode_answer(self):
        assert self.call_howdoi('make a log scale d3')
        assert self.call_howdoi('python unittest -n3')
        assert self.call_howdoi('parse html regex -a')
        assert self.call_howdoi('delete remote git branch -a')

    def test_colorize(self):
        query = self.queries[0]
        normal = self.call_howdoi(query)
        colorized = self.call_howdoi('-c ' + query)
        self.assertTrue(normal.find('[39;') is -1)
        self.assertTrue(colorized.find('[39;') is not -1)

    def test_get_text_without_links(self):
        html = '''\n
The halting problem is basically a\n  formal way of asking if you can tell\n  whether or not an arbitrary program\n  will eventually halt.
\n  \n
In other words, can you write a\n  program called a halting oracle,\n  HaltingOracle(program, input), which\n  returns true if program(input) would\n  eventually halt, and which returns\n  false if it wouldn't?
\n  \n
The answer is: no, you can't.
\n'''
        paragraph = pq(html)
        expected_output = '''The halting problem is basically a\n  formal way of asking if you can tell\n  whether or not an arbitrary program\n  will eventually halt.\n\n  \n  \nIn other words, can you write a\n  program called a halting oracle,\n  HaltingOracle(program, input), which\n  returns true if program(input) would\n  eventually halt, and which returns\n  false if it wouldn't?\n\n  \n  \nThe answer is: no, you can't.\n\n'''
        actual_output = howdoi.get_text(paragraph)
        self.assertEqual(actual_output, expected_output)

    def test_get_text_with_one_link(self):
        html = '
It\'s a protocol-relative URL (typically HTTP or HTTPS). So if I\'m on http://example.org and I link (or include an image, script, etc.) to //example.com/1.png, it goes to http://example.com/1.png. If I\'m on https://example.org, it goes to https://example.com/1.png.
'
        paragraph = pq(html)
        expected_output = "It's a [protocol-relative URL](http://paulirish.com/2010/the-protocol-relative-url/) (typically HTTP or HTTPS). So if I'm on http://example.org and I link (or include an image, script, etc.) to //example.com/1.png, it goes to http://example.com/1.png. If I'm on https://example.org, it goes to https://example.com/1.png."
        actual_output = howdoi.get_text(paragraph)
        self.assertEqual(actual_output, expected_output)

    def test_get_text_with_multiple_links_test_one(self):
        html = 'Here\'s a quote from wikipedia\'s manual of style section on links (but see also their comprehensive page on External Links)'
        paragraph = pq(html)
        expected_output = "Here's a quote from [wikipedia's manual of style](http://en.wikipedia.org/wiki/Wikipedia:Manual_of_Style#Links) section on links (but see also [their comprehensive page on External Links](http://en.wikipedia.org/wiki/Wikipedia:External_links))"
        actual_output = howdoi.get_text(paragraph)
        self.assertEqual(actual_output, expected_output)

    def test_get_text_with_multiple_links_test_two(self):
        html = 'For example, if I were to reference apple.com as the subject of a sentence - or to talk about Apple\'s website as the topic of conversation. This being different to perhaps recommendations for reading our article about Apple\'s website.'
        paragraph = pq(html)
        expected_output = "For example, if I were to reference [apple.com](http://www.apple.com/) as the subject of a sentence - or to talk about [Apple's website](http://www.apple.com/) as the topic of conversation. This being different to perhaps recommendations for reading [our article about Apple's website](https://ux.stackexchange.com/q/14872/6046)."
        actual_output = howdoi.get_text(paragraph)
        self.assertEqual(actual_output, expected_output)

    def test_get_text_with_link_but_with_copy_duplicating_the_href(self):
        html = 'https://github.com/jquery/jquery/blob/56136897f241db22560b58c3518578ca1453d5c7/src/manipulation.js#L451'
        paragraph = pq(html)
        expected_output = 'https://github.com/jquery/jquery/blob/56136897f241db22560b58c3518578ca1453d5c7/src/manipulation.js#L451'
        actual_output = howdoi.get_text(paragraph)
        self.assertEqual(actual_output, expected_output)

    def test_get_text_with_a_link_but_copy_is_within_nested_div(self):
        html = 'If the function is from a source file available on the filesystem, then inspect.getsource(foo) might be of help:'
        paragraph = pq(html)
        expected_output = 'If the function is from a source file available on the filesystem, then [inspect.getsource(foo)](https://docs.python.org/3/library/inspect.html#inspect.getsource) might be of help:'
        actual_output = howdoi.get_text(paragraph)
        self.assertEqual(actual_output, expected_output)

    def test_get_questions(self):
        links = ['https://stackoverflow.com/questions/tagged/cat', 'http://rads.stackoverflow.com/amzn/click/B007KAZ166',
                 'https://stackoverflow.com/questions/40108569/how-to-get-the-last-line-of-a-file-using-cat-command']
        expected_output = [
            'https://stackoverflow.com/questions/40108569/how-to-get-the-last-line-of-a-file-using-cat-command']
        actual_output = howdoi._get_questions(links)
        self.assertSequenceEqual(actual_output, expected_output)

    def test_help_queries(self):
        help_queries = self.help_queries

        for query in help_queries:
            output = self.call_howdoi(query)
            self.assertTrue(output)
            self.assertIn('few popular howdoi commands', output)
            self.assertIn('retrieve n number of answers', output)
            self.assertIn(
                'Specify the search engine you want to use e.g google,bing',
                output
            )

    def test_format_url_to_filename(self):
        url = 'https://stackoverflow.com/questions/tagged/cat'
        INVALID_FILENAME_CHARACTERS = ['/', '\\', '%']
        filename = howdoi._format_url_to_filename(url, 'html')
        self.assertTrue(filename)
        self.assertTrue(filename.endswith('html'))
        for invalid_character in INVALID_FILENAME_CHARACTERS:
            self.assertNotIn(invalid_character, filename)

    def test_help_queries_are_properly_validated(self):
        help_queries = self.help_queries
        for query in help_queries:
            is_valid_help_query = howdoi._is_help_query(query)
            self.assertTrue(is_valid_help_query)
        bad_help_queries = [self.queries[0],
                            self.bad_queries[0], 'use how do i']

        for query in bad_help_queries:
            self.assertFalse(howdoi._is_help_query(query))

class HowdoiTestCaseEnvProxies(unittest.TestCase):

    def setUp(self):
        self.temp_get_proxies = howdoi.getproxies

    def tearDown(self):
        howdoi.getproxies = self.temp_get_proxies

    def test_get_proxies1(self):
        def getproxies1():
            proxies = {'http': 'wwwproxy.company.com',
                       'https': 'wwwproxy.company.com',
                       'ftp': 'ftpproxy.company.com'}
            return proxies

        howdoi.getproxies = getproxies1
        filtered_proxies = howdoi.get_proxies()
        self.assertTrue('http://' in filtered_proxies['http'])
        self.assertTrue('http://' in filtered_proxies['https'])
        self.assertTrue('ftp' not in filtered_proxies.keys())

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># pylint: disable=invalid-name
# pylint: disable=missing-docstring
# pylint: disable=too-few-public-methods
# pylint: disable=too-many-instance-attributes
# pylint: disable=too-many-locals

import json
import os
import re
import subprocess
import sys
import warnings
import yaml

TESTCASES = list()
INVENTORY = 'test/fixtures/hosts'

HERE = os.path.abspath(os.path.dirname(__file__))
ROLE = re.match(
    r'^.*\/ansible-eos-([^/\s]+)\/test/arista-ansible-role-test$', HERE).group(1)
RUN_CONFIG_BACKUP = '_eos_role_test_{}_running'.format(ROLE)
START_CONFIG_BACKUP = '_eos_role_test_{}_startup'.format(ROLE)

EOS_ROLE_PLAYBOOK = 'test/arista-ansible-role-test/eos_role.yml'
EOS_MODULE_PLAYBOOK = 'test/arista-ansible-role-test/eos_module.yml'

LOG_FILE = '{}/roletest.log'.format(HERE)
try:
    os.remove(LOG_FILE)
except OSError:
    pass
LOG = open(LOG_FILE, 'w')

SEPARATOR = '    ' + '*' * 50

class TestCase(object):
    def __init__(self, **kwargs):
        self.name = kwargs['name']
        self.module = kwargs['module']

        self.host = None

        self.inventory = kwargs.get('inventory')
        self.exitcode = kwargs.get('exitcode', 0)
        self.idempotent = kwargs.get('idempotent', True)
        self.changed = kwargs.get('changed', True)
        self.present = kwargs.get('present')
        self.absent = kwargs.get('absent')

        self.arguments = kwargs.get('arguments', list())
        self.variables = dict()

        # optional properties
        self.setup = kwargs.get('setup', list())
        self.teardown = kwargs.get('teardown', list())

    def __str__(self):
        return self.name

class TestModule(object):
    def __init__(self, testcase):
        self.testcase = testcase
        self.description = 'Test [%s]: %s' % (testcase.module, testcase.name)

    def __call__(self):
        self.output('Run first pass')
        response = self.run_module()
        for device in response:
            hostname = device.keys()[0]
            reported = int(device[hostname]['changed'])
            expected = int(self.testcase.changed)
            msg = ("First pass role execution reported {} task change(s), "
                   "expected {}".format(reported, expected))
            self.output(msg)
            assert reported == expected, msg

        if self.testcase.idempotent:
            self.output('Run second pass')
            response = self.run_module()
            for device in response:
                hostname = device.keys()[0]
                reported = int(device[hostname]['changed'])
                msg = (
                    "Second pass role execution reported {} task change(s), "
                    "expected 0".format(reported))
                self.output(msg)
                assert not reported, msg

        if self.testcase.present:
            desc = 'Validate present configuration'
            self.output(desc)
            response = self.run_validation(self.testcase.present, desc=desc)
            for device in response:
                hostname = device.keys()[0]
                # Result should contain an empty list of updates
                delim = " ---\n"
                updates = device[hostname]['updates']
                msg = ("{} - Expected configuration\n{}{}\n{}not found "
                       "on device '{}'".format(desc, delim,
                                               '\n'.join(updates), delim,
                                               hostname))
                assert device[hostname]['updates'] == [], msg
                # Result should show no changes
                msg = ("{} - Device '{}' reported no updates, but "
                       "returned 'changed'".format(desc, hostname))
                assert device[hostname]['changed'] == False, msg

        if self.testcase.absent:
            desc = 'Validate absent configuration'
            self.output(desc)
            response = self.run_validation(self.testcase.absent, desc=desc)
            for device in response:
                hostname = device.keys()[0]
                # Result should show change has taken place
                msg = (
                    "{} - Entire absent configuration found on device '{}'".
                    format(desc, hostname)
                )
                assert device[hostname]['changed'] == True, msg
                # Compare changes with expected values, sorted at global level
                updates = '\n'.join(device[hostname]['updates'])
                updates = re.split(r'\n(?=\S)', updates)
                updates = '\n'.join(sorted(updates))
                absent = re.split(r'\n(?=\S)', self.testcase.absent.rstrip())
                absent = '\n'.join(sorted(absent))
                msg = ("{} - Some part of absent configuration found "
                       "on device '{}'".format(desc, hostname))
                assert updates == absent, msg

    def setUp(self):
        print("\n{}\n".format(SEPARATOR) +
              "  See run log for complete output:\n  {}".format(LOG_FILE) +
              "\n{}\n".format(SEPARATOR))

        LOG.write("\n\n\n{}\n".format(SEPARATOR) +
                  "  Begin log for {}".format(self.description) +
                  "\n{}\n\n".format(SEPARATOR))

        if self.testcase.setup:
            self.output('Running test case setup commands')
            setup_cmds = self.testcase.setup
            if not isinstance(setup_cmds, list):
                setup_cmds = setup_cmds.splitlines()
            self.output("{}".format(setup_cmds))

            args = {
                'module': 'eos_command',
                'description': 'Run test case setup commands',
                'cmds': ['configure terminal'] + setup_cmds,
            }

            arguments = [json.dumps(args)]

            ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                                  arguments=arguments)

            if ret_code != 0:
                LOG.write("Playbook stdout:\n\n{}".format(out))
                LOG.write("Playbook stderr:\n\n{}".format(err))
                raise

    def tearDown(self):
        if self.testcase.teardown:
            self.output('Running test case teardown commands')
            teardown_cmds = self.testcase.teardown
            if not isinstance(teardown_cmds, list):
                teardown_cmds = teardown_cmds.splitlines()
            self.output("{}\n".format(teardown_cmds))

            args = {
                'module': 'eos_command',
                'description': 'Run test case teardown_cmds commands',
                'cmds': ['configure terminal'] + teardown_cmds,
            }

            arguments = [json.dumps(args)]

            ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                                  arguments=arguments)

            if ret_code != 0:
                self.output("Playbook stdout:\n\n{}".format(out))
                self.output("Playbook stderr:\n\n{}".format(err))
                warnings.warn("\nError in test case teardown\n\n{}".format(
                    out))

    @classmethod
    def output(cls, text):
        print '>>', str(text)
        LOG.write('++ {}'.format(text) + '\n')

    def run_module(self):
        (retcode, out, _) = self.execute_module()
        out_stripped = re.sub(r'\"config\": \"! Command:.*\\nend\"',
                              '\"config\": \"--- stripped for space ---\"',
                              out)
        LOG.write("PLaybook stdout:\n\n{}".format(out_stripped))
        msg = "Return code: {}, Expected code: {}".format(retcode, self.testcase.exitcode)
        self.output(msg)
        assert retcode == self.testcase.exitcode, msg
        return self.parse_response(out)

    def execute_module(self):
        arguments = [json.dumps(self.testcase.arguments)]
        arguments.append(json.dumps(
            {'rolename': "ansible-eos-{}".format(ROLE)}))
        return ansible_playbook(EOS_ROLE_PLAYBOOK, arguments=arguments)

    def parse_response(self, output, validate=False):
        # Get all the lines after the 'PLAY RECAP ****...' header
        lines = re.sub(r'^.*PLAY RECAP \*+', '', output, 0, re.S).split('\n')
        # Remove any empty lines from the list
        lines = [x for x in lines if x]

        recap = []
        for line in lines:
            match = re.search(r'^(\S+)\s+\:\s+' \
                              r'ok=(\d+)\s+' \
                              r'changed=(\d+)\s+' \
                              r'unreachable=(\d+)\s+' \
                              r'failed=(\d+)', line)
            if not match:
                self.output("Playbook stdout:\n\n{}".format(output))
                raise ValueError("Unable to parse Ansible output for "
                                 "recap information")
            (name, okcount, changed, unreach, failed) = match.groups()
            recap.append({name: {'ok': okcount,
                                 'changed': changed,
                                 'unreachable': unreach,
                                 'failed': failed}})

        if not validate:
            return recap

        updates = []
        for device in recap:
            hostname = device.keys()[0]
            match = re.search(r'(?<!skipping: )\[%s\] => (\{.*\})' % hostname,
                              output, re.M)
            if not match:
                self.output("Playbook stdout:\n\n{}".format(output))
                raise ValueError("Unable to parse Ansible output for "
                                 "result validation")
            result = json.loads(match.group(1))
            updates.append({hostname: result})

        return updates

    def run_validation(self, src, desc='Validate configuration'):
        args = {'module': 'eos_template', 'description': desc, 'src': src, }
        arguments = [json.dumps(args)]
        (ret_code, out, _) = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                              arguments=arguments,
                                              options=['--check'])
        LOG.write(out)
        assert ret_code == 0, "Validation playbook failed execution"
        return self.parse_response(out, validate=True)

def filter_modules(modules, filenames):
    if modules:
        modules = ['{0}.yml'.format(s) for s in modules.split(',')]
        return list(set(modules).intersection(filenames))
    return filenames

def setup():
    print >> sys.stderr, "Test Suite Setup:"

    run_backup = "  Backing up running-config on nodes ..."
    print >> sys.stderr, run_backup
    LOG.write('++ ' + run_backup.strip())
    args = {
        'module': 'eos_command',
        'description': 'Back up running-config on node',
        'cmds': [
            'configure terminal',
            'copy running-config {}'.format(RUN_CONFIG_BACKUP)
        ],
    }
    arguments = [json.dumps(args)]

    ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                          arguments=arguments)

    if ret_code != 0:
        LOG.write(str(">> ansible-playbook {} stdout:\n".format(EOS_MODULE_PLAYBOOK), out))
        LOG.write(str(">> ansible-playbook {} stddrr:\n".format(EOS_MODULE_PLAYBOOK), err))
        teardown()
        raise RuntimeError("Error in Test Suite Setup")

    run_backup = "  Backing up startup-config on nodes ..."
    print >> sys.stderr, run_backup
    LOG.write('++ ' + run_backup.strip())
    args = {
        'module': 'eos_command',
        'description': 'Back up startup-config on node',
        'cmds': [
            'configure terminal',
            'copy startup-config {}'.format(START_CONFIG_BACKUP)
        ],
    }
    arguments = [json.dumps(args)]

    ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                          arguments=arguments)

    if ret_code != 0:
        LOG.write(str(">> ansible-playbook {} stdout:\n".format(EOS_MODULE_PLAYBOOK), out))
        LOG.write(str(">> ansible-playbook {} stddrr:\n".format(EOS_MODULE_PLAYBOOK), err))
        teardown()
        raise RuntimeError("Error in Test Suite Setup")

    print >> sys.stderr, "  Gathering test cases ..."

    modules = os.environ.get('ANSIBLE_ROLE_TEST_CASES')

    testcases_home = os.path.join(HERE, 'testcases')
    if not os.path.exists(testcases_home):
        print >> sys.stderr, "\n  ***** Testcase directory not found!! *****"
        teardown()
        raise RuntimeError("Testcase path '{}' does not exist".format(testcases_home))

    filenames = os.listdir(testcases_home)

    for module in filter_modules(modules, filenames):
        path = os.path.join(testcases_home, module)
        definition = yaml.load(open(path))

        defaults = definition.get('defaults', {})
        testcases = definition.get('testcases', [])
        if not testcases:
            print >> sys.stderr, ("\n  ***** No testcases defined in "
                                  "module {} *****\n".format(module))
        else:
            for testcase in definition.get('testcases', []):
                kwargs = defaults.copy()
                kwargs.update(testcase)
                TESTCASES.append(TestCase(**kwargs))

    print >> sys.stderr, "  Setup complete\n"

def teardown():
    print >> sys.stderr, "\nTest Suite Teardown:"

    no_teardown = os.environ.get('NO_ANSIBLE_ROLE_TEST_TEARDOWN')

    if no_teardown:
        print >> sys.stderr, ("{}\n"
                              "  Skipping test suite teardown due to "
                              "NO_ANSIBLE_ROLE_TEST_TEARDOWN\n"
                              "  To restore each device to pre-test state "
                              "execute the following commands\n"
                              "  - configure terminal\n"
                              "  - configure replace {}\n"
                              "  - delete {}\n"
                              "  - copy {} startup-config\n"
                              "  - delete {}\n"
                              "{}".format(SEPARATOR, RUN_CONFIG_BACKUP,
                                          RUN_CONFIG_BACKUP,
                                          START_CONFIG_BACKUP,
                                          START_CONFIG_BACKUP, SEPARATOR))
    else:
        # Restore the running-config on the nodes
        # ---------------------------------------
        restore_backup = "  Restoring running-config on nodes ..."
        print >> sys.stderr, restore_backup
        LOG.write('++ ' + restore_backup.strip())
        args = {
            'module': 'eos_command',
            'description': 'Restore running-config from backup',
            'cmds': [
                'configure terminal',
                'configure replace {}'.format(RUN_CONFIG_BACKUP),
                'delete {}'.format(RUN_CONFIG_BACKUP),
            ],
        }
        arguments = [json.dumps(args)]

        # ret_code, out, err = ansible_playbook(CMD_PLAY, arguments=arguments)
        ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                              arguments=arguments)

        if ret_code != 0:
            msg = "Error restoring running-config on nodes\n" \
                  "Running ansible-playbook {} -e {}\n" \
                  ">> stdout: {}\n" \
                  ">> stderr: {}\n".format(EOS_MODULE_PLAYBOOK, arguments, out, err)
            warnings.warn(msg)

        # Restore the startup-config on the nodes
        # ---------------------------------------
        restore_backup = "  Restoring startup-config on nodes ..."
        print >> sys.stderr, restore_backup
        LOG.write('++ ' + restore_backup.strip())
        args = {
            'module': 'eos_command',
            'description': 'Restore startup-config from backup',
            'cmds': [
                'configure terminal',
                'copy {} startup-config'.format(START_CONFIG_BACKUP),
                'delete {}'.format(START_CONFIG_BACKUP),
            ],
        }
        arguments = [json.dumps(args)]

        # ret_code, out, err = ansible_playbook(CMD_PLAY, arguments=arguments)
        ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                              arguments=arguments)

        if ret_code != 0:
            msg = "Error restoring startup-config on nodes\n" \
                  "Running ansible-playbook {} -e {}\n" \
                  ">> stdout: {}\n" \
                  ">> stderr: {}\n".format(EOS_MODULE_PLAYBOOK, arguments, out, err)
            warnings.warn(msg)

    print >> sys.stderr, "  Teardown complete"

def test_module():
    for testcase in TESTCASES:
        yield TestModule(testcase)

def ansible_playbook(playbook, arguments=None, options=None):
    if arguments is None:
        arguments = []
    if options is None:
        options = []

    command = ['ansible-playbook']

    command.append(playbook)
    command.extend(['-i', INVENTORY])
    for arg in arguments:
        command.extend(['-e', arg])
    for opt in options:
        command.append(opt)
    command.append('-vvv')

    # Format the command string for output on error - for easier
    # copy/paste for manual run
    cmdstr = ''
    for segment in command:
        if segment[0] == '{':
            cmdstr = cmdstr + "\'{}\' ".format(segment)
        else:
            cmdstr = cmdstr + "{} ".format(segment)
    LOG.write("-- Ansible playbook command:\n-- {}\n".format(cmdstr))

    stdout = subprocess.PIPE
    stderr = subprocess.PIPE
    proc = subprocess.Popen(command, stdout=stdout, stderr=stderr)
    out, err = proc.communicate()

    return (proc.returncode, out, err)
<commit_msg>Squashed 'test/arista-ansible-role-test/' changes from 2c77a57..80011bd<commit_after># pylint: disable=invalid-name
# pylint: disable=missing-docstring
# pylint: disable=too-few-public-methods
# pylint: disable=too-many-instance-attributes
# pylint: disable=too-many-locals

import json
import os
import re
import subprocess
import sys
import warnings
import yaml

TESTCASES = list()
INVENTORY = 'test/fixtures/hosts'

HERE = os.path.abspath(os.path.dirname(__file__))
ROLE = re.match(
    r'^.*\/ansible-eos-([^/\s]+)\/test/arista-ansible-role-test$', HERE).group(1)
RUN_CONFIG_BACKUP = '_eos_role_test_{}_running'.format(ROLE)
START_CONFIG_BACKUP = '_eos_role_test_{}_startup'.format(ROLE)

EOS_ROLE_PLAYBOOK = 'test/arista-ansible-role-test/eos_role.yml'
EOS_MODULE_PLAYBOOK = 'test/arista-ansible-role-test/eos_module.yml'

LOG_FILE = '{}/roletest.log'.format(HERE)
try:
    os.remove(LOG_FILE)
except OSError:
    pass
LOG = open(LOG_FILE, 'w')

SEPARATOR = '    ' + '*' * 50

class TestCase(object):
    def __init__(self, **kwargs):
        self.name = kwargs['name']
        self.module = kwargs['module']

        self.host = None

        self.inventory = kwargs.get('inventory')
        self.exitcode = kwargs.get('exitcode', 0)
        self.idempotent = kwargs.get('idempotent', True)
        self.changed = kwargs.get('changed', True)
        self.present = kwargs.get('present')
        self.absent = kwargs.get('absent')

        self.arguments = kwargs.get('arguments', list())
        self.variables = dict()

        # optional properties
        self.setup = kwargs.get('setup', list())
        self.teardown = kwargs.get('teardown', list())

    def __str__(self):
        return self.name

class TestModule(object):
    def __init__(self, testcase):
        self.testcase = testcase
        self.description = 'Test [%s]: %s' % (testcase.module, testcase.name)

    def __call__(self):
        self.output('Run first pass')
        response = self.run_module()
        for device in response:
            hostname = device.keys()[0]
            reported = int(device[hostname]['changed'])
            expected = int(self.testcase.changed)
            msg = ("First pass role execution reported {} task change(s), "
                   "expected {}".format(reported, expected))
            self.output(msg)
            assert reported == expected, msg

        if self.testcase.idempotent:
            self.output('Run second pass')
            response = self.run_module()
            for device in response:
                hostname = device.keys()[0]
                reported = int(device[hostname]['changed'])
                msg = (
                    "Second pass role execution reported {} task change(s), "
                    "expected 0".format(reported))
                self.output(msg)
                assert not reported, msg

        if self.testcase.present:
            desc = 'Validate present configuration'
            self.output(desc)
            response = self.run_validation(self.testcase.present, desc=desc)
            for device in response:
                hostname = device.keys()[0]
                # Result should contain an empty list of updates
                delim = " ---\n"
                updates = device[hostname]['updates']
                msg = ("{} - Expected configuration\n{}{}\n{}not found "
                       "on device '{}'".format(desc, delim,
                                               '\n'.join(updates), delim,
                                               hostname))
                assert device[hostname]['updates'] == [], msg
                # Result should show no changes
                msg = ("{} - Device '{}' reported no updates, but "
                       "returned 'changed'".format(desc, hostname))
                assert device[hostname]['changed'] == False, msg

        if self.testcase.absent:
            desc = 'Validate absent configuration'
            self.output(desc)
            response = self.run_validation(self.testcase.absent, desc=desc)
            for device in response:
                hostname = device.keys()[0]
                # Result should show change has taken place
                msg = (
                    "{} - Entire absent configuration found on device '{}'".
                    format(desc, hostname)
                )
                assert device[hostname]['changed'] == True, msg
                # Compare changes with expected values, sorted at global level
                updates = '\n'.join(device[hostname]['updates'])
                updates = re.split(r'\n(?=\S)', updates)
                updates = '\n'.join(sorted(updates))
                absent = re.split(r'\n(?=\S)', self.testcase.absent.rstrip())
                absent = '\n'.join(sorted(absent))
                msg = ("{} - Some part of absent configuration found "
                       "on device '{}'".format(desc, hostname))
                assert updates == absent, msg

    def setUp(self):
        print("\n{}\n".format(SEPARATOR) +
              "  See run log for complete output:\n  {}".format(LOG_FILE) +
              "\n{}\n".format(SEPARATOR))

        LOG.write("\n\n\n{}\n".format(SEPARATOR) +
                  "  Begin log for {}".format(self.description) +
                  "\n{}\n\n".format(SEPARATOR))

        if self.testcase.setup:
            self.output('Running test case setup commands')
            setup_cmds = self.testcase.setup
            if not isinstance(setup_cmds, list):
                setup_cmds = setup_cmds.splitlines()
            self.output("{}".format(setup_cmds))

            args = {
                'module': 'eos_command',
                'description': 'Run test case setup commands',
                'cmds': ['configure terminal'] + setup_cmds,
            }

            arguments = [json.dumps(args)]

            ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                                  arguments=arguments)

            if ret_code != 0:
                LOG.write("Playbook stdout:\n\n{}".format(out))
                LOG.write("Playbook stderr:\n\n{}".format(err))
                raise

    def tearDown(self):
        if self.testcase.teardown:
            self.output('Running test case teardown commands')
            teardown_cmds = self.testcase.teardown
            if not isinstance(teardown_cmds, list):
                teardown_cmds = teardown_cmds.splitlines()
            self.output("{}\n".format(teardown_cmds))

            args = {
                'module': 'eos_command',
                'description': 'Run test case teardown_cmds commands',
                'cmds': ['configure terminal'] + teardown_cmds,
            }

            arguments = [json.dumps(args)]

            ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                                  arguments=arguments)

            if ret_code != 0:
                self.output("Playbook stdout:\n\n{}".format(out))
                self.output("Playbook stderr:\n\n{}".format(err))
                warnings.warn("\nError in test case teardown\n\n{}".format(
                    out))

    @classmethod
    def output(cls, text):
        print '>>', str(text)
        LOG.write('++ {}'.format(text) + '\n')

    def run_module(self):
        (retcode, out, _) = self.execute_module()
        out_stripped = re.sub(r'\"config\": \"! Command:.*\\nend\"',
                              '\"config\": \"--- stripped for space ---\"',
                              out)
        LOG.write("PLaybook stdout:\n\n{}".format(out_stripped))
        msg = "Return code: {}, Expected code: {}".format(retcode, self.testcase.exitcode)
        self.output(msg)
        assert retcode == self.testcase.exitcode, msg
        return self.parse_response(out)

    def execute_module(self):
        arguments = [json.dumps(self.testcase.arguments)]
        arguments.append(json.dumps(
            {'rolename': "ansible-eos-{}".format(ROLE)}))
        return ansible_playbook(EOS_ROLE_PLAYBOOK, arguments=arguments)

    def parse_response(self, output, validate=False):
        # Get all the lines after the 'PLAY RECAP ****...' header
        lines = re.sub(r'^.*PLAY RECAP \*+', '', output, 0, re.S).split('\n')
        # Remove any empty lines from the list
        lines = [x for x in lines if x]

        recap = []
        for line in lines:
            match = re.search(r'^(\S+)\s+\:\s+' \
                              r'ok=(\d+)\s+' \
                              r'changed=(\d+)\s+' \
                              r'unreachable=(\d+)\s+' \
                              r'failed=(\d+)', line)
            if not match:
                self.output("Playbook stdout:\n\n{}".format(output))
                raise ValueError("Unable to parse Ansible output for "
                                 "recap information")
            (name, okcount, changed, unreach, failed) = match.groups()
            recap.append({name: {'ok': okcount,
                                 'changed': changed,
                                 'unreachable': unreach,
                                 'failed': failed}})

        if not validate:
            return recap

        updates = []
        for device in recap:
            hostname = device.keys()[0]
            match = re.search(r'(?<!skipping: )\[%s\] => (\{.*\})' % hostname,
                              output, re.M)
            if not match:
                self.output("Playbook stdout:\n\n{}".format(output))
                raise ValueError("Unable to parse Ansible output for "
                                 "result validation")
            result = json.loads(match.group(1))
            updates.append({hostname: result})

        return updates

    def run_validation(self, src, desc='Validate configuration'):
        args = {'module': 'eos_template', 'description': desc, 'src': src, }
        arguments = [json.dumps(args)]
        (ret_code, out, _) = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                              arguments=arguments,
                                              options=['--check'])
        LOG.write(out)
        assert ret_code == 0, "Validation playbook failed execution"
        return self.parse_response(out, validate=True)

def filter_modules(modules, filenames):
    if modules:
        modules = ['{0}.yml'.format(s) for s in modules.split(',')]
        return list(set(modules).intersection(filenames))
    return filenames

def setup():
    print >> sys.stderr, "Test Suite Setup:"

    run_backup = "  Backing up running-config on nodes ..."
    print >> sys.stderr, run_backup
    LOG.write('++ ' + run_backup.strip())
    args = {
        'module': 'eos_command',
        'description': 'Back up running-config on node',
        'cmds': [
            'configure terminal',
            'copy running-config {}'.format(RUN_CONFIG_BACKUP)
        ],
    }
    arguments = [json.dumps(args)]

    ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                          arguments=arguments)

    if ret_code != 0:
        LOG.write(">> ansible-playbook {} stdout:\n{}".format(EOS_MODULE_PLAYBOOK, out))
        LOG.write(">> ansible-playbook {} stddrr:\n{}".format(EOS_MODULE_PLAYBOOK, err))
        teardown()
        raise RuntimeError("Error in Test Suite Setup")

    run_backup = "  Backing up startup-config on nodes ..."
    print >> sys.stderr, run_backup
    LOG.write('++ ' + run_backup.strip())
    args = {
        'module': 'eos_command',
        'description': 'Back up startup-config on node',
        'cmds': [
            'configure terminal',
            'copy startup-config {}'.format(START_CONFIG_BACKUP)
        ],
    }
    arguments = [json.dumps(args)]

    ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                          arguments=arguments)

    if ret_code != 0:
        LOG.write(">> ansible-playbook {} stdout:\n{}".format(EOS_MODULE_PLAYBOOK, out))
        LOG.write(">> ansible-playbook {} stddrr:\n{}".format(EOS_MODULE_PLAYBOOK, err))
        teardown()
        raise RuntimeError("Error in Test Suite Setup")

    print >> sys.stderr, "  Gathering test cases ..."

    modules = os.environ.get('ANSIBLE_ROLE_TEST_CASES')

    testcases_home = os.path.join(HERE, 'testcases')
    if not os.path.exists(testcases_home):
        print >> sys.stderr, "\n  ***** Testcase directory not found!! *****"
        teardown()
        raise RuntimeError("Testcase path '{}' does not exist".format(testcases_home))

    filenames = os.listdir(testcases_home)

    for module in filter_modules(modules, filenames):
        path = os.path.join(testcases_home, module)
        definition = yaml.load(open(path))

        defaults = definition.get('defaults', {})
        testcases = definition.get('testcases', [])
        if not testcases:
            print >> sys.stderr, ("\n  ***** No testcases defined in "
                                  "module {} *****\n".format(module))
        else:
            for testcase in definition.get('testcases', []):
                kwargs = defaults.copy()
                kwargs.update(testcase)
                TESTCASES.append(TestCase(**kwargs))

    print >> sys.stderr, "  Setup complete\n"

def teardown():
    print >> sys.stderr, "\nTest Suite Teardown:"

    no_teardown = os.environ.get('NO_ANSIBLE_ROLE_TEST_TEARDOWN')

    if no_teardown:
        print >> sys.stderr, ("{}\n"
                              "  Skipping test suite teardown due to "
                              "NO_ANSIBLE_ROLE_TEST_TEARDOWN\n"
                              "  To restore each device to pre-test state "
                              "execute the following commands\n"
                              "  - configure terminal\n"
                              "  - configure replace {}\n"
                              "  - delete {}\n"
                              "  - copy {} startup-config\n"
                              "  - delete {}\n"
                              "{}".format(SEPARATOR, RUN_CONFIG_BACKUP,
                                          RUN_CONFIG_BACKUP,
                                          START_CONFIG_BACKUP,
                                          START_CONFIG_BACKUP, SEPARATOR))
    else:
        # Restore the running-config on the nodes
        # ---------------------------------------
        restore_backup = "  Restoring running-config on nodes ..."
        print >> sys.stderr, restore_backup
        LOG.write('++ ' + restore_backup.strip())
        args = {
            'module': 'eos_command',
            'description': 'Restore running-config from backup',
            'cmds': [
                'configure terminal',
                'configure replace {}'.format(RUN_CONFIG_BACKUP),
                'delete {}'.format(RUN_CONFIG_BACKUP),
            ],
        }
        arguments = [json.dumps(args)]

        # ret_code, out, err = ansible_playbook(CMD_PLAY, arguments=arguments)
        ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                              arguments=arguments)

        if ret_code != 0:
            msg = "Error restoring running-config on nodes\n" \
                  "Running ansible-playbook {} -e {}\n" \
                  ">> stdout: {}\n" \
                  ">> stderr: {}\n".format(EOS_MODULE_PLAYBOOK, arguments, out, err)
            warnings.warn(msg)

        # Restore the startup-config on the nodes
        # ---------------------------------------
        restore_backup = "  Restoring startup-config on nodes ..."
        print >> sys.stderr, restore_backup
        LOG.write('++ ' + restore_backup.strip())
        args = {
            'module': 'eos_command',
            'description': 'Restore startup-config from backup',
            'cmds': [
                'configure terminal',
                'copy {} startup-config'.format(START_CONFIG_BACKUP),
                'delete {}'.format(START_CONFIG_BACKUP),
            ],
        }
        arguments = [json.dumps(args)]

        # ret_code, out, err = ansible_playbook(CMD_PLAY, arguments=arguments)
        ret_code, out, err = ansible_playbook(EOS_MODULE_PLAYBOOK,
                                              arguments=arguments)

        if ret_code != 0:
            msg = "Error restoring startup-config on nodes\n" \
                  "Running ansible-playbook {} -e {}\n" \
                  ">> stdout: {}\n" \
                  ">> stderr: {}\n".format(EOS_MODULE_PLAYBOOK, arguments, out, err)
            warnings.warn(msg)

    print >> sys.stderr, "  Teardown complete"

def test_module():
    for testcase in TESTCASES:
        yield TestModule(testcase)

def ansible_playbook(playbook, arguments=None, options=None):
    if arguments is None:
        arguments = []
    if options is None:
        options = []

    command = ['ansible-playbook']

    command.append(playbook)
    command.extend(['-i', INVENTORY])
    for arg in arguments:
        command.extend(['-e', arg])
    for opt in options:
        command.append(opt)
    command.append('-vvv')

    # Format the command string for output on error - for easier
    # copy/paste for manual run
    cmdstr = ''
    for segment in command:
        if segment[0] == '{':
            cmdstr = cmdstr + "\'{}\' ".format(segment)
        else:
            cmdstr = cmdstr + "{} ".format(segment)
    LOG.write("-- Ansible playbook command:\n-- {}\n".format(cmdstr))

    stdout = subprocess.PIPE
    stderr = subprocess.PIPE
    proc = subprocess.Popen(command, stdout=stdout, stderr=stderr)
    out, err = proc.communicate()

    return (proc.returncode, out, err)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# coding:utf-8
###############################################################################
# MIT License
#
# Copyright (c) 2016 Hajime Nakagami
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
###############################################################################
import sys
import unittest
import datetime
import time
import nmongo
try:
    from decimal import Decimal
except ImportError:
    from nmongo import Decimal

class TestMongo(unittest.TestCase):
    host = 'localhost'
    database = 'test_nmongo'
    port = 27017

    def assertEqualDict(self, d1, d2):
        self.assertEqual(set(d1.keys()), set(d2.keys()))
        for k,v in d1.items():
            self.assertEqual(d2.get(k), v)

    def test_nmongo(self):
        db = nmongo.connect(self.host, self.database,  port=self.port)
        r = db.pets.drop()
        mongo_version = [int(n) for n in db.serverBuildInfo()['version'].split('.')]

        data1 = {
            'name': 'Kitty',
            'gender': 'f',
            'age': 0,
            'species': 'cat',
        }
        if mongo_version > [3, 2]:
            data1['weight'] = Decimal("123.4")

        if sys.implementation.name != 'micropython':
            data1['birth'] = datetime.datetime(1974, 11, 1, 1, 1)
        data2 = {
            'name': 'Snoopy',
            'gender': 'm',
            'age': 0,
            'species': 'cat',
        }
        if mongo_version > [3, 2]:
            data2['weight'] = Decimal("1234.0")
        data3 = {
            'name': 'Kuri',
            'gender': 'm',
            'age': 0,
            'species': 'ferret',
        }
        if mongo_version > [3, 2]:
            data3['weight'] = Decimal("NaN")

        # Create
        db.pets.insert(data1)
        db.pets.insert([data2, data3])
        self.assertIn('pets', db.getCollectionNames())

        # index
        db.pets.createIndex(
            {'name': 1, 'gender': -1},
            options={'name': 'named_pets_index'}
        )
        db.pets.createIndex(
            {'name': -1},
            options={'name': 'desc_name_pets_index'}
        )
        self.assertIn(
            'named_pets_index',
            [idx['name'] for idx in db.pets.getIndexes()]
        )
        db.pets.dropIndex('named_pets_index')
        self.assertTrue('named_pets_index' not in [idx['name'] for idx in db.pets.getIndexes()])
        self.assertEqual(len(db.pets.getIndexes()), 2)

        db.pets.dropIndexes()
        self.assertEqual(len(db.pets.getIndexes()), 1)

        # collection methods
        self.assertTrue(db.pets.stats()['ok'])
        db.createCollection('testcapped', {'capped': True, 'size': 1024})
        self.assertTrue(db.testcapped.isCapped())

        # database methods
        self.assertTrue(db.serverStatus()['ok'])
        self.assertEqual(db.stats()['db'], self.database)

        # mapReduce
        self.assertTrue(
            db.pets.mapReduce(
                "function(){}",             # map
                "function(key, values){}",  # reduce
                {'out': {'inline': 1}}
            )['ok']
        )

        # group
        self.assertTrue(
            db.pets.group(
                {'name': 1, 'gender': 1},     # key
                "function (c, r) {}",       # reduce
                {},                         # initial
                cond={'gender': 'm'},
            )['ok']
        )

        # Read
        cur = db.pets.find(projection={'_id': 0})
        self.assertEqualDict(cur.fetchone(), data1)

        self.assertEqual(db.pets.count(), 3)

        self.assertEqual(set(db.pets.distinct('gender')), set(['m', 'f']))

        # findAndModify
        prev_data1 = db.pets.findAndModify(
            query={'name': 'Kitty'},
            update={'$inc': {'age': 1}},
        )
        next_data1 = db.pets.findOne({'name': 'Kitty'})
        self.assertEqual(prev_data1['age'] + 1, next_data1['age'])

        # Update
        r = db.pets.update(
            {'name': 'Snoopy'},
            {'name': 'Snoopy', 'gender': 'f', 'age': 10, 'species': 'wolf'}
        )
        self.assertTrue(r['ok'])
        self.assertEqual(r['nModified'], 1)
        self.assertEqualDict(
            db.pets.findOne({'name': 'Snoopy'}, projection={'_id': 0}),
            {'name': 'Snoopy', 'gender': 'f', 'age': 10, 'species': 'wolf'},
        )

        r = db.pets.updateOne(
            {'name': 'Snoopy'},          # query
            {'$set': {'gender': 'm'}, '$inc': {'age': 1}},  # update
        )
        self.assertTrue(r['ok'])
        self.assertEqual(r['nModified'], 1)
        self.assertEqual(
            db.pets.findOne({'name': 'Snoopy'}, projection={'_id': 0}),
            {'name': 'Snoopy', 'gender': 'm', 'age': 11, 'species': 'wolf'},
        )

        self.assertEqual(db.pets.count(), 3)
        r = db.pets.updateMany(
            {},                     # query
            {'$inc': {'age': 1}},   # update
        )
        self.assertTrue(r['ok'])
        self.assertEqual(r['n'], 3)
        self.assertEqual(r['nModified'], 3)
        r = db.pets.updateMany(
            {'name': {'$eq': 'Kitty'}},     # query
            {'$set': {'gender': 'f'}},      # update
        )
        self.assertTrue(r['ok'])
        self.assertEqual(r['n'], 1)
        self.assertEqual(r['nModified'], 0)

        # Delete
        self.assertEqual(db.pets.count(), 3)
        db.pets.insert(data1)
        self.assertEqual(db.pets.count(), 4)
        self.assertEqual(db.pets.remove({'name': 'Kitty', 'age': 0}), 1)
        self.assertEqual(db.pets.count(), 3)
        self.assertEqual(db.pets.findOneAndDelete({'name': 'Kitty'})['age'], 2)
        self.assertEqual(db.pets.count(), 2)

        # insertOne, insertMany
        db.pets.drop()
        oid = db.pets.insertOne(data1)
        self.assertEqual(
            db.pets.find({'_id': oid}).fetchone()['name'],
            data1['name']
        )
        oids = db.pets.insertMany([data2, data3])
        self.assertEqual(
            db.pets.find({'_id': oids[0]}).fetchone()['name'],
            data2['name']
        )

        db.close()

    def test_decimal(self):
        datum = [
            [100, (0, (1, 0, 0), 0), '100'],
            [-100, (1, (1, 0, 0), 0), '-100'],
            ['100', (0, (1, 0, 0), 0), '100'],
            ['-100', (1, (1, 0, 0), 0), '-100'],
            ['12.3456789', (0, (1, 2, 3, 4, 5, 6, 7, 8, 9), -7), '12.3456789'],
            ['123.456789', (0, (1, 2, 3, 4, 5, 6, 7, 8, 9), -6), '123.456789'],
            ['1234.56789', (0, (1, 2, 3, 4, 5, 6, 7, 8, 9), -5), '1234.56789'],
            ['-12.3456789', (1, (1, 2, 3, 4, 5, 6, 7, 8, 9), -7), '-12.3456789'],
            ['-123.456789', (1, (1, 2, 3, 4, 5, 6, 7, 8, 9), -6), '-123.456789'],
            ['-1234.56789', (1, (1, 2, 3, 4, 5, 6, 7, 8, 9), -5), '-1234.56789'],
            ['NaN', (0, (), 'n'), 'NaN'],
            ['-NaN', (1, (), 'n'), '-NaN'],
            ['sNaN', (0, (), 'N'), 'sNaN'],
            ['-sNaN', (1, (), 'N'), '-sNaN'],
            ['Infinity', (0, (0, ), 'F'), 'Infinity'],
            ['-Infinity', (1, (0, ), 'F'), '-Infinity'],
            ['Inf', (0, (0, ), 'F'), 'Infinity'],
            ['-Inf', (1, (0, ), 'F'), '-Infinity'],
        ]
        for data in datum:
            self.assertEqual(tuple(Decimal(data[0]).as_tuple()), data[1])
            self.assertEqual(str(Decimal(data[0])), data[2])

if __name__ == "__main__":
    unittest.main()
<commit_msg>fix test for mongo3.2<commit_after>#!/usr/bin/env python3
# coding:utf-8
###############################################################################
# MIT License
#
# Copyright (c) 2016 Hajime Nakagami
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
###############################################################################
import sys
import unittest
import datetime
import time
import nmongo
try:
    from decimal import Decimal
except ImportError:
    from nmongo import Decimal

class TestMongo(unittest.TestCase):
    host = 'localhost'
    database = 'test_nmongo'
    port = 27017

    def assertEqualDict(self, d1, d2):
        self.assertEqual(set(d1.keys()), set(d2.keys()))
        for k,v in d1.items():
            self.assertEqual(d2.get(k), v)

    def test_nmongo(self):
        db = nmongo.connect(self.host, self.database,  port=self.port)
        r = db.pets.drop()
        mongo_version = [int(n) for n in db.serverBuildInfo()['version'].split('.')][:2]

        data1 = {
            'name': 'Kitty',
            'gender': 'f',
            'age': 0,
            'species': 'cat',
        }
        if mongo_version > [3, 2]:
            data1['weight'] = Decimal("123.4")

        if sys.implementation.name != 'micropython':
            data1['birth'] = datetime.datetime(1974, 11, 1, 1, 1)
        data2 = {
            'name': 'Snoopy',
            'gender': 'm',
            'age': 0,
            'species': 'cat',
        }
        if mongo_version > [3, 2]:
            data2['weight'] = Decimal("1234.0")
        data3 = {
            'name': 'Kuri',
            'gender': 'm',
            'age': 0,
            'species': 'ferret',
        }
        if mongo_version > [3, 2]:
            data3['weight'] = Decimal("NaN")

        # Create
        db.pets.insert(data1)
        db.pets.insert([data2, data3])
        self.assertIn('pets', db.getCollectionNames())

        # index
        db.pets.createIndex(
            {'name': 1, 'gender': -1},
            options={'name': 'named_pets_index'}
        )
        db.pets.createIndex(
            {'name': -1},
            options={'name': 'desc_name_pets_index'}
        )
        self.assertIn(
            'named_pets_index',
            [idx['name'] for idx in db.pets.getIndexes()]
        )
        db.pets.dropIndex('named_pets_index')
        self.assertTrue('named_pets_index' not in [idx['name'] for idx in db.pets.getIndexes()])
        self.assertEqual(len(db.pets.getIndexes()), 2)

        db.pets.dropIndexes()
        self.assertEqual(len(db.pets.getIndexes()), 1)

        # collection methods
        self.assertTrue(db.pets.stats()['ok'])
        db.createCollection('testcapped', {'capped': True, 'size': 1024})
        self.assertTrue(db.testcapped.isCapped())

        # database methods
        self.assertTrue(db.serverStatus()['ok'])
        self.assertEqual(db.stats()['db'], self.database)

        # mapReduce
        self.assertTrue(
            db.pets.mapReduce(
                "function(){}",             # map
                "function(key, values){}",  # reduce
                {'out': {'inline': 1}}
            )['ok']
        )

        # group
        self.assertTrue(
            db.pets.group(
                {'name': 1, 'gender': 1},     # key
                "function (c, r) {}",       # reduce
                {},                         # initial
                cond={'gender': 'm'},
            )['ok']
        )

        # Read
        cur = db.pets.find(projection={'_id': 0})
        self.assertEqualDict(cur.fetchone(), data1)

        self.assertEqual(db.pets.count(), 3)

        self.assertEqual(set(db.pets.distinct('gender')), set(['m', 'f']))

        # findAndModify
        prev_data1 = db.pets.findAndModify(
            query={'name': 'Kitty'},
            update={'$inc': {'age': 1}},
        )
        next_data1 = db.pets.findOne({'name': 'Kitty'})
        self.assertEqual(prev_data1['age'] + 1, next_data1['age'])

        # Update
        r = db.pets.update(
            {'name': 'Snoopy'},
            {'name': 'Snoopy', 'gender': 'f', 'age': 10, 'species': 'wolf'}
        )
        self.assertTrue(r['ok'])
        self.assertEqual(r['nModified'], 1)
        self.assertEqualDict(
            db.pets.findOne({'name': 'Snoopy'}, projection={'_id': 0}),
            {'name': 'Snoopy', 'gender': 'f', 'age': 10, 'species': 'wolf'},
        )

        r = db.pets.updateOne(
            {'name': 'Snoopy'},          # query
            {'$set': {'gender': 'm'}, '$inc': {'age': 1}},  # update
        )
        self.assertTrue(r['ok'])
        self.assertEqual(r['nModified'], 1)
        self.assertEqual(
            db.pets.findOne({'name': 'Snoopy'}, projection={'_id': 0}),
            {'name': 'Snoopy', 'gender': 'm', 'age': 11, 'species': 'wolf'},
        )

        self.assertEqual(db.pets.count(), 3)
        r = db.pets.updateMany(
            {},                     # query
            {'$inc': {'age': 1}},   # update
        )
        self.assertTrue(r['ok'])
        self.assertEqual(r['n'], 3)
        self.assertEqual(r['nModified'], 3)
        r = db.pets.updateMany(
            {'name': {'$eq': 'Kitty'}},     # query
            {'$set': {'gender': 'f'}},      # update
        )
        self.assertTrue(r['ok'])
        self.assertEqual(r['n'], 1)
        self.assertEqual(r['nModified'], 0)

        # Delete
        self.assertEqual(db.pets.count(), 3)
        db.pets.insert(data1)
        self.assertEqual(db.pets.count(), 4)
        self.assertEqual(db.pets.remove({'name': 'Kitty', 'age': 0}), 1)
        self.assertEqual(db.pets.count(), 3)
        self.assertEqual(db.pets.findOneAndDelete({'name': 'Kitty'})['age'], 2)
        self.assertEqual(db.pets.count(), 2)

        # insertOne, insertMany
        db.pets.drop()
        oid = db.pets.insertOne(data1)
        self.assertEqual(
            db.pets.find({'_id': oid}).fetchone()['name'],
            data1['name']
        )
        oids = db.pets.insertMany([data2, data3])
        self.assertEqual(
            db.pets.find({'_id': oids[0]}).fetchone()['name'],
            data2['name']
        )

        db.close()

    def test_decimal(self):
        datum = [
            [100, (0, (1, 0, 0), 0), '100'],
            [-100, (1, (1, 0, 0), 0), '-100'],
            ['100', (0, (1, 0, 0), 0), '100'],
            ['-100', (1, (1, 0, 0), 0), '-100'],
            ['12.3456789', (0, (1, 2, 3, 4, 5, 6, 7, 8, 9), -7), '12.3456789'],
            ['123.456789', (0, (1, 2, 3, 4, 5, 6, 7, 8, 9), -6), '123.456789'],
            ['1234.56789', (0, (1, 2, 3, 4, 5, 6, 7, 8, 9), -5), '1234.56789'],
            ['-12.3456789', (1, (1, 2, 3, 4, 5, 6, 7, 8, 9), -7), '-12.3456789'],
            ['-123.456789', (1, (1, 2, 3, 4, 5, 6, 7, 8, 9), -6), '-123.456789'],
            ['-1234.56789', (1, (1, 2, 3, 4, 5, 6, 7, 8, 9), -5), '-1234.56789'],
            ['NaN', (0, (), 'n'), 'NaN'],
            ['-NaN', (1, (), 'n'), '-NaN'],
            ['sNaN', (0, (), 'N'), 'sNaN'],
            ['-sNaN', (1, (), 'N'), '-sNaN'],
            ['Infinity', (0, (0, ), 'F'), 'Infinity'],
            ['-Infinity', (1, (0, ), 'F'), '-Infinity'],
            ['Inf', (0, (0, ), 'F'), 'Infinity'],
            ['-Inf', (1, (0, ), 'F'), '-Infinity'],
        ]
        for data in datum:
            self.assertEqual(tuple(Decimal(data[0]).as_tuple()), data[1])
            self.assertEqual(str(Decimal(data[0])), data[2])

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import unittest

import openpr

class TestOpenpr(unittest.TestCase):
    def test_extract_service_and_module_gh_https(self):
        self.assertEqual(
            ('github.com', 'yoichi/openpr'),
            openpr.extract_service_and_module(
                'https://github.com/yoichi/openpr.git'))

    def test_extract_service_and_module_gh_ssh(self):
        self.assertEqual(
            ('github.com', 'yoichi/openpr'),
            openpr.extract_service_and_module(
                'git@github.com:yoichi/openpr.git'))

    def test_extract_service_and_module_bb_https(self):
        self.assertEqual(
            ('bitbucket.org', 'yoichi22/sandbox'),
            openpr.extract_service_and_module(
                'https://yoichi22@bitbucket.org/yoichi22/sandbox.git'))

    def test_extract_service_and_module_bb_ssh(self):
        self.assertEqual(
            ('bitbucket.org', 'yoichi22/sandbox'),
            openpr.extract_service_and_module(
                'git@bitbucket.org:yoichi22/sandbox.git'))

    def test_extract_pull_request_number_gh(self):
        self.assertEqual(
            '123',
            openpr.extract_pull_request_number(
                '0000000 Merge pull request #123 from foo/master\n'
                '1111111 Merge pull request #456 from bar/master'))

    def test_extract_pull_request_number_bb(self):
        self.assertEqual(
            '123',
            openpr.extract_pull_request_number(
                '0000000 Merged in foo/master (pull request #123)\n'
                '1111111 Merged in bar/master (pull request #456)'))

if __name__ == '__main__':
    unittest.main()
<commit_msg>add tests for get_default_tracking_branch<commit_after>#!/usr/bin/env python
import os
import shutil
import subprocess
import tempfile
import unittest

import openpr

class TempDir(object):
    def __enter__(self):
        self._directory = tempfile.mkdtemp()
        return self._directory

    def __exit__(self, *args):
        shutil.rmtree(self._directory)

class ChangeDir(object):
    def __init__(self, directory):
        self._new_cwd = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            directory)
        self._old_cwd = os.getcwd()

    def __enter__(self):
        os.chdir(self._new_cwd)

    def __exit__(self, *args):
        os.chdir(self._old_cwd)

class TestOpenpr(unittest.TestCase):
    def test_extract_service_and_module_gh_https(self):
        self.assertEqual(
            ('github.com', 'yoichi/openpr'),
            openpr.extract_service_and_module(
                'https://github.com/yoichi/openpr.git'))

    def test_extract_service_and_module_gh_ssh(self):
        self.assertEqual(
            ('github.com', 'yoichi/openpr'),
            openpr.extract_service_and_module(
                'git@github.com:yoichi/openpr.git'))

    def test_extract_service_and_module_bb_https(self):
        self.assertEqual(
            ('bitbucket.org', 'yoichi22/sandbox'),
            openpr.extract_service_and_module(
                'https://yoichi22@bitbucket.org/yoichi22/sandbox.git'))

    def test_extract_service_and_module_bb_ssh(self):
        self.assertEqual(
            ('bitbucket.org', 'yoichi22/sandbox'),
            openpr.extract_service_and_module(
                'git@bitbucket.org:yoichi22/sandbox.git'))

    def test_extract_pull_request_number_gh(self):
        self.assertEqual(
            '123',
            openpr.extract_pull_request_number(
                '0000000 Merge pull request #123 from foo/master\n'
                '1111111 Merge pull request #456 from bar/master'))

    def test_extract_pull_request_number_bb(self):
        self.assertEqual(
            '123',
            openpr.extract_pull_request_number(
                '0000000 Merged in foo/master (pull request #123)\n'
                '1111111 Merged in bar/master (pull request #456)'))

    def test_get_default_tracking_branch(self):
        """Detect default branch from remotes/origin/HEAD."""
        branch = 'trunk'
        call = subprocess.call
        with TempDir() as temp_dir:
            with ChangeDir(temp_dir):
                call(['git', 'init', '--bare', 'foo_origin'])
                call(['git', 'clone', 'foo_origin', 'foo_temp'])
            with ChangeDir(os.path.join(temp_dir, 'foo_temp')):
                call(['git', 'checkout', '-b', 'trunk'])
                call(['git', 'commit', '--allow-empty', '-m', 'message'])
                call(['git', 'push', 'origin', branch])
            with ChangeDir(os.path.join(temp_dir, 'foo_origin')):
                call(['git', 'symbolic-ref', 'HEAD',
                      'refs/heads/{}'.format(branch)])
            with ChangeDir(temp_dir):
                call(['git', 'clone', 'foo_origin', 'foo'])
            with ChangeDir(os.path.join(temp_dir, 'foo')):
                remote = 'origin'
                self.assertEqual(openpr.get_default_tracking_branch(remote),
                                 '{}/{}'.format(remote, branch))

    def test_get_default_tracking_branch_fallback(self):
        """Without remotes/origin/HEAD, fallback to origin/master."""
        call = subprocess.call
        with TempDir() as temp_dir:
            with ChangeDir(temp_dir):
                call(['git', 'init', '--bare', 'foo_origin'])
                call(['git', 'clone', 'foo_origin', 'foo'])
            with ChangeDir(os.path.join(temp_dir, 'foo')):
                call(['git', 'commit', '--allow-empty', '-m', 'message'])
                call(['git', 'push', '-u', 'origin', 'master'])
                remote = 'origin'
                self.assertEqual(openpr.get_default_tracking_branch(remote),
                                 '{}/master'.format(remote))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from autonomio.commands import data, train, test
import pandas as pd

# datasets
temp = data('tweet_sentiment')
temp = data('election_in_twitter')
temp = data('sites_category_and_vec')
temp = data('parties_and_employment')
temp = data('programmatic_ad_fraud')

# create dataset for rest of the tests
temp = data('random_tweets')
temp = temp.head(100)

# x variable input modes
tr = train(1,'neg',temp,dims=1)
tr = train([1,5],'neg',temp,dims=4)
tr = train(['reach_score','influence_score'],'neg',temp,dims=2)
tr = train([1,2,3,4,5],'neg',temp,dims=5)

# y variable flattening mode
tr = train(1,'quality_score',temp,dims=1,flatten='median')
tr = train(1,'quality_score',temp,dims=1,flatten=6)
tr = train(1,'quality_score',temp,dims=1,flatten=.5)
tr = train(1,'quality_score',temp,dims=1,flatten='mean')

# model saving and loading
tr = train('text','neg',temp,save_model='test_model')
te = test(temp,'test_model', labels='handle')

l = [	'funnel',
        'brick',
        'triangle',
        'rhombus',
        'long_funnel',
        'diamond',
        'hexagon',
        'stairs']

tr = train(1, 'neg', temp, dims=1, layers=1, validation=True)
tr = train(1, 'neg', temp, dims=1, validation=.6)

for i in l:

    if i in (l[0:3]): #funnel, brick, triangle
        tr = train(1, 'neg', temp, dims=1, shape=i, double_check=True)

    elif i in (l[3:4]): #only rhombus
        tr = train(1, 'neg', temp, dims=1, shape=i, neuron_max=1, layers=8)
        tr = train(1, 'neg', temp, dims=1, shape=i, neuron_max=9, layers=4)

	if i in (l[5:6]): #long_funnel, diamond
		tr = train(1, 'neg', temp, dims=1, shape=i, layers=6)
		tr = train(1, 'neg', temp, dims=1, shape=i, layers=5)

    elif i in (l[6:8]): #hexagon, stairs
    	tr = train(1, 'neg', temp, dims=1, shape=i, layers=4)
    	tr = train(1, 'neg', temp, dims=1, shape=i, layers=6)
    	tr = train(1, 'neg', temp, dims=1, shape=i, layers=7)
    	tr = train(1, 'neg', temp, dims=1, shape=i, layers=8) <commit_msg>small changes<commit_after>from autonomio.commands import data, train, test
import pandas as pd

# datasets
temp = data('tweet_sentiment')
temp = data('election_in_twitter')
temp = data('sites_category_and_vec')
temp = data('parties_and_employment')
temp = data('programmatic_ad_fraud')

# create dataset for rest of the tests
temp = data('random_tweets')
temp = temp.head(100)

# x variable input modes
tr = train(1,'neg',temp,dims=1)
tr = train([1,5],'neg',temp,dims=4)
tr = train(['reach_score','influence_score'],'neg',temp,dims=2)
tr = train([1,2,3,4,5],'neg',temp,dims=5)

# y variable flattening mode
tr = train(1,'quality_score',temp,dims=1,flatten='median')
tr = train(1,'quality_score',temp,dims=1,flatten=6)
tr = train(1,'quality_score',temp,dims=1,flatten=.5)
tr = train(1,'quality_score',temp,dims=1,flatten='mean')

# model saving and loading
tr = train('text','neg',temp,save_model='test_model')
te = test(temp,'test_model', labels='handle')

l = [	'funnel',
        'brick',
        'triangle',
        'rhombus',
        'long_funnel',
        'diamond',
        'hexagon',
        'stairs']

#for validation
tr = train(1, 'neg', temp, dims=1, validation=True)
tr = train(1, 'neg', temp, dims=1, validation=.6)

for i in l:

    if i in (l[0:3]): #funnel, brick, triangle
        tr = train(1, 'neg', temp, dims=1, double_check=True)

    elif i in (l[3:4]): #only rhombus
        tr = train(1, 'neg', temp, dims=1, shape=i, neuron_max=1, layers=8)
        tr = train(1, 'neg', temp, dims=1, shape=i, neuron_max=9, layers=4)

	if i in (l[5:6]): #long_funnel, diamond
		tr = train(1, 'neg', temp, dims=1, shape=i, layers=6)
		tr = train(1, 'neg', temp, dims=1, shape=i, layers=5)

    elif i in (l[6:8]): #hexagon, stairs
    	tr = train(1, 'neg', temp, dims=1, shape=i, layers=4)
    	tr = train(1, 'neg', temp, dims=1, shape=i, layers=6)
    	tr = train(1, 'neg', temp, dims=1, shape=i, layers=7)
    	tr = train(1, 'neg', temp, dims=1, shape=i, layers=8) <|endoftext|>