<commit_before># -*- coding: utf-8 -*-
'''
Tests for the state runner
'''

# Import Python Libs
from __future__ import absolute_import, print_function, unicode_literals
import errno
import os
import shutil
import signal
import tempfile
import textwrap
import threading
from salt.ext.six.moves import queue

# Import Salt Testing Libs
from tests.support.case import ShellCase
from tests.support.unit import skipIf
from tests.support.paths import TMP

# Import Salt Libs
import salt.utils.platform
import salt.utils.event
import salt.utils.files
import salt.utils.json
import salt.utils.stringutils
import salt.utils.yaml

# Import 3rd-party libs
from salt.ext import six

class StateRunnerTest(ShellCase):
    '''
    Test the state runner.
    '''
    def add_to_queue(self, q, cmd):
        '''
        helper method to add salt-run
        return data to a queue
        '''
        ret = self.run_run(cmd)
        q.put(ret)
        q.task_done()

    def test_orchestrate_output(self):
        '''
        Ensure the orchestrate runner outputs useful state data.

        In Issue #31330, the output only contains ['outputter:', '    highstate'],
        and not the full stateful return. This tests ensures we don't regress in that
        manner again.

        Also test against some sample "good" output that would be included in a correct
        orchestrate run.
        '''
        #ret_output = self.run_run_plus('state.orchestrate', 'orch.simple')['out']
        ret_output = self.run_run('state.orchestrate orch.simple')
        bad_out = ['outputter:', '    highstate']
        good_out = ['    Function: salt.state',
                    '      Result: True',
                    'Succeeded: 1 (changed=1)',
                    'Failed:    0',
                    'Total states run:     1']

        # First, check that we don't have the "bad" output that was displaying in
        # Issue #31330 where only the highstate outputter was listed
        self.assertIsNot(bad_out, ret_output)

        # Now test that some expected good sample output is present in the return.
        for item in good_out:
            self.assertIn(item, ret_output)

    def test_orchestrate_nested(self):
        '''
        test salt-run state.orchestrate and failhard with nested orchestration
        '''
        if os.path.exists('/tmp/ewu-2016-12-13'):
            os.remove('/tmp/ewu-2016-12-13')

        _, code = self.run_run(
                'state.orchestrate nested-orch.outer',
                with_retcode=True)

        self.assertFalse(os.path.exists('/tmp/ewu-2016-12-13'))
        self.assertNotEqual(code, 0)

    def test_orchestrate_state_and_function_failure(self):
        '''
        Ensure that returns from failed minions are in the changes dict where
        they belong, so they can be programatically analyzed.

        See https://github.com/saltstack/salt/issues/43204
        '''
        self.run_run('saltutil.sync_modules')
        ret = salt.utils.json.loads(
            '\n'.join(
                self.run_run('state.orchestrate orch.issue43204 --out=json')
            )
        )
        # Drill down to the changes dict
        state_ret = ret['data']['master']['salt_|-Step01_|-Step01_|-state']['changes']
        func_ret = ret['data']['master']['salt_|-Step02_|-runtests_helpers.nonzero_retcode_return_false_|-function']['changes']

        # Remove duration and start time from the results, since they would
        # vary with each run and that would make it impossible to test.
        for item in ('duration', 'start_time'):
            state_ret['ret']['minion']['test_|-test fail with changes_|-test fail with changes_|-fail_with_changes'].pop(item)

        self.assertEqual(
            state_ret,
            {
                'out': 'highstate',
                'ret': {
                    'minion': {
                        'test_|-test fail with changes_|-test fail with changes_|-fail_with_changes': {
                            '__id__': 'test fail with changes',
                            '__run_num__': 0,
                            '__sls__': 'orch.issue43204.fail_with_changes',
                            'changes': {
                                'testing': {
                                    'new': 'Something pretended to change',
                                    'old': 'Unchanged'
                                }
                            },
                            'comment': 'Failure!',
                            'name': 'test fail with changes',
                            'result': False,
                        }
                    }
                }
            }
        )

        self.assertEqual(
            func_ret,
            {'out': 'highstate', 'ret': {'minion': False}}
        )

    def test_orchestrate_target_exists(self):
        '''
        test orchestration when target exists
        while using multiple states
        '''
        ret = self.run_run('state.orchestrate orch.target-exists')

        first = ['          ID: core',
                 '    Function: salt.state',
                 '      Result: True']

        second = ['          ID: test-state',
                 '    Function: salt.state',
                 '      Result: True']

        third = ['          ID: cmd.run',
                 '    Function: salt.function',
                 '      Result: True']

        ret_out = [first, second, third]

        for out in ret_out:
            for item in out:
                self.assertIn(item, ret)

    def test_orchestrate_retcode(self):
        '''
        Test orchestration with nonzero retcode set in __context__
        '''
        self.run_run('saltutil.sync_runners')
        self.run_run('saltutil.sync_wheel')
        ret = '\n'.join(self.run_run('state.orchestrate orch.retcode'))

        for result in ('          ID: test_runner_success\n'
                       '    Function: salt.runner\n'
                       '        Name: runtests_helpers.success\n'
                       '      Result: True',

                       '          ID: test_runner_failure\n'
                       '    Function: salt.runner\n'
                       '        Name: runtests_helpers.failure\n'
                       '      Result: False',

                       '          ID: test_wheel_success\n'
                       '    Function: salt.wheel\n'
                       '        Name: runtests_helpers.success\n'
                       '      Result: True',

                       '          ID: test_wheel_failure\n'
                       '    Function: salt.wheel\n'
                       '        Name: runtests_helpers.failure\n'
                       '      Result: False'):
            self.assertIn(result, ret)

    def test_orchestrate_target_doesnt_exists(self):
        '''
        test orchestration when target doesnt exist
        while using multiple states
        '''
        ret = self.run_run('state.orchestrate orch.target-doesnt-exists')

        first = ['No minions matched the target. No command was sent, no jid was assigned.',
                 '          ID: core',
                 '    Function: salt.state',
                 '      Result: False']

        second = ['          ID: test-state',
                 '    Function: salt.state',
                 '      Result: True']

        third = ['          ID: cmd.run',
                 '    Function: salt.function',
                 '      Result: True']

        ret_out = [first, second, third]

        for out in ret_out:
            for item in out:
                self.assertIn(item, ret)

    def test_state_event(self):
        '''
        test to ensure state.event
        runner returns correct data
        '''
        q = queue.Queue(maxsize=0)

        cmd = 'state.event salt/job/*/new count=1'
        expect = '"minions": ["minion"]'
        server_thread = threading.Thread(target=self.add_to_queue, args=(q, cmd))
        server_thread.setDaemon(True)
        server_thread.start()

        while q.empty():
            self.run_salt('minion test.ping --static')
        out = q.get()
        self.assertIn(expect, six.text_type(out))

        server_thread.join()

@skipIf(salt.utils.platform.is_windows(), '*NIX-only test')
class OrchEventTest(ShellCase):
    '''
    Tests for orchestration events
    '''
    def setUp(self):
        self.timeout = 60
        self.master_d_dir = os.path.join(self.get_config_dir(), 'master.d')
        try:
            os.makedirs(self.master_d_dir)
        except OSError as exc:
            if exc.errno != errno.EEXIST:
                raise

        self.conf = tempfile.NamedTemporaryFile(
            mode='w',
            suffix='.conf',
            dir=self.master_d_dir,
            delete=True,
        )
        self.base_env = tempfile.mkdtemp(dir=TMP)
        self.addCleanup(shutil.rmtree, self.base_env)
        self.addCleanup(self.conf.close)
        for attr in ('timeout', 'master_d_dir', 'conf', 'base_env'):
            self.addCleanup(delattr, self, attr)
        # Force a reload of the configuration now that our temp config file has
        # been removed.
        self.addCleanup(self.run_run_plus, 'test.arg', __reload_config=True)

    def alarm_handler(self, signal, frame):
        raise Exception('Timeout of {0} seconds reached'.format(self.timeout))

    def write_conf(self, data):
        '''
        Dump the config dict to the conf file
        '''
        self.conf.write(salt.utils.yaml.safe_dump(data, default_flow_style=False))
        self.conf.flush()

    def test_jid_in_ret_event(self):
        '''
        Test to confirm that the ret event for the orchestration contains the
        jid for the jobs spawned.
        '''
        self.write_conf({
            'fileserver_backend': ['roots'],
            'file_roots': {
                'base': [self.base_env],
            },
        })

        state_sls = os.path.join(self.base_env, 'test_state.sls')
        with salt.utils.files.fopen(state_sls, 'w') as fp_:
            fp_.write(salt.utils.stringutils.to_str(textwrap.dedent('''
                date:
                  cmd.run
            ''')))

        orch_sls = os.path.join(self.base_env, 'test_orch.sls')
        with salt.utils.files.fopen(orch_sls, 'w') as fp_:
            fp_.write(salt.utils.stringutils.to_str(textwrap.dedent('''
                date_cmd:
                  salt.state:
                    - tgt: minion
                    - sls: test_state

                ping_minion:
                  salt.function:
                    - name: test.ping
                    - tgt: minion

                fileserver.file_list:
                  salt.runner

                config.values:
                  salt.wheel
            ''')))

        listener = salt.utils.event.get_event(
            'master',
            sock_dir=self.master_opts['sock_dir'],
            transport=self.master_opts['transport'],
            opts=self.master_opts)

        jid = self.run_run_plus(
            'state.orchestrate',
            'test_orch',
            __reload_config=True).get('jid')

        if jid is None:
            raise Exception('jid missing from run_run_plus output')

        signal.signal(signal.SIGALRM, self.alarm_handler)
        signal.alarm(self.timeout)
        try:
            while True:
                event = listener.get_event(full=True)
                if event is None:
                    continue

                if event['tag'] == 'salt/run/{0}/ret'.format(jid):
                    # Don't wrap this in a try/except. We want to know if the
                    # data structure is different from what we expect!
                    ret = event['data']['return']['data']['master']
                    for job in ret:
                        self.assertTrue('__jid__' in ret[job])
                    break
        finally:
            del listener
            signal.alarm(0)
<commit_msg>Increase timeout for orchestration test<commit_after># -*- coding: utf-8 -*-
'''
Tests for the state runner
'''

# Import Python Libs
from __future__ import absolute_import, print_function, unicode_literals
import errno
import os
import shutil
import signal
import tempfile
import textwrap
import threading
from salt.ext.six.moves import queue

# Import Salt Testing Libs
from tests.support.case import ShellCase
from tests.support.unit import skipIf
from tests.support.paths import TMP

# Import Salt Libs
import salt.utils.platform
import salt.utils.event
import salt.utils.files
import salt.utils.json
import salt.utils.stringutils
import salt.utils.yaml

# Import 3rd-party libs
from salt.ext import six

class StateRunnerTest(ShellCase):
    '''
    Test the state runner.
    '''
    def add_to_queue(self, q, cmd):
        '''
        helper method to add salt-run
        return data to a queue
        '''
        ret = self.run_run(cmd)
        q.put(ret)
        q.task_done()

    def test_orchestrate_output(self):
        '''
        Ensure the orchestrate runner outputs useful state data.

        In Issue #31330, the output only contains ['outputter:', '    highstate'],
        and not the full stateful return. This tests ensures we don't regress in that
        manner again.

        Also test against some sample "good" output that would be included in a correct
        orchestrate run.
        '''
        #ret_output = self.run_run_plus('state.orchestrate', 'orch.simple')['out']
        ret_output = self.run_run('state.orchestrate orch.simple')
        bad_out = ['outputter:', '    highstate']
        good_out = ['    Function: salt.state',
                    '      Result: True',
                    'Succeeded: 1 (changed=1)',
                    'Failed:    0',
                    'Total states run:     1']

        # First, check that we don't have the "bad" output that was displaying in
        # Issue #31330 where only the highstate outputter was listed
        self.assertIsNot(bad_out, ret_output)

        # Now test that some expected good sample output is present in the return.
        for item in good_out:
            self.assertIn(item, ret_output)

    def test_orchestrate_nested(self):
        '''
        test salt-run state.orchestrate and failhard with nested orchestration
        '''
        if os.path.exists('/tmp/ewu-2016-12-13'):
            os.remove('/tmp/ewu-2016-12-13')

        _, code = self.run_run(
                'state.orchestrate nested-orch.outer',
                with_retcode=True)

        self.assertFalse(os.path.exists('/tmp/ewu-2016-12-13'))
        self.assertNotEqual(code, 0)

    def test_orchestrate_state_and_function_failure(self):
        '''
        Ensure that returns from failed minions are in the changes dict where
        they belong, so they can be programatically analyzed.

        See https://github.com/saltstack/salt/issues/43204
        '''
        self.run_run('saltutil.sync_modules')
        ret = salt.utils.json.loads(
            '\n'.join(
                self.run_run('state.orchestrate orch.issue43204 --out=json')
            )
        )
        # Drill down to the changes dict
        state_ret = ret['data']['master']['salt_|-Step01_|-Step01_|-state']['changes']
        func_ret = ret['data']['master']['salt_|-Step02_|-runtests_helpers.nonzero_retcode_return_false_|-function']['changes']

        # Remove duration and start time from the results, since they would
        # vary with each run and that would make it impossible to test.
        for item in ('duration', 'start_time'):
            state_ret['ret']['minion']['test_|-test fail with changes_|-test fail with changes_|-fail_with_changes'].pop(item)

        self.assertEqual(
            state_ret,
            {
                'out': 'highstate',
                'ret': {
                    'minion': {
                        'test_|-test fail with changes_|-test fail with changes_|-fail_with_changes': {
                            '__id__': 'test fail with changes',
                            '__run_num__': 0,
                            '__sls__': 'orch.issue43204.fail_with_changes',
                            'changes': {
                                'testing': {
                                    'new': 'Something pretended to change',
                                    'old': 'Unchanged'
                                }
                            },
                            'comment': 'Failure!',
                            'name': 'test fail with changes',
                            'result': False,
                        }
                    }
                }
            }
        )

        self.assertEqual(
            func_ret,
            {'out': 'highstate', 'ret': {'minion': False}}
        )

    def test_orchestrate_target_exists(self):
        '''
        test orchestration when target exists
        while using multiple states
        '''
        ret = self.run_run('state.orchestrate orch.target-exists')

        first = ['          ID: core',
                 '    Function: salt.state',
                 '      Result: True']

        second = ['          ID: test-state',
                 '    Function: salt.state',
                 '      Result: True']

        third = ['          ID: cmd.run',
                 '    Function: salt.function',
                 '      Result: True']

        ret_out = [first, second, third]

        for out in ret_out:
            for item in out:
                self.assertIn(item, ret)

    def test_orchestrate_retcode(self):
        '''
        Test orchestration with nonzero retcode set in __context__
        '''
        self.run_run('saltutil.sync_runners')
        self.run_run('saltutil.sync_wheel')
        ret = '\n'.join(self.run_run('state.orchestrate orch.retcode', timeout=120))

        for result in ('          ID: test_runner_success\n'
                       '    Function: salt.runner\n'
                       '        Name: runtests_helpers.success\n'
                       '      Result: True',

                       '          ID: test_runner_failure\n'
                       '    Function: salt.runner\n'
                       '        Name: runtests_helpers.failure\n'
                       '      Result: False',

                       '          ID: test_wheel_success\n'
                       '    Function: salt.wheel\n'
                       '        Name: runtests_helpers.success\n'
                       '      Result: True',

                       '          ID: test_wheel_failure\n'
                       '    Function: salt.wheel\n'
                       '        Name: runtests_helpers.failure\n'
                       '      Result: False'):
            self.assertIn(result, ret)

    def test_orchestrate_target_doesnt_exists(self):
        '''
        test orchestration when target doesnt exist
        while using multiple states
        '''
        ret = self.run_run('state.orchestrate orch.target-doesnt-exists')

        first = ['No minions matched the target. No command was sent, no jid was assigned.',
                 '          ID: core',
                 '    Function: salt.state',
                 '      Result: False']

        second = ['          ID: test-state',
                 '    Function: salt.state',
                 '      Result: True']

        third = ['          ID: cmd.run',
                 '    Function: salt.function',
                 '      Result: True']

        ret_out = [first, second, third]

        for out in ret_out:
            for item in out:
                self.assertIn(item, ret)

    def test_state_event(self):
        '''
        test to ensure state.event
        runner returns correct data
        '''
        q = queue.Queue(maxsize=0)

        cmd = 'state.event salt/job/*/new count=1'
        expect = '"minions": ["minion"]'
        server_thread = threading.Thread(target=self.add_to_queue, args=(q, cmd))
        server_thread.setDaemon(True)
        server_thread.start()

        while q.empty():
            self.run_salt('minion test.ping --static')
        out = q.get()
        self.assertIn(expect, six.text_type(out))

        server_thread.join()

@skipIf(salt.utils.platform.is_windows(), '*NIX-only test')
class OrchEventTest(ShellCase):
    '''
    Tests for orchestration events
    '''
    def setUp(self):
        self.timeout = 60
        self.master_d_dir = os.path.join(self.get_config_dir(), 'master.d')
        try:
            os.makedirs(self.master_d_dir)
        except OSError as exc:
            if exc.errno != errno.EEXIST:
                raise

        self.conf = tempfile.NamedTemporaryFile(
            mode='w',
            suffix='.conf',
            dir=self.master_d_dir,
            delete=True,
        )
        self.base_env = tempfile.mkdtemp(dir=TMP)
        self.addCleanup(shutil.rmtree, self.base_env)
        self.addCleanup(self.conf.close)
        for attr in ('timeout', 'master_d_dir', 'conf', 'base_env'):
            self.addCleanup(delattr, self, attr)
        # Force a reload of the configuration now that our temp config file has
        # been removed.
        self.addCleanup(self.run_run_plus, 'test.arg', __reload_config=True)

    def alarm_handler(self, signal, frame):
        raise Exception('Timeout of {0} seconds reached'.format(self.timeout))

    def write_conf(self, data):
        '''
        Dump the config dict to the conf file
        '''
        self.conf.write(salt.utils.yaml.safe_dump(data, default_flow_style=False))
        self.conf.flush()

    def test_jid_in_ret_event(self):
        '''
        Test to confirm that the ret event for the orchestration contains the
        jid for the jobs spawned.
        '''
        self.write_conf({
            'fileserver_backend': ['roots'],
            'file_roots': {
                'base': [self.base_env],
            },
        })

        state_sls = os.path.join(self.base_env, 'test_state.sls')
        with salt.utils.files.fopen(state_sls, 'w') as fp_:
            fp_.write(salt.utils.stringutils.to_str(textwrap.dedent('''
                date:
                  cmd.run
            ''')))

        orch_sls = os.path.join(self.base_env, 'test_orch.sls')
        with salt.utils.files.fopen(orch_sls, 'w') as fp_:
            fp_.write(salt.utils.stringutils.to_str(textwrap.dedent('''
                date_cmd:
                  salt.state:
                    - tgt: minion
                    - sls: test_state

                ping_minion:
                  salt.function:
                    - name: test.ping
                    - tgt: minion

                fileserver.file_list:
                  salt.runner

                config.values:
                  salt.wheel
            ''')))

        listener = salt.utils.event.get_event(
            'master',
            sock_dir=self.master_opts['sock_dir'],
            transport=self.master_opts['transport'],
            opts=self.master_opts)

        jid = self.run_run_plus(
            'state.orchestrate',
            'test_orch',
            __reload_config=True).get('jid')

        if jid is None:
            raise Exception('jid missing from run_run_plus output')

        signal.signal(signal.SIGALRM, self.alarm_handler)
        signal.alarm(self.timeout)
        try:
            while True:
                event = listener.get_event(full=True)
                if event is None:
                    continue

                if event['tag'] == 'salt/run/{0}/ret'.format(jid):
                    # Don't wrap this in a try/except. We want to know if the
                    # data structure is different from what we expect!
                    ret = event['data']['return']['data']['master']
                    for job in ret:
                        self.assertTrue('__jid__' in ret[job])
                    break
        finally:
            del listener
            signal.alarm(0)
<|endoftext|>


<commit_before>"""
Read a blast file generated with -outfmt '6 std qlen slen staxids sscinames' and generate a list of taxonomies that match.

Our taxonomy has kingdom / phylum / genus / species
"""

import os
import sys
import argparse
import taxon
import re

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Read a blast file and create a tuple of [query / kingdom / phylum / genus / species]")
    parser.add_argument('-f', help='blast file(s). Note this must have taxids as column 14. You may specify more than one file', nargs='+')
    parser.add_argument('-t', help='taxonomy directory (default=/home2/db/taxonomy/current/)', default='/home2/db/taxonomy/current/')
    parser.add_argument('-v', help='verbose output', action="store_true")
    args = parser.parse_args()

    want = ['superkingdom', 'phylum', 'genus', 'species']

    if args.v:
        sys.stderr.write("Reading taxonomy\n")
    taxa=taxon.read_nodes(directory=args.t)
    names,blastname = taxon.read_names(directory=args.t)
    if args.v:
        sys.stderr.write("Read taxonomy\n")

    for blastf in args.f:
        if args.v:
            sys.stderr.write("Reading {}\n".format(blastf))
        with open(blastf, 'r') as fin:
            for l in fin:
                p=l.strip().split("\t")

                for tid in p[14].split(";"):
                    level = {}
                    results = [p[0], tid]

                    while tid != '0' and tid != '1' and tid in taxa and taxa[tid].parent != '1':
                        if taxa[tid].rank in want:
                            level[taxa[tid].rank] = names[tid].name
                        tid = taxa[tid].parent

                    # we are only going to print the entries that have all four levels
                    for w in want:
                        if w in level:
                            results.append(level[w])
                        else:
                            continue

                    print("\t".join(results))

<commit_msg>Choosing what to print<commit_after>"""
Read a blast file generated with -outfmt '6 std qlen slen staxids sscinames' and generate a list of taxonomies that match.

Our taxonomy has kingdom / phylum / genus / species
"""

import os
import sys
import argparse
import taxon
import re

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Read a blast file and create a tuple of [query / kingdom / phylum / genus / species]")
    parser.add_argument('-f', help='blast file(s). Note this must have taxids as column 14. You may specify more than one file', nargs='+')
    parser.add_argument('-t', help='taxonomy directory (default=/home2/db/taxonomy/current/)', default='/home2/db/taxonomy/current/')
    parser.add_argument('-v', help='verbose output', action="store_true")
    args = parser.parse_args()

    want = ['superkingdom', 'phylum', 'genus', 'species']

    if args.v:
        sys.stderr.write("Reading taxonomy\n")
    taxa=taxon.read_nodes(directory=args.t)
    names,blastname = taxon.read_names(directory=args.t)
    if args.v:
        sys.stderr.write("Read taxonomy\n")

    for blastf in args.f:
        if args.v:
            sys.stderr.write("Reading {}\n".format(blastf))
        with open(blastf, 'r') as fin:
            for l in fin:
                p=l.strip().split("\t")

                for tid in p[14].split(";"):
                    level = {}
                    results = [p[0], tid]

                    while tid != '0' and tid != '1' and tid in taxa and taxa[tid].parent != '1':
                        if taxa[tid].rank in want:
                            level[taxa[tid].rank] = names[tid].name
                        tid = taxa[tid].parent

                    # we are only going to print the entries that have all four levels
                    toprint = True
                    for w in want:
                        if w in level:
                            results.append(level[w])
                        else:
                            toprint = False

                    if toprint:
                        print("\t".join(results))

<|endoftext|>


<commit_before>import pytest

from . import *

def find_or_create_contact(client, email):
    contact = client.contact(email)

    if not contact:
        contact = client.contacts().add({
            'first_name':     'First',
            'last_name':      'Last',
            'address1':       '1 Main Street',
            'city':           'Anytown',
            'state_province': 'CO',
            'postal_code':    '80301',
            'country':        'US',
            'email_address':  email,
            'phone':          '+1 303 5551212'
        })

    return contact

@pytest.fixture
def contact(client):
    return find_or_create_contact(client, 'user@host.com')

class TestRegistrations:

    @pytest.mark.xfail(reason = "Registration currently failing in sandbox API", strict = True)
    def test_domain_registration_availability(self, client):
        status = client.find('google.com')
        assert not status.available
        assert not client.check('google.com')

        status = client.find(unregistered_domain_name)
        assert status.available
        assert client.check(unregistered_domain_name)

    @pytest.mark.xfail(reason = "Registration currently failing in sandbox API", strict = True)
    def test_registering_domain(self, client, contact):
        domain = client.register(unregistered_domain_name, contact)

        assert isinstance(domain, dnsimple.models.Domain)
        assert domain.name == unregistered_domain_name

        status = client.find(unregistered_domain_name)
        assert not status.available
        assert not client.check(unregistered_domain_name)

        domain = client.register('google.com', contact)
        assert domain is None

    def test_transferring_domain(self, client, contact):
        registered_domain_name = unregistered_domain_name # This was registered in previous test

        target_contact = find_or_create_contact(client, 'other@host.com')

        # Transferring within the account will fail, but this at least exercises the API code
        assert not client.transfer(registered_domain_name, target_contact)
<commit_msg>Revert "Skip failing registration integration tests"<commit_after>import pytest

from . import *

def find_or_create_contact(client, email):
    contact = client.contact(email)

    if not contact:
        contact = client.contacts().add({
            'first_name':     'First',
            'last_name':      'Last',
            'address1':       '1 Main Street',
            'city':           'Anytown',
            'state_province': 'CO',
            'postal_code':    '80301',
            'country':        'US',
            'email_address':  email,
            'phone':          '+1 303 5551212'
        })

    return contact

@pytest.fixture
def contact(client):
    return find_or_create_contact(client, 'user@host.com')

class TestRegistrations:

    def test_domain_registration_availability(self, client):
        status = client.find('google.com')
        assert not status.available
        assert not client.check('google.com')

        status = client.find(unregistered_domain_name)
        assert status.available
        assert client.check(unregistered_domain_name)

    def test_registering_domain(self, client, contact):
        domain = client.register(unregistered_domain_name, contact)

        assert isinstance(domain, dnsimple.models.Domain)
        assert domain.name == unregistered_domain_name

        status = client.find(unregistered_domain_name)
        assert not status.available
        assert not client.check(unregistered_domain_name)

        domain = client.register('google.com', contact)
        assert domain is None

    def test_transferring_domain(self, client, contact):
        registered_domain_name = unregistered_domain_name # This was registered in previous test

        target_contact = find_or_create_contact(client, 'other@host.com')

        # Transferring within the account will fail, but this at least exercises the API code
        assert not client.transfer(registered_domain_name, target_contact)
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
import peewee
import random
from discograph.library.sqlite.SqliteModel import SqliteModel

class SqliteEntity(SqliteModel):

    ### PEEWEE FIELDS ###

    name = peewee.TextField()
    entity_id = peewee.IntegerField()
    entity_type = peewee.IntegerField()

    ### PEEWEE META ###

    class Meta:
        db_table = 'entity'

    ### PRIVATE METHODS ###

    @classmethod
    def _load_from_mongo_class(cls, mongo_class):
        import discograph
        entity_type = 1
        if mongo_class == discograph.Label:
            entity_type = 2
        query = mongo_class.objects().no_cache().timeout(False)
        query = query.only('discogs_id', 'name')
        count = query.count()
        rows = []
        for i, mongo_document in enumerate(query, 1):
            if mongo_document.discogs_id and mongo_document.name:
                rows.append(dict(
                    entity_id=mongo_document.discogs_id,
                    entity_type=entity_type,
                    name=mongo_document.name,
                    random=random.random(),
                    ))
            if len(rows) == 100:
                cls.insert_many(rows).execute()
                rows = []
                print('[{}] Processing {}... {} of {} [{:.3f}%]'.format(
                    cls.__name__,
                    mongo_class.__name__,
                    i,
                    count,
                    (float(i) / count) * 100),
                    )
        if rows:
            cls.insert_many(rows).execute()
            print('[{}] Processing {}... {} of {} [{:.3f}%]'.format(
                cls.__name__,
                mongo_class.__name__,
                i,
                count,
                (float(i) / count) * 100),
                )

    ### PUBLIC METHODS ###

    @classmethod
    def bootstrap(cls):
        import discograph
        cls.drop_table(True)
        discograph.SqliteFTSEntity.drop_table(True)
        cls.create_table()
        discograph.SqliteFTSEntity.create_table(
            content=discograph.SqliteEntity,
            tokenize='unicode61',
            )
        cls._load_from_mongo_class(discograph.Artist)
        cls._load_from_mongo_class(discograph.Label)
        discograph.SqliteFTSEntity.rebuild()
        discograph.SqliteFTSEntity.optimize()

    @classmethod
    def from_artist_id(cls, artist_id):
        where_clause = cls.entity_id == id
        where_clause &= cls.entity_type == 1
        return cls.select().where(where_clause).get()

    @classmethod
    def from_label_id(cls, label_id):
        where_clause = cls.entity_id == id
        where_clause &= cls.entity_type == 2
        return cls.select().where(where_clause).get()

    ### PUBLIC PROPERTIES ###

    @property
    def discogs_id(self):
        return self.id<commit_msg>Defined index on SqliteEntity.<commit_after># -*- encoding: utf-8 -*-
import peewee
import random
from discograph.library.sqlite.SqliteModel import SqliteModel

class SqliteEntity(SqliteModel):

    ### PEEWEE FIELDS ###

    name = peewee.TextField()
    entity_id = peewee.IntegerField()
    entity_type = peewee.IntegerField()

    ### PEEWEE META ###

    class Meta:
        db_table = 'entity'
        indexes = (
            (('entity_id', 'entity_type'), True),
            )

    ### PRIVATE METHODS ###

    @classmethod
    def _load_from_mongo_class(cls, mongo_class):
        import discograph
        entity_type = 1
        if mongo_class == discograph.Label:
            entity_type = 2
        query = mongo_class.objects().no_cache().timeout(False)
        query = query.only('discogs_id', 'name')
        count = query.count()
        rows = []
        for i, mongo_document in enumerate(query, 1):
            if mongo_document.discogs_id and mongo_document.name:
                rows.append(dict(
                    entity_id=mongo_document.discogs_id,
                    entity_type=entity_type,
                    name=mongo_document.name,
                    random=random.random(),
                    ))
            if len(rows) == 100:
                cls.insert_many(rows).execute()
                rows = []
                print('[{}] Processing {}... {} of {} [{:.3f}%]'.format(
                    cls.__name__,
                    mongo_class.__name__,
                    i,
                    count,
                    (float(i) / count) * 100),
                    )
        if rows:
            cls.insert_many(rows).execute()
            print('[{}] Processing {}... {} of {} [{:.3f}%]'.format(
                cls.__name__,
                mongo_class.__name__,
                i,
                count,
                (float(i) / count) * 100),
                )

    ### PUBLIC METHODS ###

    @classmethod
    def bootstrap(cls):
        import discograph
        cls.drop_table(True)
        discograph.SqliteFTSEntity.drop_table(True)
        cls.create_table()
        discograph.SqliteFTSEntity.create_table(
            content=discograph.SqliteEntity,
            tokenize='unicode61',
            )
        cls._load_from_mongo_class(discograph.Artist)
        cls._load_from_mongo_class(discograph.Label)
        discograph.SqliteFTSEntity.rebuild()
        discograph.SqliteFTSEntity.optimize()

    @classmethod
    def from_artist_id(cls, artist_id):
        where_clause = cls.entity_id == id
        where_clause &= cls.entity_type == 1
        return cls.select().where(where_clause).get()

    @classmethod
    def from_label_id(cls, label_id):
        where_clause = cls.entity_id == id
        where_clause &= cls.entity_type == 2
        return cls.select().where(where_clause).get()

    ### PUBLIC PROPERTIES ###

    @property
    def discogs_id(self):
        return self.id<|endoftext|>


<commit_before>"""
Read a blast file generated with -outfmt '6 std qlen slen staxids sscinames' and generate a list of taxonomies that match.

Our taxonomy has kingdom / phylum / genus / species
"""

import os
import sys
import argparse
import taxon
import re

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Read a blast file and create a tuple of [query / kingdom / phylum / genus / species]")
    parser.add_argument('-f', help='blast file. Note this must have taxids as column 14', required=True)
    parser.add_argument('-t', help='taxonomy directory (default=/home2/db/taxonomy/current/)', default='/home2/db/taxonomy/current/')
    parser.add_argument('-v', help='verbose output', action="store_true")
    args = parser.parse_args()

    want = ['superkingdom', 'phylum', 'genus', 'species']

    sys.stderr.write("Reading taxonomy\n")
    taxa=taxon.read_nodes(directory=args.t)
    names,blastname = taxon.read_names(directory=args.t)
    sys.stderr.write("Read taxonomy\n")

    with open(args.f, 'r') as fin:
        for l in fin:
            p=l.strip().split("\t")

            for tid in p[14].split(";"):
                tid = p[14]
                level = {}
                while tid != '0' and tid != '1' and tid in taxa and taxa[tid].parent != '1':
                    if taxa[tid].rank in want:
                        level[taxa[tid].rank] = names[tid].name
                    tid = taxa[tid].parent

                results = [p[0], tid]
                for w in want:
                    if w in level:
                        results.append(level[w])
                    else:
                        results.append("")

                print("\t".join(results))

<commit_msg>Better parser<commit_after>"""
Read a blast file generated with -outfmt '6 std qlen slen staxids sscinames' and generate a list of taxonomies that match.

Our taxonomy has kingdom / phylum / genus / species
"""

import os
import sys
import argparse
import taxon
import re

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Read a blast file and create a tuple of [query / kingdom / phylum / genus / species]")
    parser.add_argument('-f', help='blast file. Note this must have taxids as column 14', required=True)
    parser.add_argument('-t', help='taxonomy directory (default=/home2/db/taxonomy/current/)', default='/home2/db/taxonomy/current/')
    parser.add_argument('-v', help='verbose output', action="store_true")
    args = parser.parse_args()

    want = ['superkingdom', 'phylum', 'genus', 'species']

    sys.stderr.write("Reading taxonomy\n")
    taxa=taxon.read_nodes(directory=args.t)
    names,blastname = taxon.read_names(directory=args.t)
    sys.stderr.write("Read taxonomy\n")

    with open(args.f, 'r') as fin:
        for l in fin:
            p=l.strip().split("\t")

            for tid in p[14].split(";"):
                level = {}
                while tid != '0' and tid != '1' and tid in taxa and taxa[tid].parent != '1':
                    if taxa[tid].rank in want:
                        level[taxa[tid].rank] = names[tid].name
                    tid = taxa[tid].parent

                results = [p[0], tid]
                for w in want:
                    if w in level:
                        results.append(level[w])
                    else:
                        results.append("")

                print("\t".join(results))

<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import functools
import os
import random
import subprocess
import sys
import time
from pathlib import Path

import app.cs_logger
from utils.common import BASE_DIR, LOGS_DIR

ENV = os.environ["CS_ENV"]
_log = app.cs_logger.get_logger(ENV, BASE_DIR)

RETRY = True
RETRY_MAX_TIMES = 5
RETRY_WAIT = 3
BACKOFF = False
BACKOFF_BASE = 5
BACKOFF_MAX_EXP = 4
BACKOFF_MAX_TIMES = 5

def retryable(fn=None, retry=RETRY, backoff=BACKOFF, retry_wait=RETRY_WAIT,
              retry_max_times=RETRY_MAX_TIMES, backoff_base=BACKOFF_BASE,
              backoff_max_exp=BACKOFF_MAX_EXP,
              backoff_max_times=BACKOFF_MAX_TIMES, _testing=False):
    if fn is None:
        return functools.partial(retryable)

    def _backoff_exps():
        valid_exp = lambda i: i if i < backoff_max_exp else backoff_max_exp
        yield from (valid_exp(i) for i in range(backoff_max_times))
    retryable._backoff_exps = _backoff_exps

    def _log_attempt(from_backoff, attempt_num, attempt_max, sleep_time):
        attempt_type = "Backoff" if from_backoff else "Retry"
        msg = "%s attempt %d of %d failed. Sleeping for %d seconds."
        _log.info(msg, attempt_type, attempt_num, attempt_max, sleep_time)

    def _wait():
        if not retry and not backoff:
            yield 0

        if retry:
            for i in range(retry_max_times):
                yield retry_wait
                _log_attempt(False, i+1, retry_max_times, retry_wait)
                time.sleep(retry_wait if not _testing else 0)

        if backoff:
            for i, exp in enumerate(_backoff_exps()):
                wait = backoff_base + random.randrange(0, backoff_base**exp)
                yield wait
                _log_attempt(True, i+1, backoff_max_times, wait)
                time.sleep(wait if not _testing else 0)

    @functools.wraps(fn)
    def _retry(*args, **kwargs):
        _retry._retryable_last_exception = None
        _retry._retryable_sleep_periods = []

        for next_sleep_period in _wait():
            _retry._retryable_sleep_periods.append(next_sleep_period)
            try:
                result = fn(*args, **kwargs)
                _retry.retryable_last_exception = None
                return result
            except Exception as e:
                _retry._retryable_last_exception = e
                _log.exception("Exception encountered in a retryable function")

        if _retry._retryable_last_exception:
            raise _retry._retryable_last_exception

    return _retry

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("path",
                        help="The path of the executable to retry on failure.")
    parser.add_argument("arguments",
                        nargs="*",
                        help="Arguments to pass to the executable.")
    parser.add_argument("--retry",
                        action="store_true",
                        default=RETRY,
                        help="Whether retry loops are enabled.")
    parser.add_argument("--retry-max-times",
                        type=int,
                        default=RETRY_MAX_TIMES,
                        help="The maximum number of retry attempts.")
    parser.add_argument("--retry-wait",
                        type=int,
                        default=RETRY_WAIT,
                        help="The wait period between retries in seconds.")
    parser.add_argument("--backoff",
                        action="store_true",
                        default=BACKOFF,
                        help="Whether exponential backoff is enabled.")
    parser.add_argument("--backoff-base",
                        type=int,
                        default=BACKOFF_BASE,
                        help=("The base for exponential backoff, i.e. the " +
                              "the minimum wait window size."))
    parser.add_argument("--backoff-max-exp",
                        type=int,
                        default=BACKOFF_MAX_EXP,
                        help=("The exponent for exponential backoff, i.e. " +
                              "the maximum wait window size."))
    parser.add_argument("--backoff-max-times",
                        type=int,
                        default=BACKOFF_MAX_TIMES,
                        help="The maximum number of backoff attempts.")

    args = parser.parse_args()
    return args

def run_subprocess(**kwargs):
    kwargs.setdefault("retry", RETRY)
    kwargs.setdefault("retry_max_times", RETRY_MAX_TIMES)
    kwargs.setdefault("retry_wait", RETRY_WAIT)
    kwargs.setdefault("backoff", BACKOFF)
    kwargs.setdefault("backoff_base", BACKOFF_BASE)
    kwargs.setdefault("backoff_max_exp", BACKOFF_MAX_EXP)
    kwargs.setdefault("backoff_max_times", BACKOFF_MAX_TIMES)

    path = kwargs.pop("path")
    arguments = kwargs.pop("arguments", [])

    @retryable(**kwargs)
    def _exec():
        statement = [args.path] + args.arguments
        completed = subprocess.run(statement)
        return completed.returncode

    returncode = _exec()
    sys.exit(returncode)

if __name__ == "__main__":
    try:
        args = parse_args()
        run_subprocess(**vars(args))
    except KeyboardInterrupt:
        pass

<commit_msg>Tweak retryable partial function and remove reliance on utils.common<commit_after>#!/usr/bin/env python3

import argparse
import functools
import os
import random
import subprocess
import sys
import time
from pathlib import Path

import app.cs_logger

ENV = os.environ["CS_ENV"]
BASE_DIR = str(Path(__file__).parents[1])
_log = app.cs_logger.get_logger(ENV, BASE_DIR)

RETRY = True
RETRY_MAX_TIMES = 5
RETRY_WAIT = 3
BACKOFF = False
BACKOFF_BASE = 5
BACKOFF_MAX_EXP = 4
BACKOFF_MAX_TIMES = 5

def retryable(fn=None, retry=RETRY, backoff=BACKOFF, retry_wait=RETRY_WAIT,
              retry_max_times=RETRY_MAX_TIMES, backoff_base=BACKOFF_BASE,
              backoff_max_exp=BACKOFF_MAX_EXP,
              backoff_max_times=BACKOFF_MAX_TIMES, _testing=False):
    if fn is None:
        return functools.partial(retryable, retry=retry, backoff=backoff,
                                 retry_wait=retry_wait,
                                 retry_max_times=retry_max_times,
                                 backoff_base=backoff_base,
                                 backoff_max_exp=backoff_max_exp,
                                 backoff_max_times=backoff_max_times)

    def _backoff_exps():
        valid_exp = lambda i: i if i < backoff_max_exp else backoff_max_exp
        yield from (valid_exp(i) for i in range(backoff_max_times))
    retryable._backoff_exps = _backoff_exps

    def _log_attempt(from_backoff, attempt_num, attempt_max, sleep_time):
        attempt_type = "Backoff" if from_backoff else "Retry"
        msg = "%s attempt %d of %d failed. Sleeping for %d seconds."
        _log.info(msg, attempt_type, attempt_num, attempt_max, sleep_time)

    def _wait():
        if not retry and not backoff:
            yield 0

        if retry:
            for i in range(retry_max_times):
                yield retry_wait
                _log_attempt(False, i+1, retry_max_times, retry_wait)
                time.sleep(retry_wait if not _testing else 0)

        if backoff:
            for i, exp in enumerate(_backoff_exps()):
                wait = backoff_base + random.randrange(0, backoff_base**exp)
                yield wait
                _log_attempt(True, i+1, backoff_max_times, wait)
                time.sleep(wait if not _testing else 0)

    @functools.wraps(fn)
    def _retry(*args, **kwargs):
        _retry._retryable_last_exception = None
        _retry._retryable_sleep_periods = []

        for next_sleep_period in _wait():
            _retry._retryable_sleep_periods.append(next_sleep_period)
            try:
                result = fn(*args, **kwargs)
                _retry.retryable_last_exception = None
                return result
            except Exception as e:
                _retry._retryable_last_exception = e
                _log.exception("Exception encountered in a retryable function")

        if _retry._retryable_last_exception:
            raise _retry._retryable_last_exception

    return _retry

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("path",
                        help="The path of the executable to retry on failure.")
    parser.add_argument("arguments",
                        nargs="*",
                        help="Arguments to pass to the executable.")
    parser.add_argument("--retry",
                        action="store_true",
                        default=RETRY,
                        help="Whether retry loops are enabled.")
    parser.add_argument("--retry-max-times",
                        type=int,
                        default=RETRY_MAX_TIMES,
                        help="The maximum number of retry attempts.")
    parser.add_argument("--retry-wait",
                        type=int,
                        default=RETRY_WAIT,
                        help="The wait period between retries in seconds.")
    parser.add_argument("--backoff",
                        action="store_true",
                        default=BACKOFF,
                        help="Whether exponential backoff is enabled.")
    parser.add_argument("--backoff-base",
                        type=int,
                        default=BACKOFF_BASE,
                        help=("The base for exponential backoff, i.e. the " +
                              "the minimum wait window size."))
    parser.add_argument("--backoff-max-exp",
                        type=int,
                        default=BACKOFF_MAX_EXP,
                        help=("The exponent for exponential backoff, i.e. " +
                              "the maximum wait window size."))
    parser.add_argument("--backoff-max-times",
                        type=int,
                        default=BACKOFF_MAX_TIMES,
                        help="The maximum number of backoff attempts.")

    args = parser.parse_args()
    return args

def run_subprocess(**kwargs):
    kwargs.setdefault("retry", RETRY)
    kwargs.setdefault("retry_max_times", RETRY_MAX_TIMES)
    kwargs.setdefault("retry_wait", RETRY_WAIT)
    kwargs.setdefault("backoff", BACKOFF)
    kwargs.setdefault("backoff_base", BACKOFF_BASE)
    kwargs.setdefault("backoff_max_exp", BACKOFF_MAX_EXP)
    kwargs.setdefault("backoff_max_times", BACKOFF_MAX_TIMES)

    path = kwargs.pop("path")
    arguments = kwargs.pop("arguments", [])

    @retryable(**kwargs)
    def _exec():
        statement = [args.path] + args.arguments
        completed = subprocess.run(statement)
        return completed.returncode

    returncode = _exec()
    sys.exit(returncode)

if __name__ == "__main__":
    try:
        args = parse_args()
        run_subprocess(**vars(args))
    except KeyboardInterrupt:
        pass

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab

"""
meza
~~~~

Provides methods for reading and processing data from tabular formatted files

Attributes:
    CURRENCIES [tuple(unicode)]: Currency symbols to remove from decimal
        strings.
    ENCODING (str): Default file encoding.
    DEFAULT_DATETIME (obj): Default datetime object
"""

from __future__ import (
    absolute_import, division, print_function, with_statement,
    unicode_literals)

import sys
import logging

from datetime import datetime as dt
from builtins import *

__version__ = '0.28.5'

__title__ = 'meza'
__package_name__ = 'meza'
__author__ = 'Reuben Cummings'
__description__ = 'A Python toolkit for processing tabular data'
__email__ = 'reubano@gmail.com'
__license__ = 'MIT'
__copyright__ = 'Copyright 2015 Reuben Cummings'

CURRENCIES = ('$', '£', '€')
ENCODING = 'utf-8'
DEFAULT_DATETIME = dt(9999, 12, 31, 0, 0, 0)

if sys.version_info.major >= 3:
    import csv
else:
    from . import unicsv as csv

logging.basicConfig()
csv = csv
<commit_msg>Bump to version 0.28.6<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab

"""
meza
~~~~

Provides methods for reading and processing data from tabular formatted files

Attributes:
    CURRENCIES [tuple(unicode)]: Currency symbols to remove from decimal
        strings.
    ENCODING (str): Default file encoding.
    DEFAULT_DATETIME (obj): Default datetime object
"""

from __future__ import (
    absolute_import, division, print_function, with_statement,
    unicode_literals)

import sys
import logging

from datetime import datetime as dt
from builtins import *

__version__ = '0.28.6'

__title__ = 'meza'
__package_name__ = 'meza'
__author__ = 'Reuben Cummings'
__description__ = 'A Python toolkit for processing tabular data'
__email__ = 'reubano@gmail.com'
__license__ = 'MIT'
__copyright__ = 'Copyright 2015 Reuben Cummings'

CURRENCIES = ('$', '£', '€')
ENCODING = 'utf-8'
DEFAULT_DATETIME = dt(9999, 12, 31, 0, 0, 0)

if sys.version_info.major >= 3:
    import csv
else:
    from . import unicsv as csv

logging.basicConfig()
csv = csv
<|endoftext|>


<commit_before>"""
Utility class for setting up an RNN.

Copyright (c) 2017 Frank Derry Wanye

Date: 16 July, 2017
"""

# Specify documentation format
__docformat__ = 'restructedtext en'

import argparse
import logging

def parse_arguments():
    """
    Parses the command line arguments and returns the namespace with those
    arguments.

    :type return: Namespace object
    :param return: The Namespace object containing the values of all supported
                   command-line arguments.
    """
    arg_parse = argparse.ArgumentParser()
    arg_parse.add_argument("-d", "--dir", default="./grurnn",
                           help="Directory for storing logs.")
    arg_parse.add_argument("-f", "--filename", default="grurnn.log",
                           help="Name of the log file to use.")
    arg_parse.add_argument("-e", "--epochs", default=10, type=int,
                           help="Number of epochs for which to train the RNN.")
    arg_parse.add_argument("-m", "--max", default=None, type=int,
                           help="The maximum number of examples to train on.")
    arg_parse.add_argument("-p", "--patience", default=100000, type=int,
                           help="Number of examples to train before evaluating"
                                + " loss.")
    arg_parse.add_argument("-t", "--test", action="store_true",
                           help="Treat run as test, do not save models")
    arg_parse.add_argument("-l", "--learn_rate", default=0.005, type=float,
                           help="The learning rate to be used in training.")
    arg_parse.add_argument("-o", "--model", default=None,
                           help="Previously trained model to load on init.")
    arg_parse.add_argument("-a", "--anneal", type=float, default=0.00001,
                           help="Sets the minimum possible learning rate.")
    arg_parse.add_argument("-s", "--dataset", default="./datasets/stories.pkl",
                           help="The path to the dataset to be used in "
                                " training.")
    arg_parse.add_argument("-r", "--truncate", type=int, default=100,
                           help="The backpropagate truncate value.")
    arg_parse.add_argument("-i", "--hidden_size", type=int, default=100,
                           help="The size of the hidden layers in the RNN.")
    arg_parse.add_argument("-b", "--embed_size", type=int, default=100,
                           help="The size of the embedding layer in the RNN.")
    return arg_parse.parse_args()
# End of parse_arguments()
<commit_msg>Broke up cmd-args into multiple methods<commit_after>"""
Utility class for setting up an RNN.

Copyright (c) 2017 Frank Derry Wanye

Date: 16 July, 2017
"""

# Specify documentation format
__docformat__ = 'restructedtext en'

import argparse
import logging

def parse_arguments():
    """
    Parses the command line arguments and returns the namespace with those
    arguments.

    :type return: Namespace object
    :param return: The Namespace object containing the values of all supported
                   command-line arguments.
    """
    arg_parse = argparse.ArgumentParser()
    __add_log_arguments__(arg_parse)
    __add_rnn_arguments__(arg_parse)
    __add_train_arguments__(arg_parse)
    return arg_parse.parse_args()
# End of parse_arguments()

def __add_log_arguments__(parser):
    """
    Adds arguments for setting up the logger to the given argument parser.
    Arguments added:
    --logger-name
    --log-filename
    --log-dir

    :type parser: argparse.ArgumentParser
    :param parser: The argument parser to which to add the logger arguments.
    """
    parser.add_argument("-ln", "--logger-name", default="TEST",
                        help="The name of the logger to be used. Defaults to 'TEST'")
    parser.add_argument("-lf", "--log-filename", default="terry.log",
                        help="The name of the file to which the logging will be done.")
    parser.add_argument("-ld", "--log-dir", default="./logging",
                        help="The path to the directory where the log file will be stored.")
# End of __add_log_arguments__()

def __add_rnn_arguments__(parser):
    """
    Adds arguments for setting up an RNN to the given argument parser.
    Arguments added:
    --dataset
    --hidden-size
    --embed-size
    --model

    :type parser: argparse.ArgumentParser
    :param parser: The argument parser to which to add the logger arguments.
    """
    parser.add_argument("-ds", "--dataset", default="./datasets/stories.pkl",
                        help="The path to the dataset to be used for training.")
    parser.add_argument("-hs", "--hidden_size", type=int, default=100,
                        help="The size of the hidden layers in the RNN.")
    parser.add_argument("-es", "--embed_size", type=int, default=100,
                        help="The size of the embedding layer in the RNN.")
    parser.add_argument("-m", "--model", default=None,
                        help="The previously trained model to load on init.")
# End of __add_rnn_arguments__()

def __add_train_arguments__(parser):
    """
    Adds arguments for training an RNN to the given argument parser.
    Arguments added:
    --epochs
    --max
    --patience
    --test
    --learning_rate
    --anneal
    --truncate

    :type parser: argparse.ArgumentParser
    :param parser: The argument parser to which to add the logger arguments.
    """
    parser.add_argument("-e", "--epochs", default=10, type=int,
                        help="The number of epochs for which to train the RNN.")
    parser.add_argument("-max", "--max", default=None, type=int,
                        help="The maximum number of examples to train on.")
    parser.add_argument("-p", "--patience", default=100000, type=int,
                        help="The number of examples to train before evaluating loss.")
    parser.add_argument("-t", "--test", action="store_true",
                        help="Treat run as test, do not save models")
    parser.add_argument("-l", "--learn_rate", default=0.005, type=float,
                        help="The learning rate to be used in training.")
    parser.add_argument("-a", "--anneal", type=float, default=0.00001,
                        help="The minimum possible learning rate.")
    parser.add_argument("-r", "--truncate", type=int, default=100,
                        help="The backpropagate truncate value.")
# End of __add_train_arguments__()

def setup_logger():
    """
    Sets up a logger
    """
    logger = logging.logger()
# End of setup_logger
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf.global_settings import PASSWORD_HASHERS as default_hashers
from django.contrib.auth.hashers import (is_password_usable, BasePasswordHasher,
    check_password, make_password, PBKDF2PasswordHasher, load_hashers,
    PBKDF2SHA1PasswordHasher, get_hasher, identify_hasher, UNUSABLE_PASSWORD)
from django.utils import unittest
from django.utils.unittest import skipUnless

try:
    import crypt
except ImportError:
    crypt = None

try:
    import bcrypt
except ImportError:
    bcrypt = None

class TestUtilsHashPass(unittest.TestCase):

    def setUp(self):
        load_hashers(password_hashers=default_hashers)

    def test_simple(self):
        encoded = make_password('lètmein')
        self.assertTrue(encoded.startswith('pbkdf2_sha256$'))
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))

    def test_pkbdf2(self):
        encoded = make_password('lètmein', 'seasalt', 'pbkdf2_sha256')
        self.assertEqual(encoded,
            'pbkdf2_sha256$10000$seasalt$CWWFdHOWwPnki7HvkcqN9iA2T3KLW1cf2uZ5kvArtVY=')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "pbkdf2_sha256")

    def test_sha1(self):
        encoded = make_password('lètmein', 'seasalt', 'sha1')
        self.assertEqual(encoded,
            'sha1$seasalt$cff36ea83f5706ce9aa7454e63e431fc726b2dc8')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "sha1")

    def test_md5(self):
        encoded = make_password('lètmein', 'seasalt', 'md5')
        self.assertEqual(encoded,
                         'md5$seasalt$3f86d0d3d465b7b458c231bf3555c0e3')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "md5")

    def test_unsalted_md5(self):
        encoded = make_password('lètmein', '', 'unsalted_md5')
        self.assertEqual(encoded, '88a434c88cca4e900f7874cd98123f43')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "unsalted_md5")
        # Alternate unsalted syntax
        alt_encoded = "md5$$%s" % encoded
        self.assertTrue(is_password_usable(alt_encoded))
        self.assertTrue(check_password('lètmein', alt_encoded))
        self.assertFalse(check_password('lètmeinz', alt_encoded))

    def test_unsalted_sha1(self):
        encoded = make_password('lètmein', '', 'unsalted_sha1')
        self.assertEqual(encoded, 'sha1$$6d138ca3ae545631b3abd71a4f076ce759c5700b')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "unsalted_sha1")
        # Raw SHA1 isn't acceptable
        alt_encoded = encoded[6:]
        self.assertFalse(check_password('lètmein', alt_encoded))

    @skipUnless(crypt, "no crypt module to generate password.")
    def test_crypt(self):
        encoded = make_password('lètmei', 'ab', 'crypt')
        self.assertEqual(encoded, 'crypt$$ab1Hv2Lg7ltQo')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmei', encoded))
        self.assertFalse(check_password('lètmeiz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "crypt")

    @skipUnless(bcrypt, "bcrypt not installed")
    def test_bcrypt_sha256(self):
        encoded = make_password('lètmein', hasher='bcrypt_sha256')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(encoded.startswith('bcrypt_sha256$'))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "bcrypt_sha256")

        # Verify that password truncation no longer works
        password = ('VSK0UYV6FFQVZ0KG88DYN9WADAADZO1CTSIVDJUNZSUML6IBX7LN7ZS3R5'
                    'JGB3RGZ7VI7G7DJQ9NI8BQFSRPTG6UWTTVESA5ZPUN')
        encoded = make_password(password, hasher='bcrypt_sha256')
        self.assertTrue(check_password(password, encoded))
        self.assertFalse(check_password(password[:72], encoded))

    @skipUnless(bcrypt, "bcrypt not installed")
    def test_bcrypt(self):
        encoded = make_password('lètmein', hasher='bcrypt')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(encoded.startswith('bcrypt$'))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "bcrypt")

    def test_unusable(self):
        encoded = make_password(None)
        self.assertFalse(is_password_usable(encoded))
        self.assertFalse(check_password(None, encoded))
        self.assertFalse(check_password(UNUSABLE_PASSWORD, encoded))
        self.assertFalse(check_password('', encoded))
        self.assertFalse(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertRaises(ValueError, identify_hasher, encoded)

    def test_bad_algorithm(self):
        with self.assertRaises(ValueError):
            make_password('lètmein', hasher='lolcat')
        self.assertRaises(ValueError, identify_hasher, "lolcat$salt$hash")

    def test_bad_encoded(self):
        self.assertFalse(is_password_usable('lètmein_badencoded'))
        self.assertFalse(is_password_usable(''))

    def test_low_level_pkbdf2(self):
        hasher = PBKDF2PasswordHasher()
        encoded = hasher.encode('lètmein', 'seasalt')
        self.assertEqual(encoded,
            'pbkdf2_sha256$10000$seasalt$CWWFdHOWwPnki7HvkcqN9iA2T3KLW1cf2uZ5kvArtVY=')
        self.assertTrue(hasher.verify('lètmein', encoded))

    def test_low_level_pbkdf2_sha1(self):
        hasher = PBKDF2SHA1PasswordHasher()
        encoded = hasher.encode('lètmein', 'seasalt')
        self.assertEqual(encoded,
            'pbkdf2_sha1$10000$seasalt$oAfF6vgs95ncksAhGXOWf4Okq7o=')
        self.assertTrue(hasher.verify('lètmein', encoded))

    def test_upgrade(self):
        self.assertEqual('pbkdf2_sha256', get_hasher('default').algorithm)
        for algo in ('sha1', 'md5'):
            encoded = make_password('lètmein', hasher=algo)
            state = {'upgraded': False}
            def setter(password):
                state['upgraded'] = True
            self.assertTrue(check_password('lètmein', encoded, setter))
            self.assertTrue(state['upgraded'])

    def test_no_upgrade(self):
        encoded = make_password('lètmein')
        state = {'upgraded': False}
        def setter():
            state['upgraded'] = True
        self.assertFalse(check_password('WRONG', encoded, setter))
        self.assertFalse(state['upgraded'])

    def test_no_upgrade_on_incorrect_pass(self):
        self.assertEqual('pbkdf2_sha256', get_hasher('default').algorithm)
        for algo in ('sha1', 'md5'):
            encoded = make_password('lètmein', hasher=algo)
            state = {'upgraded': False}
            def setter():
                state['upgraded'] = True
            self.assertFalse(check_password('WRONG', encoded, setter))
            self.assertFalse(state['upgraded'])

    def test_load_library_no_algorithm(self):
        with self.assertRaises(ValueError) as e:
            BasePasswordHasher()._load_library()
        self.assertEqual("Hasher 'BasePasswordHasher' doesn't specify a "
                         "library attribute", str(e.exception))

    def test_load_library_importerror(self):
        PlainHasher = type(str('PlainHasher'), (BasePasswordHasher,),
                           {'algorithm': 'plain', 'library': 'plain'})
        with self.assertRaises(ValueError) as e:
            PlainHasher()._load_library()
        self.assertEqual("Couldn't load 'PlainHasher' algorithm library: "
                         "No module named plain", str(e.exception))
<commit_msg>Worked around Python 3.3 modified exception repr<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf.global_settings import PASSWORD_HASHERS as default_hashers
from django.contrib.auth.hashers import (is_password_usable, BasePasswordHasher,
    check_password, make_password, PBKDF2PasswordHasher, load_hashers,
    PBKDF2SHA1PasswordHasher, get_hasher, identify_hasher, UNUSABLE_PASSWORD)
from django.utils import six
from django.utils import unittest
from django.utils.unittest import skipUnless

try:
    import crypt
except ImportError:
    crypt = None

try:
    import bcrypt
except ImportError:
    bcrypt = None

class TestUtilsHashPass(unittest.TestCase):

    def setUp(self):
        load_hashers(password_hashers=default_hashers)

    def test_simple(self):
        encoded = make_password('lètmein')
        self.assertTrue(encoded.startswith('pbkdf2_sha256$'))
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))

    def test_pkbdf2(self):
        encoded = make_password('lètmein', 'seasalt', 'pbkdf2_sha256')
        self.assertEqual(encoded,
            'pbkdf2_sha256$10000$seasalt$CWWFdHOWwPnki7HvkcqN9iA2T3KLW1cf2uZ5kvArtVY=')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "pbkdf2_sha256")

    def test_sha1(self):
        encoded = make_password('lètmein', 'seasalt', 'sha1')
        self.assertEqual(encoded,
            'sha1$seasalt$cff36ea83f5706ce9aa7454e63e431fc726b2dc8')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "sha1")

    def test_md5(self):
        encoded = make_password('lètmein', 'seasalt', 'md5')
        self.assertEqual(encoded,
                         'md5$seasalt$3f86d0d3d465b7b458c231bf3555c0e3')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "md5")

    def test_unsalted_md5(self):
        encoded = make_password('lètmein', '', 'unsalted_md5')
        self.assertEqual(encoded, '88a434c88cca4e900f7874cd98123f43')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "unsalted_md5")
        # Alternate unsalted syntax
        alt_encoded = "md5$$%s" % encoded
        self.assertTrue(is_password_usable(alt_encoded))
        self.assertTrue(check_password('lètmein', alt_encoded))
        self.assertFalse(check_password('lètmeinz', alt_encoded))

    def test_unsalted_sha1(self):
        encoded = make_password('lètmein', '', 'unsalted_sha1')
        self.assertEqual(encoded, 'sha1$$6d138ca3ae545631b3abd71a4f076ce759c5700b')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "unsalted_sha1")
        # Raw SHA1 isn't acceptable
        alt_encoded = encoded[6:]
        self.assertFalse(check_password('lètmein', alt_encoded))

    @skipUnless(crypt, "no crypt module to generate password.")
    def test_crypt(self):
        encoded = make_password('lètmei', 'ab', 'crypt')
        self.assertEqual(encoded, 'crypt$$ab1Hv2Lg7ltQo')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(check_password('lètmei', encoded))
        self.assertFalse(check_password('lètmeiz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "crypt")

    @skipUnless(bcrypt, "bcrypt not installed")
    def test_bcrypt_sha256(self):
        encoded = make_password('lètmein', hasher='bcrypt_sha256')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(encoded.startswith('bcrypt_sha256$'))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "bcrypt_sha256")

        # Verify that password truncation no longer works
        password = ('VSK0UYV6FFQVZ0KG88DYN9WADAADZO1CTSIVDJUNZSUML6IBX7LN7ZS3R5'
                    'JGB3RGZ7VI7G7DJQ9NI8BQFSRPTG6UWTTVESA5ZPUN')
        encoded = make_password(password, hasher='bcrypt_sha256')
        self.assertTrue(check_password(password, encoded))
        self.assertFalse(check_password(password[:72], encoded))

    @skipUnless(bcrypt, "bcrypt not installed")
    def test_bcrypt(self):
        encoded = make_password('lètmein', hasher='bcrypt')
        self.assertTrue(is_password_usable(encoded))
        self.assertTrue(encoded.startswith('bcrypt$'))
        self.assertTrue(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertEqual(identify_hasher(encoded).algorithm, "bcrypt")

    def test_unusable(self):
        encoded = make_password(None)
        self.assertFalse(is_password_usable(encoded))
        self.assertFalse(check_password(None, encoded))
        self.assertFalse(check_password(UNUSABLE_PASSWORD, encoded))
        self.assertFalse(check_password('', encoded))
        self.assertFalse(check_password('lètmein', encoded))
        self.assertFalse(check_password('lètmeinz', encoded))
        self.assertRaises(ValueError, identify_hasher, encoded)

    def test_bad_algorithm(self):
        with self.assertRaises(ValueError):
            make_password('lètmein', hasher='lolcat')
        self.assertRaises(ValueError, identify_hasher, "lolcat$salt$hash")

    def test_bad_encoded(self):
        self.assertFalse(is_password_usable('lètmein_badencoded'))
        self.assertFalse(is_password_usable(''))

    def test_low_level_pkbdf2(self):
        hasher = PBKDF2PasswordHasher()
        encoded = hasher.encode('lètmein', 'seasalt')
        self.assertEqual(encoded,
            'pbkdf2_sha256$10000$seasalt$CWWFdHOWwPnki7HvkcqN9iA2T3KLW1cf2uZ5kvArtVY=')
        self.assertTrue(hasher.verify('lètmein', encoded))

    def test_low_level_pbkdf2_sha1(self):
        hasher = PBKDF2SHA1PasswordHasher()
        encoded = hasher.encode('lètmein', 'seasalt')
        self.assertEqual(encoded,
            'pbkdf2_sha1$10000$seasalt$oAfF6vgs95ncksAhGXOWf4Okq7o=')
        self.assertTrue(hasher.verify('lètmein', encoded))

    def test_upgrade(self):
        self.assertEqual('pbkdf2_sha256', get_hasher('default').algorithm)
        for algo in ('sha1', 'md5'):
            encoded = make_password('lètmein', hasher=algo)
            state = {'upgraded': False}
            def setter(password):
                state['upgraded'] = True
            self.assertTrue(check_password('lètmein', encoded, setter))
            self.assertTrue(state['upgraded'])

    def test_no_upgrade(self):
        encoded = make_password('lètmein')
        state = {'upgraded': False}
        def setter():
            state['upgraded'] = True
        self.assertFalse(check_password('WRONG', encoded, setter))
        self.assertFalse(state['upgraded'])

    def test_no_upgrade_on_incorrect_pass(self):
        self.assertEqual('pbkdf2_sha256', get_hasher('default').algorithm)
        for algo in ('sha1', 'md5'):
            encoded = make_password('lètmein', hasher=algo)
            state = {'upgraded': False}
            def setter():
                state['upgraded'] = True
            self.assertFalse(check_password('WRONG', encoded, setter))
            self.assertFalse(state['upgraded'])

    def test_load_library_no_algorithm(self):
        with self.assertRaises(ValueError) as e:
            BasePasswordHasher()._load_library()
        self.assertEqual("Hasher 'BasePasswordHasher' doesn't specify a "
                         "library attribute", str(e.exception))

    def test_load_library_importerror(self):
        PlainHasher = type(str('PlainHasher'), (BasePasswordHasher,),
                           {'algorithm': 'plain', 'library': 'plain'})
        # Python 3.3 adds quotes around module name
        with six.assertRaisesRegex(self, ValueError,
                "Couldn't load 'PlainHasher' algorithm library: No module named '?plain'?"):
            PlainHasher()._load_library()
<|endoftext|>


<commit_before># Django settings for mics project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': '',                      # Or path to database file if using sqlite3.
        # The following settings are not used with sqlite3:
        'USER': '',
        'PASSWORD': '',
        'HOST': '',                      # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
        'PORT': '',                      # Set to empty string for default.
    }
}

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '6-bycz-+xpv@9+u8b^)#$-l&3cheum3i4cb_6$u6s%j6uu6s91'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'mics.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'mics.wsgi.application'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    'django_nose',
    'south',
    'survey',
    'lettuce.django',
    'mptt',
    'django_extensions',
    'rapidsms.contrib.locations',
    'django_mptt_admin',
    # Uncomment the next line to enable the admin:
    # 'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'

try:
    from localsettings import *
except ImportError:
    pass<commit_msg>fixing build<commit_after># Django settings for mics project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': '',                      # Or path to database file if using sqlite3.
        # The following settings are not used with sqlite3:
        'USER': '',
        'PASSWORD': '',
        'HOST': '',                      # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
        'PORT': '',                      # Set to empty string for default.
    }
}

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '6-bycz-+xpv@9+u8b^)#$-l&3cheum3i4cb_6$u6s%j6uu6s91'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'mics.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'mics.wsgi.application'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    'django_nose',
    'south',
    'survey',
    'lettuce.django',
    'mptt',
    'django_extensions',
    'rapidsms.contrib.locations',
    # Uncomment the next line to enable the admin:
    # 'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'

try:
    from localsettings import *
except ImportError:
    pass<|endoftext|>


<commit_before>import re

from bs4 import BeautifulSoup
from nltk.corpus import stopwords
import requests
from sqlalchemy.orm import sessionmaker

from models import create_db_tables, db_connect, WebsitesContent
from google import search

MAX_RESULTS_LIMIT = 50

def get_queries():
    """Returns a list of queries we want to index data.
    Note: for now it is a hardcoded list but it will be replaced with data fetched from Twitter.
    """
    return ['cars']

def index_query_data(query, db_session):
    """Fetch google pages for a query and index their content. """
    words = []
    for url in search(query, stop=MAX_RESULTS_LIMIT):
        try:
            words = extract_words_from_url(url)
            db_session.add(WebsitesContent(link=url, words=words))
            db_session.commit()
        except:
            print url

    return words

def extract_words_from_url(url):
    """Use BeautifulSoup to extract visible text from a web page. """
    def visible(element):
        """Method used to filter visible text elements. """
        if element.parent.name in ['style', 'script', '[document]', 'head', 'link']:
            return False
        elif re.match('<!--.*-->', element.encode('utf-8')):
            return False

        value = element.encode('utf-8')
        return not value.isspace()

    html = requests.get(url, verify=False).content
    soup = BeautifulSoup(html, 'html.parser')
    texts = soup.findAll(text=True)

    visible_texts = [text for text in texts if visible(text)]

    words = []
    stop_words = stopwords.words('english')
    for text in visible_texts:
        for word in re.findall(r"[\w']+", text):
            if not word.isdigit() and word.lower() not in stop_words:
                words.append(word)

    return words

def main():
    """Based on a list of queries:
     - get the first 50 results from google
     - scrap their text
     - store text in the db
     - get demographics for page
     - store demographics
    """
    engine = db_connect()
    create_db_tables(engine)
    Session = sessionmaker(bind=engine)
    session = Session()

    quereies = get_queries()

    for query in quereies:
        index_query_data(query, session)

    session.close()

if __name__ == '__main__':
    main()
<commit_msg>All words should be in lower case.<commit_after>import re

from bs4 import BeautifulSoup
from nltk.corpus import stopwords
import requests
from sqlalchemy.orm import sessionmaker

from models import create_db_tables, db_connect, WebsitesContent
from google import search

MAX_RESULTS_LIMIT = 50

def get_queries():
    """Returns a list of queries we want to index data.
    Note: for now it is a hardcoded list but it will be replaced with data fetched from Twitter.
    """
    return ['cars']

def index_query_data(query, db_session):
    """Fetch google pages for a query and index their content. """
    words = []
    for url in search(query, stop=MAX_RESULTS_LIMIT):
        try:
            words = extract_words_from_url(url)
            db_session.add(WebsitesContent(link=url, words=words))
            db_session.commit()
        except:
            print url

    return words

def extract_words_from_url(url):
    """Use BeautifulSoup to extract visible text from a web page. """
    def visible(element):
        """Method used to filter visible text elements. """
        if element.parent.name in ['style', 'script', '[document]', 'head', 'link']:
            return False
        elif re.match('<!--.*-->', element.encode('utf-8')):
            return False

        value = element.encode('utf-8')
        return not value.isspace()

    html = requests.get(url, verify=False).content
    soup = BeautifulSoup(html, 'html.parser')
    texts = soup.findAll(text=True)

    visible_texts = [text for text in texts if visible(text)]

    words = []
    stop_words = stopwords.words('english')
    for text in visible_texts:
        for word in re.findall(r"[\w']+", text):
            word = word.lower()
            if not word.isdigit() and word not in stop_words:
                words.append(word)

    return words

def main():
    """Based on a list of queries:
     - get the first 50 results from google
     - scrap their text
     - store text in the db
     - get demographics for page
     - store demographics
    """
    engine = db_connect()
    create_db_tables(engine)
    Session = sessionmaker(bind=engine)
    session = Session()

    quereies = get_queries()

    for query in quereies:
        index_query_data(query, session)

    session.close()

if __name__ == '__main__':
    main()
<|endoftext|>