<commit_before>import datetime

from django.conf import settings
from django.contrib import admin
from django.contrib.auth.models import User
from django.contrib.sessions.middleware import SessionMiddleware
from django.contrib.sites.models import Site
from django.core import mail
from django.core.exceptions import ImproperlyConfigured
from django.core.handlers.wsgi import WSGIRequest
from django.test import Client
from django.test import TestCase

from registration import forms
from registration import signals
from registration.admin import RegistrationAdmin
from registration.backends import get_backend
from registration.backends.default import DefaultBackend
from registration.backends.simple import SimpleBackend
from registration.models import RegistrationProfile

class _MockRequestClient(Client):
    """
    A ``django.test.Client`` subclass which can return mock
    ``HttpRequest`` objects.

    """
    def request(self, **request):
        """
        Rather than issuing a request and returning the response, this
        simply constructs an ``HttpRequest`` object and returns it.

        """
        environ = {
            'HTTP_COOKIE': self.cookies,
            'PATH_INFO': '/',
            'QUERY_STRING': '',
            'REMOTE_ADDR': '127.0.0.1',
            'REQUEST_METHOD': 'GET',
            'SCRIPT_NAME': '',
            'SERVER_NAME': 'testserver',
            'SERVER_PORT': '80',
            'SERVER_PROTOCOL': 'HTTP/1.1',
            'wsgi.version': (1,0),
            'wsgi.url_scheme': 'http',
            'wsgi.errors': self.errors,
            'wsgi.multiprocess':True,
            'wsgi.multithread': False,
            'wsgi.run_once': False,
            'wsgi.input': None,
            }
        environ.update(self.defaults)
        environ.update(request)
        request = WSGIRequest(environ)

        # We have to manually add a session since we'll be bypassing
        # the middleware chain.
        session_middleware = SessionMiddleware()
        session_middleware.process_request(request)
        return request

def _mock_request():
    """
    Construct and return a mock ``HttpRequest`` object; this is used
    in testing backend methods which expect an ``HttpRequest`` but
    which are not being called from views.

    """
    return _MockRequestClient().request()

class BackendRetrievalTests(TestCase):
    """
    Test that utilities for retrieving the active backend work
    properly.

    """
    def test_get_backend(self):
        """
        Verify that ``get_backend()`` returns the correct value when
        passed a valid backend.

        """
        self.failUnless(isinstance(get_backend('registration.backends.default.DefaultBackend'),
                                   DefaultBackend))

    def test_backend_error_invalid(self):
        """
        Test that a nonexistent/unimportable backend raises the
        correct exception.

        """
        self.assertRaises(ImproperlyConfigured, get_backend,
                          'registration.backends.doesnotexist.NonExistentBackend')

    def test_backend_attribute_error(self):
        """
        Test that a backend module which exists but does not have a
        class of the specified name raises the correct exception.

        """
        self.assertRaises(ImproperlyConfigured, get_backend,
                          'registration.backends.default.NonexistentBackend')

class DefaultRegistrationBackendTests(TestCase):
    """
    Test the default registration backend.

    Running these tests successfull will require two templates to be
    created for the sending of activation emails; details on these
    templates and their contexts may be found in the documentation for
    the default backend.

    """
    backend = DefaultBackend()

    def setUp(self):
        """
        Create an instance of the default backend for use in testing,
        and set ``ACCOUNT_ACTIVATION_DAYS`` if it's not set already.

        """
        self.old_activation = getattr(settings, 'ACCOUNT_ACTIVATION_DAYS', None)
        if self.old_activation is None:
            settings.ACCOUNT_ACTIVATION_DAYS = 7 # pragma: no cover

    def tearDown(self):
        """
        Yank out ``ACCOUNT_ACTIVATION_DAYS`` back out if it wasn't
        originally set.

        """
        if self.old_activation is None:
            settings.ACCOUNT_ACTIVATION_DAYS = self.old_activation # pragma: no cover

    def test_registration(self):
        """
        Test the registration process: registration creates a new
        inactive account and a new profile with activation key,
        populates the correct account data and sends an activation
        email.

        """
        new_user = self.backend.register(_mock_request(),
                                         username='bob',
                                         email='bob@example.com',
                                         password1='secret')

        # Details of the returned user must match what went in.
        self.assertEqual(new_user.username, 'bob')
        self.failUnless(new_user.check_password('secret'))
        self.assertEqual(new_user.email, 'bob@example.com')

        # New user must not be active.
        self.failIf(new_user.is_active)

        # A registration profile was created, and an activation email
        # was sent.
        self.assertEqual(RegistrationProfile.objects.count(), 1)
        self.assertEqual(len(mail.outbox), 1)

    def test_registration_no_sites(self):
        """
        Test that registration still functions properly when
        ``django.contrib.sites`` is not installed; the fallback will
        be a ``RequestSite`` instance.

        """
        Site._meta.installed = False
        new_user = self.backend.register(_mock_request(),
                                         username='bob',
                                         email='bob@example.com',
                                         password1='secret')

        self.assertEqual(new_user.username, 'bob')
        self.failUnless(new_user.check_password('secret'))
        self.assertEqual(new_user.email, 'bob@example.com')

        self.failIf(new_user.is_active)

        self.assertEqual(RegistrationProfile.objects.count(), 1)
        self.assertEqual(len(mail.outbox), 1)

        Site._meta.installed = True

    def test_valid_activation(self):
        """
        Test the activation process: activating within the permitted
        window sets the account's ``is_active`` field to ``True`` and
        resets the activation key.

        """
        valid_user = self.backend.register(_mock_request(),
                                           username='alice',
                                           email='alice@example.com',
                                           password1='swordfish')

        valid_profile = RegistrationProfile.objects.get(user=valid_user)
        activated = self.backend.activate(_mock_request(),
                                          valid_profile.activation_key)
        self.assertEqual(activated.username, valid_user.username)
        self.failUnless(activated.is_active)

        # Fetch the profile again to verify its activation key has
        # been reset.
        valid_profile = RegistrationProfile.objects.get(user=valid_user)
        self.assertEqual(valid_profile.activation_key,
                         RegistrationProfile.ACTIVATED)

    def test_invalid_activation(self):
        """
        Test the activation process: trying to activate outside the
        permitted window fails, and leaves the account inactive.

        """
        expired_user = self.backend.register(_mock_request(),
                                             username='bob',
                                             email='bob@example.com',
                                             password1='secret')

        expired_user.date_joined = expired_user.date_joined - datetime.timedelta(days=settings.ACCOUNT_ACTIVATION_DAYS)
        expired_user.save()
        expired_profile = RegistrationProfile.objects.get(user=expired_user)
        self.failIf(self.backend.activate(_mock_request(),
                                          expired_profile.activation_key))
        self.failUnless(expired_profile.activation_key_expired())

    def test_allow(self):
        """
        Test that the setting ``REGISTRATION_OPEN`` appropriately
        controls whether registration is permitted.

        """
        old_allowed = getattr(settings, 'REGISTRATION_OPEN', True)
        settings.REGISTRATION_OPEN = True
        self.failUnless(self.backend.registration_allowed(_mock_request()))

        settings.REGISTRATION_OPEN = False
        self.failIf(self.backend.registration_allowed(_mock_request()))
        settings.REGISTRATION_OPEN = old_allowed

    def test_form_class(self):
        """
        Test that the default form class returned is
        ``registration.forms.RegistrationForm``.

        """
        self.failUnless(self.backend.get_form_class(_mock_request()) is forms.RegistrationForm)

    def test_post_registration_redirect(self):
        """
        Test that the default post-registration redirect is the named
        pattern ``registration_complete``.

        """
        self.assertEqual(self.backend.post_registration_redirect(_mock_request(), User()),
                         ('registration_complete', (), {}))

    def test_registration_signal(self):
        """
        Test that registering a user sends the ``user_registered``
        signal.

        """
        def receiver(sender, **kwargs):
            self.failUnless('user' in kwargs)
            self.assertEqual(kwargs['user'].username, 'bob')
            self.failUnless('request' in kwargs)
            self.failUnless(isinstance(kwargs['request'], WSGIRequest))
            received_signals.append(kwargs.get('signal'))

        received_signals = []
        signals.user_registered.connect(receiver, sender=self.backend.__class__)

        self.backend.register(_mock_request(),
                              username='bob',
                              email='bob@example.com',
                              password1='secret')

        self.assertEqual(len(received_signals), 1)
        self.assertEqual(received_signals, [signals.user_registered])

    def test_activation_signal_success(self):
        """
        Test that successfully activating a user sends the
        ``user_activated`` signal.

        """
        def receiver(sender, **kwargs):
            self.failUnless('user' in kwargs)
            self.assertEqual(kwargs['user'].username, 'bob')
            self.failUnless('request' in kwargs)
            self.failUnless(isinstance(kwargs['request'], WSGIRequest))
            received_signals.append(kwargs.get('signal'))

        received_signals = []
        signals.user_activated.connect(receiver, sender=self.backend.__class__)

        new_user = self.backend.register(_mock_request(),
                                         username='bob',
                                         email='bob@example.com',
                                         password1='secret')
        profile = RegistrationProfile.objects.get(user=new_user)
        self.backend.activate(_mock_request(), profile.activation_key)

        self.assertEqual(len(received_signals), 1)
        self.assertEqual(received_signals, [signals.user_activated])

    def test_activation_signal_failure(self):
        """
        Test that an unsuccessful activation attempt does not send the
        ``user_activated`` signal.

        """
        receiver = lambda sender, **kwargs: received_signals.append(kwargs.get('signal'))

        received_signals = []
        signals.user_activated.connect(receiver, sender=self.backend.__class__)

        new_user = self.backend.register(_mock_request(),
                                         username='bob',
                                         email='bob@example.com',
                                         password1='secret')
        new_user.date_joined -= datetime.timedelta(days=settings.ACCOUNT_ACTIVATION_DAYS + 1)
        new_user.save()
        profile = RegistrationProfile.objects.get(user=new_user)
        self.backend.activate(_mock_request(), profile.activation_key)

        self.assertEqual(len(received_signals), 0)

    def test_email_send_action(self):
        """
        Test re-sending of activation emails via admin action.

        """
        admin_class = RegistrationAdmin(RegistrationProfile, admin.site)

        alice = self.backend.register(_mock_request(),
                                      username='alice',
                                      email='alice@example.com',
                                      password1='swordfish')

        admin_class.resend_activation_email(_mock_request(),
                                            RegistrationProfile.objects.all())
        self.assertEqual(len(mail.outbox), 2) # One on registering, one more on the resend.

        RegistrationProfile.objects.filter(user=alice).update(activation_key=RegistrationProfile.ACTIVATED)
        admin_class.resend_activation_email(_mock_request(),
                                            RegistrationProfile.objects.all())
        self.assertEqual(len(mail.outbox), 2) # No additional email because the account has activated.

    def test_email_send_action_no_sites(self):
        """
        Test re-sending of activation emails via admin action when
        ``django.contrib.sites`` is not installed; the fallback will
        be a ``RequestSite`` instance.

        """
        Site._meta.installed = False
        admin_class = RegistrationAdmin(RegistrationProfile, admin.site)

        alice = self.backend.register(_mock_request(),
                                      username='alice',
                                      email='alice@example.com',
                                      password1='swordfish')

        admin_class.resend_activation_email(_mock_request(),
                                            RegistrationProfile.objects.all())
        self.assertEqual(len(mail.outbox), 2) # One on registering, one more on the resend.

        RegistrationProfile.objects.filter(user=alice).update(activation_key=RegistrationProfile.ACTIVATED)
        admin_class.resend_activation_email(_mock_request(),
                                            RegistrationProfile.objects.all())
        self.assertEqual(len(mail.outbox), 2) # No additional email because the account has activated.
        Site._meta.installed = True

    def test_activation_action(self):
        """
        Test manual activation of users view admin action.

        """
        admin_class = RegistrationAdmin(RegistrationProfile, admin.site)

        alice = self.backend.register(_mock_request(),
                                      username='alice',
                                      email='alice@example.com',
                                      password1='swordfish')

        admin_class.activate_users(_mock_request(),
                                   RegistrationProfile.objects.all())
        self.failUnless(User.objects.get(username='alice').is_active)

class SimpleRegistrationBackendTests(TestCase):
    """
    Test the simple registration backend, which does signup and
    immediate activation.

    """
    backend = SimpleBackend()

    def test_registration(self):
        """
        Test the registration process: registration creates a new
        inactive account and a new profile with activation key,
        populates the correct account data and sends an activation
        email.

        """
        new_user = self.backend.register(_mock_request(),
                                         username='bob',
                                         email='bob@example.com',
                                         password1='secret')

        # Details of the returned user must match what went in.
        self.assertEqual(new_user.username, 'bob')
        self.failUnless(new_user.check_password('secret'))
        self.assertEqual(new_user.email, 'bob@example.com')

        # New user must not be active.
        self.failUnless(new_user.is_active)

    def test_allow(self):
        """
        Test that the setting ``REGISTRATION_OPEN`` appropriately
        controls whether registration is permitted.

        """
        old_allowed = getattr(settings, 'REGISTRATION_OPEN', True)
        settings.REGISTRATION_OPEN = True
        self.failUnless(self.backend.registration_allowed(_mock_request()))

        settings.REGISTRATION_OPEN = False
        self.failIf(self.backend.registration_allowed(_mock_request()))
        settings.REGISTRATION_OPEN = old_allowed

    def test_form_class(self):
        """
        Test that the default form class returned is
        ``registration.forms.RegistrationForm``.

        """
        self.failUnless(self.backend.get_form_class(_mock_request()) is forms.RegistrationForm)

    def test_post_registration_redirect(self):
        """
        Test that the default post-registration redirect is the public
        URL of the new user account.

        """
        new_user = self.backend.register(_mock_request(),
                                         username='bob',
                                         email='bob@example.com',
                                         password1='secret')

        self.assertEqual(self.backend.post_registration_redirect(_mock_request(), new_user),
                         (new_user.get_absolute_url(), (), {}))

    def test_registration_signal(self):
        """
        Test that registering a user sends the ``user_registered``
        signal.

        """
        def receiver(sender, **kwargs):
            self.failUnless('user' in kwargs)
            self.assertEqual(kwargs['user'].username, 'bob')
            self.failUnless('request' in kwargs)
            self.failUnless(isinstance(kwargs['request'], WSGIRequest))
            received_signals.append(kwargs.get('signal'))

        received_signals = []
        signals.user_registered.connect(receiver, sender=self.backend.__class__)

        self.backend.register(_mock_request(),
                              username='bob',
                              email='bob@example.com',
                              password1='secret')

        self.assertEqual(len(received_signals), 1)
        self.assertEqual(received_signals, [signals.user_registered])

    def test_activation(self):
        """
        Test that activating against this backend is an error.

        """
        self.assertRaises(NotImplementedError, self.backend.activate,
                          request=_mock_request())

    def test_post_activation_redirect(self):
        """
        Test that asking for a post-activation redirect from this
        backend is an error.

        """
        self.assertRaises(NotImplementedError, self.backend.post_activation_redirect,
                          request=_mock_request(), user=User())
<commit_msg>Typo in docstring.<commit_after>import datetime

from django.conf import settings
from django.contrib import admin
from django.contrib.auth.models import User
from django.contrib.sessions.middleware import SessionMiddleware
from django.contrib.sites.models import Site
from django.core import mail
from django.core.exceptions import ImproperlyConfigured
from django.core.handlers.wsgi import WSGIRequest
from django.test import Client
from django.test import TestCase

from registration import forms
from registration import signals
from registration.admin import RegistrationAdmin
from registration.backends import get_backend
from registration.backends.default import DefaultBackend
from registration.backends.simple import SimpleBackend
from registration.models import RegistrationProfile

class _MockRequestClient(Client):
    """
    A ``django.test.Client`` subclass which can return mock
    ``HttpRequest`` objects.

    """
    def request(self, **request):
        """
        Rather than issuing a request and returning the response, this
        simply constructs an ``HttpRequest`` object and returns it.

        """
        environ = {
            'HTTP_COOKIE': self.cookies,
            'PATH_INFO': '/',
            'QUERY_STRING': '',
            'REMOTE_ADDR': '127.0.0.1',
            'REQUEST_METHOD': 'GET',
            'SCRIPT_NAME': '',
            'SERVER_NAME': 'testserver',
            'SERVER_PORT': '80',
            'SERVER_PROTOCOL': 'HTTP/1.1',
            'wsgi.version': (1,0),
            'wsgi.url_scheme': 'http',
            'wsgi.errors': self.errors,
            'wsgi.multiprocess':True,
            'wsgi.multithread': False,
            'wsgi.run_once': False,
            'wsgi.input': None,
            }
        environ.update(self.defaults)
        environ.update(request)
        request = WSGIRequest(environ)

        # We have to manually add a session since we'll be bypassing
        # the middleware chain.
        session_middleware = SessionMiddleware()
        session_middleware.process_request(request)
        return request

def _mock_request():
    """
    Construct and return a mock ``HttpRequest`` object; this is used
    in testing backend methods which expect an ``HttpRequest`` but
    which are not being called from views.

    """
    return _MockRequestClient().request()

class BackendRetrievalTests(TestCase):
    """
    Test that utilities for retrieving the active backend work
    properly.

    """
    def test_get_backend(self):
        """
        Verify that ``get_backend()`` returns the correct value when
        passed a valid backend.

        """
        self.failUnless(isinstance(get_backend('registration.backends.default.DefaultBackend'),
                                   DefaultBackend))

    def test_backend_error_invalid(self):
        """
        Test that a nonexistent/unimportable backend raises the
        correct exception.

        """
        self.assertRaises(ImproperlyConfigured, get_backend,
                          'registration.backends.doesnotexist.NonExistentBackend')

    def test_backend_attribute_error(self):
        """
        Test that a backend module which exists but does not have a
        class of the specified name raises the correct exception.

        """
        self.assertRaises(ImproperlyConfigured, get_backend,
                          'registration.backends.default.NonexistentBackend')

class DefaultRegistrationBackendTests(TestCase):
    """
    Test the default registration backend.

    Running these tests successfully will require two templates to be
    created for the sending of activation emails; details on these
    templates and their contexts may be found in the documentation for
    the default backend.

    """
    backend = DefaultBackend()

    def setUp(self):
        """
        Create an instance of the default backend for use in testing,
        and set ``ACCOUNT_ACTIVATION_DAYS`` if it's not set already.

        """
        self.old_activation = getattr(settings, 'ACCOUNT_ACTIVATION_DAYS', None)
        if self.old_activation is None:
            settings.ACCOUNT_ACTIVATION_DAYS = 7 # pragma: no cover

    def tearDown(self):
        """
        Yank out ``ACCOUNT_ACTIVATION_DAYS`` back out if it wasn't
        originally set.

        """
        if self.old_activation is None:
            settings.ACCOUNT_ACTIVATION_DAYS = self.old_activation # pragma: no cover

    def test_registration(self):
        """
        Test the registration process: registration creates a new
        inactive account and a new profile with activation key,
        populates the correct account data and sends an activation
        email.

        """
        new_user = self.backend.register(_mock_request(),
                                         username='bob',
                                         email='bob@example.com',
                                         password1='secret')

        # Details of the returned user must match what went in.
        self.assertEqual(new_user.username, 'bob')
        self.failUnless(new_user.check_password('secret'))
        self.assertEqual(new_user.email, 'bob@example.com')

        # New user must not be active.
        self.failIf(new_user.is_active)

        # A registration profile was created, and an activation email
        # was sent.
        self.assertEqual(RegistrationProfile.objects.count(), 1)
        self.assertEqual(len(mail.outbox), 1)

    def test_registration_no_sites(self):
        """
        Test that registration still functions properly when
        ``django.contrib.sites`` is not installed; the fallback will
        be a ``RequestSite`` instance.

        """
        Site._meta.installed = False
        new_user = self.backend.register(_mock_request(),
                                         username='bob',
                                         email='bob@example.com',
                                         password1='secret')

        self.assertEqual(new_user.username, 'bob')
        self.failUnless(new_user.check_password('secret'))
        self.assertEqual(new_user.email, 'bob@example.com')

        self.failIf(new_user.is_active)

        self.assertEqual(RegistrationProfile.objects.count(), 1)
        self.assertEqual(len(mail.outbox), 1)

        Site._meta.installed = True

    def test_valid_activation(self):
        """
        Test the activation process: activating within the permitted
        window sets the account's ``is_active`` field to ``True`` and
        resets the activation key.

        """
        valid_user = self.backend.register(_mock_request(),
                                           username='alice',
                                           email='alice@example.com',
                                           password1='swordfish')

        valid_profile = RegistrationProfile.objects.get(user=valid_user)
        activated = self.backend.activate(_mock_request(),
                                          valid_profile.activation_key)
        self.assertEqual(activated.username, valid_user.username)
        self.failUnless(activated.is_active)

        # Fetch the profile again to verify its activation key has
        # been reset.
        valid_profile = RegistrationProfile.objects.get(user=valid_user)
        self.assertEqual(valid_profile.activation_key,
                         RegistrationProfile.ACTIVATED)

    def test_invalid_activation(self):
        """
        Test the activation process: trying to activate outside the
        permitted window fails, and leaves the account inactive.

        """
        expired_user = self.backend.register(_mock_request(),
                                             username='bob',
                                             email='bob@example.com',
                                             password1='secret')

        expired_user.date_joined = expired_user.date_joined - datetime.timedelta(days=settings.ACCOUNT_ACTIVATION_DAYS)
        expired_user.save()
        expired_profile = RegistrationProfile.objects.get(user=expired_user)
        self.failIf(self.backend.activate(_mock_request(),
                                          expired_profile.activation_key))
        self.failUnless(expired_profile.activation_key_expired())

    def test_allow(self):
        """
        Test that the setting ``REGISTRATION_OPEN`` appropriately
        controls whether registration is permitted.

        """
        old_allowed = getattr(settings, 'REGISTRATION_OPEN', True)
        settings.REGISTRATION_OPEN = True
        self.failUnless(self.backend.registration_allowed(_mock_request()))

        settings.REGISTRATION_OPEN = False
        self.failIf(self.backend.registration_allowed(_mock_request()))
        settings.REGISTRATION_OPEN = old_allowed

    def test_form_class(self):
        """
        Test that the default form class returned is
        ``registration.forms.RegistrationForm``.

        """
        self.failUnless(self.backend.get_form_class(_mock_request()) is forms.RegistrationForm)

    def test_post_registration_redirect(self):
        """
        Test that the default post-registration redirect is the named
        pattern ``registration_complete``.

        """
        self.assertEqual(self.backend.post_registration_redirect(_mock_request(), User()),
                         ('registration_complete', (), {}))

    def test_registration_signal(self):
        """
        Test that registering a user sends the ``user_registered``
        signal.

        """
        def receiver(sender, **kwargs):
            self.failUnless('user' in kwargs)
            self.assertEqual(kwargs['user'].username, 'bob')
            self.failUnless('request' in kwargs)
            self.failUnless(isinstance(kwargs['request'], WSGIRequest))
            received_signals.append(kwargs.get('signal'))

        received_signals = []
        signals.user_registered.connect(receiver, sender=self.backend.__class__)

        self.backend.register(_mock_request(),
                              username='bob',
                              email='bob@example.com',
                              password1='secret')

        self.assertEqual(len(received_signals), 1)
        self.assertEqual(received_signals, [signals.user_registered])

    def test_activation_signal_success(self):
        """
        Test that successfully activating a user sends the
        ``user_activated`` signal.

        """
        def receiver(sender, **kwargs):
            self.failUnless('user' in kwargs)
            self.assertEqual(kwargs['user'].username, 'bob')
            self.failUnless('request' in kwargs)
            self.failUnless(isinstance(kwargs['request'], WSGIRequest))
            received_signals.append(kwargs.get('signal'))

        received_signals = []
        signals.user_activated.connect(receiver, sender=self.backend.__class__)

        new_user = self.backend.register(_mock_request(),
                                         username='bob',
                                         email='bob@example.com',
                                         password1='secret')
        profile = RegistrationProfile.objects.get(user=new_user)
        self.backend.activate(_mock_request(), profile.activation_key)

        self.assertEqual(len(received_signals), 1)
        self.assertEqual(received_signals, [signals.user_activated])

    def test_activation_signal_failure(self):
        """
        Test that an unsuccessful activation attempt does not send the
        ``user_activated`` signal.

        """
        receiver = lambda sender, **kwargs: received_signals.append(kwargs.get('signal'))

        received_signals = []
        signals.user_activated.connect(receiver, sender=self.backend.__class__)

        new_user = self.backend.register(_mock_request(),
                                         username='bob',
                                         email='bob@example.com',
                                         password1='secret')
        new_user.date_joined -= datetime.timedelta(days=settings.ACCOUNT_ACTIVATION_DAYS + 1)
        new_user.save()
        profile = RegistrationProfile.objects.get(user=new_user)
        self.backend.activate(_mock_request(), profile.activation_key)

        self.assertEqual(len(received_signals), 0)

    def test_email_send_action(self):
        """
        Test re-sending of activation emails via admin action.

        """
        admin_class = RegistrationAdmin(RegistrationProfile, admin.site)

        alice = self.backend.register(_mock_request(),
                                      username='alice',
                                      email='alice@example.com',
                                      password1='swordfish')

        admin_class.resend_activation_email(_mock_request(),
                                            RegistrationProfile.objects.all())
        self.assertEqual(len(mail.outbox), 2) # One on registering, one more on the resend.

        RegistrationProfile.objects.filter(user=alice).update(activation_key=RegistrationProfile.ACTIVATED)
        admin_class.resend_activation_email(_mock_request(),
                                            RegistrationProfile.objects.all())
        self.assertEqual(len(mail.outbox), 2) # No additional email because the account has activated.

    def test_email_send_action_no_sites(self):
        """
        Test re-sending of activation emails via admin action when
        ``django.contrib.sites`` is not installed; the fallback will
        be a ``RequestSite`` instance.

        """
        Site._meta.installed = False
        admin_class = RegistrationAdmin(RegistrationProfile, admin.site)

        alice = self.backend.register(_mock_request(),
                                      username='alice',
                                      email='alice@example.com',
                                      password1='swordfish')

        admin_class.resend_activation_email(_mock_request(),
                                            RegistrationProfile.objects.all())
        self.assertEqual(len(mail.outbox), 2) # One on registering, one more on the resend.

        RegistrationProfile.objects.filter(user=alice).update(activation_key=RegistrationProfile.ACTIVATED)
        admin_class.resend_activation_email(_mock_request(),
                                            RegistrationProfile.objects.all())
        self.assertEqual(len(mail.outbox), 2) # No additional email because the account has activated.
        Site._meta.installed = True

    def test_activation_action(self):
        """
        Test manual activation of users view admin action.

        """
        admin_class = RegistrationAdmin(RegistrationProfile, admin.site)

        alice = self.backend.register(_mock_request(),
                                      username='alice',
                                      email='alice@example.com',
                                      password1='swordfish')

        admin_class.activate_users(_mock_request(),
                                   RegistrationProfile.objects.all())
        self.failUnless(User.objects.get(username='alice').is_active)

class SimpleRegistrationBackendTests(TestCase):
    """
    Test the simple registration backend, which does signup and
    immediate activation.

    """
    backend = SimpleBackend()

    def test_registration(self):
        """
        Test the registration process: registration creates a new
        inactive account and a new profile with activation key,
        populates the correct account data and sends an activation
        email.

        """
        new_user = self.backend.register(_mock_request(),
                                         username='bob',
                                         email='bob@example.com',
                                         password1='secret')

        # Details of the returned user must match what went in.
        self.assertEqual(new_user.username, 'bob')
        self.failUnless(new_user.check_password('secret'))
        self.assertEqual(new_user.email, 'bob@example.com')

        # New user must not be active.
        self.failUnless(new_user.is_active)

    def test_allow(self):
        """
        Test that the setting ``REGISTRATION_OPEN`` appropriately
        controls whether registration is permitted.

        """
        old_allowed = getattr(settings, 'REGISTRATION_OPEN', True)
        settings.REGISTRATION_OPEN = True
        self.failUnless(self.backend.registration_allowed(_mock_request()))

        settings.REGISTRATION_OPEN = False
        self.failIf(self.backend.registration_allowed(_mock_request()))
        settings.REGISTRATION_OPEN = old_allowed

    def test_form_class(self):
        """
        Test that the default form class returned is
        ``registration.forms.RegistrationForm``.

        """
        self.failUnless(self.backend.get_form_class(_mock_request()) is forms.RegistrationForm)

    def test_post_registration_redirect(self):
        """
        Test that the default post-registration redirect is the public
        URL of the new user account.

        """
        new_user = self.backend.register(_mock_request(),
                                         username='bob',
                                         email='bob@example.com',
                                         password1='secret')

        self.assertEqual(self.backend.post_registration_redirect(_mock_request(), new_user),
                         (new_user.get_absolute_url(), (), {}))

    def test_registration_signal(self):
        """
        Test that registering a user sends the ``user_registered``
        signal.

        """
        def receiver(sender, **kwargs):
            self.failUnless('user' in kwargs)
            self.assertEqual(kwargs['user'].username, 'bob')
            self.failUnless('request' in kwargs)
            self.failUnless(isinstance(kwargs['request'], WSGIRequest))
            received_signals.append(kwargs.get('signal'))

        received_signals = []
        signals.user_registered.connect(receiver, sender=self.backend.__class__)

        self.backend.register(_mock_request(),
                              username='bob',
                              email='bob@example.com',
                              password1='secret')

        self.assertEqual(len(received_signals), 1)
        self.assertEqual(received_signals, [signals.user_registered])

    def test_activation(self):
        """
        Test that activating against this backend is an error.

        """
        self.assertRaises(NotImplementedError, self.backend.activate,
                          request=_mock_request())

    def test_post_activation_redirect(self):
        """
        Test that asking for a post-activation redirect from this
        backend is an error.

        """
        self.assertRaises(NotImplementedError, self.backend.post_activation_redirect,
                          request=_mock_request(), user=User())
<|endoftext|>


<commit_before>"""Load the trained q table and make actions based on that.
"""

import time
import env
import rl

rl.load_q()
env.make("pygame")

while True:
    env.reset()
    for _ in range(15):
        if env.done:
            break
        action = rl.choose_action(rl.table[env.object[0]])
        env.action(action)
        time.sleep(0.03)
        env.render()
<commit_msg>Update to newest version of rl.py.<commit_after>"""Load the trained q table and make actions based on that.
"""

import time
import env
import rl

rl.load_q()
env.make("pygame")

while True:
    env.reset()
    for _ in range(15):
        if env.done:
            break
        action = rl.choose_action(env.player, "test")
        env.action(action)
        time.sleep(0.03)
        env.render()
<|endoftext|>


<commit_before>

__author__ = 'scribblemaniac'

from renderchan.module import RenderChanModule
import subprocess
import gzip
import os
import re

class RenderChanInkscapeModule(RenderChanModule):
    def __init__(self):
        RenderChanModule.__init__(self)
        if os.name == 'nt':
            self.conf['binary']=os.path.join(os.path.dirname(__file__),"..\\..\\..\\inkscape\\inkscape.exe")
        else:
            self.conf['binary']="inkscape"
        self.conf["packetSize"]=0

    def getInputFormats(self):
        return ["svg", "svgz", "ai", "cdr", "vsd"]

    def getOutputFormats(self):
        return ["png", "ps", "eps", "pdf", "emf", "wmf"]

    def analyze(self, filename):
        info={"dependencies":[]}

        if filename.endswith(".svgz"):
            f=gzip.open(filename)
        else:
            f=open(filename)

        linkPattern = re.compile(".*sodipodi:absref=\"(.*?)\".*")
        for line in f:
            pat=linkPattern.search(line)
            if pat:
                info["dependencies"].append(pat.group(1).strip())
        f.close

        return info

    def render(self, filename, outputPath, startFrame, endFrame, format, updateCompletion, extraParams={}):

        comp = 0.0
        updateCompletion(comp)

        commandline=[self.conf['binary'], "--file=" + filename, "--without-gui", "--export-width=" + extraParams["width"], "--export-height=" + extraParams["height"], "--export-%s=%s" % (format, outputPath)]
        subprocess.check_call(commandline)

        updateCompletion(1.0)
<commit_msg>Inkscape: Fix reading files (Python3)<commit_after>

__author__ = 'scribblemaniac'

from renderchan.module import RenderChanModule
import subprocess
import gzip
import os
import re

class RenderChanInkscapeModule(RenderChanModule):
    def __init__(self):
        RenderChanModule.__init__(self)
        if os.name == 'nt':
            self.conf['binary']=os.path.join(os.path.dirname(__file__),"..\\..\\..\\inkscape\\inkscape.exe")
        else:
            self.conf['binary']="inkscape"
        self.conf["packetSize"]=0

    def getInputFormats(self):
        return ["svg", "svgz", "ai", "cdr", "vsd"]

    def getOutputFormats(self):
        return ["png", "ps", "eps", "pdf", "emf", "wmf"]

    def analyze(self, filename):
        info={"dependencies":[]}

        if filename.endswith(".svgz"):
            f=gzip.open(filename, 'r')
        else:
            f=open(filename, 'r')

        linkPattern = re.compile(".*sodipodi:absref=\"(.*?)\".*")
        for line in f:
            pat=linkPattern.search(line)
            if pat:
                info["dependencies"].append(pat.group(1).strip())
        f.close

        return info

    def render(self, filename, outputPath, startFrame, endFrame, format, updateCompletion, extraParams={}):

        comp = 0.0
        updateCompletion(comp)

        commandline=[self.conf['binary'], "--file=" + filename, "--without-gui", "--export-width=" + extraParams["width"], "--export-height=" + extraParams["height"], "--export-%s=%s" % (format, outputPath)]
        subprocess.check_call(commandline)

        updateCompletion(1.0)
<|endoftext|>


<commit_before>"""
Chapter 10: Binary Trees
"""

class TNode(object):

    def __init__(self, val, node_id=None):
        self.val = val
        self.left = None
        self.right = None
        self.parent = None
        self.node_id = node_id

def check_equal(root_one, root_two):
    """
    Check that two binary trees are equal
    """
    if root_one is None and root_two is None:
        return True
    elif (root_one is None and root_two is not None) \
            or (root_one is not None and root_two is None):
        return False
    elif root_one.val != root_two.val:
        return False
    else:
        return check_equal(root_one.left, root_two.left) and \
            check_equal(root_one.right, root_two.right)

def is_balanced(root):
    """
    Problem 10.1: determine if binary tree is balanced
    """
    return balanced_helper(root)[0]

def balanced_helper(root):
    """
    Helper function to return tuple for balancing
    """
    if not root:
        return (True, 0)
    left = balanced_helper(root.left)
    right = balanced_helper(root.right)

    return (
        left[0] and right[0] and abs(left[1] - right[1]) <= 1,
        1 + max(left[1], right[1]),
    )

def is_symmetric(root):
    """
    Question 10.2: check if binary tree is symmetric
    """
    return root is None or check_symmetric(root.left, root.right)

def check_symmetric(left, right):
    """
    Helper function to recursively check symmetry of subtrees
    """
    if left is None and right is None:
        return True
    elif (left is None and right is not None) or \
            (left is not None and right is None):
        return False
    elif left.val != right.val:
        return False
    else:
        return check_symmetric(left.left, right.right) and \
            check_symmetric(left.right, right.left)

def compute_lca(root, first, second):
    """
    Question 10.3: Compute the lowest
    common ancestor in a binary tree
    """
    _, lca = compute_lca_helper(root, first, second)
    return lca

def compute_lca_helper(root, first, second):
    """
    Helper that returns
    """
    if root is None:
        return 0, None

    # get left subtree
    left_nodes, left_lca = compute_lca_helper(
        root.left,
        first,
        second
    )
    if left_nodes == 2:
        return left_nodes, left_lca

    right_nodes, right_lca = compute_lca_helper(
        root.right,
        first,
        second
    )

    if right_nodes == 2:
        return right_nodes, right_lca

    num_nodes = left_nodes + right_nodes + \
        (root == first) + (root == second)
    lca = root if num_nodes == 2 else None

    return num_nodes, lca

def compute_parent_lca(first, second):
    """
    Problem 10.4: Get least common ancestor if nodes
    have parent pointer
    """
    first_depth = get_depth(first)
    second_depth = get_depth(second)

    # swap if second deeper than first
    if second_depth > first_depth:
        tmp = first
        first = second
        second = tmp

    depth_diff = abs(first_depth - second_depth)
    while depth_diff:
        first = first.parent
        depth_diff -= 1

    while first != second:
        first = first.parent
        second = second.parent

    return first

def get_depth(node):
    """
    Helper function to get depth of node
    from root
    """
    depth = 0
    while node.parent:
        node = node.parent
        depth += 1
    return depth

def sum_root_to_leaf(root):
    """
    Question 10.5: Sum all root to
    leaf paths, given that all paths
    represent binary numbers
    """
    return sum_root_helper(root, 0)

def sum_root_helper(root, partial_sum):
    """
    Helper to keep track of partial sums as
    getting remainder of root to leaf sums
    """
    if root is None:
        return 0

    partial_sum = partial_sum * 2 + root.val
    if not root.right and not root.left:
        return partial_sum

    return sum_root_helper(root.right, partial_sum) + \
        sum_root_helper(root.left, partial_sum)

def has_leaf_sum(root, target):
    """
    Question 10.6: Check that there exists a
    root-to-leaf path that sums to the target
    """
    return has_leaf_sum_helper(root, target, 0)

def has_leaf_sum_helper(root, target, partial_sum):
    """
    Helper function to keep track of partial
    sum while trying to hit target
    """
    if root is None:
        return False

    partial_sum += root.val
    if partial_sum == target:
        return True

    return has_leaf_sum_helper(root.left, target, partial_sum) or \
        has_leaf_sum_helper(root.right, target, partial_sum)

def get_kth_inorder_record(root, k):
    """
    Question 10.7
    """
    current_num = 0
    cur = root
    prev = None

    while cur:
        next_node = None
        if cur.parent == prev:
            if cur.left:
                next_node = cur.left
            else:
                current_num += 1
                if current_num == k:
                    return cur.val
                next_node = cur.right if cur.right else cur.parent
        elif cur.left == prev:
            current_num += 1
            if current_num == k:
                return cur.val
            next_node = cur.right if cur.right else cur.parent
        else:
            next_node = cur.parent
        prev = cur
        cur = next_node
    return None

def find_successor(node):
    """
    Question 10.8: Compute the successor
    of the given node in the inorder traversal
    of the tree
    """
    if node.right is not None:
        cur = node.right

        # recurse down to left-most node
        # in right subtree
        while cur.left:
            cur = cur.left
        return cur

    while node.parent and node.parent.right == node:
        node = node.parent

    return node.parent

def inorder_traversal(root):
    """
    Question 10.9: Implement an
    inorder traversal in O(1) space
    """
    elts = []
    cur = root
    prev = None

    while cur:
        next_node = None
        if cur.parent == prev:
            # came down to cur from prev
            if cur.left:
                next_node = cur.left
            else:
                elts.append(cur.val)
                # done with left, so go up if
                # right is not empty
                # otherwise go up
                next_node = cur.right if cur.right else cur.parent
        elif cur.left == prev:
            # came up to cur from left child
            elts.append(cur.val)
            next_node = cur.right if cur.right else cur.parent
        else:
            # done with both children so move up
            next_node = cur.parent
        prev = cur
        cur = next_node
    return elts

def reconstruct_tree(preorder, inorder):
    """
    Question 10.10
    """
    node_to_inorder_idx = {}
    for idx, elt in enumerate(inorder):
        node_to_inorder_idx[elt] = idx

    return reconstruct_helper(
        preorder, 0, len(preorder),
        0, len(inorder), node_to_inorder_idx,
    )

def reconstruct_helper(
    preorder, preorder_start, preorder_end,
    inorder_start, inorder_end, node_to_inorder_idx,
):
    if preorder_end <= preorder_start or inorder_end <= inorder_start:
        return None

    # first element in preorder is root
    root_inorder_idx = node_to_inorder_idx[preorder[preorder_start]]
    left_subtree_size = root_inorder_idx - inorder_start

    root = TNode(preorder[preorder_start])
    root.left = reconstruct_helper(
        preorder, preorder_start + 1, preorder_start + 1 + left_subtree_size,
        inorder_start, root_inorder_idx, node_to_inorder_idx,
    )
    root.right = reconstruct_helper(
        preorder, preorder_start + 1 + left_subtree_size, preorder_end,
        root_inorder_idx + 1, inorder_end, node_to_inorder_idx,
    )
    return root

def reconstruct_preorder(preorder):
    """
    Question 10.11: Reconstruct binary tree from
    a preorder representation with markers
    """

    subtree_idx = 0
    root, _ = reconstruct_preorder_helper(preorder, subtree_idx)
    return root

def reconstruct_preorder_helper(preorder, subtree_idx):
    """
    Recursive helper to construct subtree
    """
    subtree_key = TNode(preorder[subtree_idx]) if preorder[
        subtree_idx] else None
    subtree_idx += 1
    if subtree_key is None:
        return None, subtree_idx

    left_subtree, left_idx = reconstruct_preorder_helper(preorder, subtree_idx)
    right_subtree, right_idx = reconstruct_preorder_helper(preorder, left_idx)
    subtree_key.left = left_subtree
    subtree_key.right = right_subtree

    return subtree_key, right_idx

def create_leaf_list(root):
    """
    Question 10.12: Create list from leaves
    of binary tree
    """
    if root.left is None and root.right is None:
        return [root.val]

    ls = []
    if root.left is not None:
        ls.extend(create_leaf_list(root.left))
    if root.right is not None:
        ls.extend(create_leaf_list(root.right))
    return ls

def calculate_tree_exterior(root):
    """
    Question 10.13: Calculate exterior of binary
    tree
    """
    if root is None:
        return []
    return [root.val] + calculate_left_exterior(root.left, True) \
        + calculate_right_exterior(root.right, True)

def calculate_left_exterior(root, is_exterior):
    """
    Build left exterior
    """
    if is_exterior:
        ls = [root.val]
        if root.left is not None:
            ls.extend(calculate_left_exterior(root.left, True))
            ls.extend(calculate_left_exterior(root.right, False))
        elif root.right is not None:
            ls.extend(calculate_left_exterior(root.right, True))
        return ls
    elif root.left is None and root.right is None:
        return [root.val]
    return calculate_left_exterior(root.left, False) \
        + calculate_left_exterior(root.right, False)

def calculate_right_exterior(root, is_exterior):
    """
    Build right exterior
    """
    if root is None:
        return []

    if is_exterior:
        ls = [root.val]
        if root.right is not None:
            ls.extend(calculate_right_exterior(root.left, False))
            ls.extend(calculate_right_exterior(root.right, True))
        elif root.left is not None:
            ls.extend(calculate_right_exterior(root.left, True))
        return ls
    elif root.right is None and root.left is None:
        return [root.val]
    return calculate_right_exterior(root.right, False) \
        + calculate_right_exterior(root.left, False)
<commit_msg>finished, uuuuuggghhhh<commit_after>"""
Chapter 10: Binary Trees
"""

class TNode(object):

    def __init__(self, val, node_id=None):
        self.val = val
        self.left = None
        self.right = None
        self.parent = None
        self.node_id = node_id

def check_equal(root_one, root_two):
    """
    Check that two binary trees are equal
    """
    if root_one is None and root_two is None:
        return True
    elif (root_one is None and root_two is not None) \
            or (root_one is not None and root_two is None):
        return False
    elif root_one.val != root_two.val:
        return False
    else:
        return check_equal(root_one.left, root_two.left) and \
            check_equal(root_one.right, root_two.right)

def is_balanced(root):
    """
    Problem 10.1: determine if binary tree is balanced
    """
    return balanced_helper(root)[0]

def balanced_helper(root):
    """
    Helper function to return tuple for balancing
    """
    if not root:
        return (True, 0)
    left = balanced_helper(root.left)
    right = balanced_helper(root.right)

    return (
        left[0] and right[0] and abs(left[1] - right[1]) <= 1,
        1 + max(left[1], right[1]),
    )

def is_symmetric(root):
    """
    Question 10.2: check if binary tree is symmetric
    """
    return root is None or check_symmetric(root.left, root.right)

def check_symmetric(left, right):
    """
    Helper function to recursively check symmetry of subtrees
    """
    if left is None and right is None:
        return True
    elif (left is None and right is not None) or \
            (left is not None and right is None):
        return False
    elif left.val != right.val:
        return False
    else:
        return check_symmetric(left.left, right.right) and \
            check_symmetric(left.right, right.left)

def compute_lca(root, first, second):
    """
    Question 10.3: Compute the lowest
    common ancestor in a binary tree
    """
    _, lca = compute_lca_helper(root, first, second)
    return lca

def compute_lca_helper(root, first, second):
    """
    Helper that returns
    """
    if root is None:
        return 0, None

    # get left subtree
    left_nodes, left_lca = compute_lca_helper(
        root.left,
        first,
        second
    )
    if left_nodes == 2:
        return left_nodes, left_lca

    right_nodes, right_lca = compute_lca_helper(
        root.right,
        first,
        second
    )

    if right_nodes == 2:
        return right_nodes, right_lca

    num_nodes = left_nodes + right_nodes + \
        (root == first) + (root == second)
    lca = root if num_nodes == 2 else None

    return num_nodes, lca

def compute_parent_lca(first, second):
    """
    Problem 10.4: Get least common ancestor if nodes
    have parent pointer
    """
    first_depth = get_depth(first)
    second_depth = get_depth(second)

    # swap if second deeper than first
    if second_depth > first_depth:
        tmp = first
        first = second
        second = tmp

    depth_diff = abs(first_depth - second_depth)
    while depth_diff:
        first = first.parent
        depth_diff -= 1

    while first != second:
        first = first.parent
        second = second.parent

    return first

def get_depth(node):
    """
    Helper function to get depth of node
    from root
    """
    depth = 0
    while node.parent:
        node = node.parent
        depth += 1
    return depth

def sum_root_to_leaf(root):
    """
    Question 10.5: Sum all root to
    leaf paths, given that all paths
    represent binary numbers
    """
    return sum_root_helper(root, 0)

def sum_root_helper(root, partial_sum):
    """
    Helper to keep track of partial sums as
    getting remainder of root to leaf sums
    """
    if root is None:
        return 0

    partial_sum = partial_sum * 2 + root.val
    if not root.right and not root.left:
        return partial_sum

    return sum_root_helper(root.right, partial_sum) + \
        sum_root_helper(root.left, partial_sum)

def has_leaf_sum(root, target):
    """
    Question 10.6: Check that there exists a
    root-to-leaf path that sums to the target
    """
    return has_leaf_sum_helper(root, target, 0)

def has_leaf_sum_helper(root, target, partial_sum):
    """
    Helper function to keep track of partial
    sum while trying to hit target
    """
    if root is None:
        return False

    partial_sum += root.val
    if partial_sum == target:
        return True

    return has_leaf_sum_helper(root.left, target, partial_sum) or \
        has_leaf_sum_helper(root.right, target, partial_sum)

def get_kth_inorder_record(root, k):
    """
    Question 10.7
    """
    current_num = 0
    cur = root
    prev = None

    while cur:
        next_node = None
        if cur.parent == prev:
            if cur.left:
                next_node = cur.left
            else:
                current_num += 1
                if current_num == k:
                    return cur.val
                next_node = cur.right if cur.right else cur.parent
        elif cur.left == prev:
            current_num += 1
            if current_num == k:
                return cur.val
            next_node = cur.right if cur.right else cur.parent
        else:
            next_node = cur.parent
        prev = cur
        cur = next_node
    return None

def find_successor(node):
    """
    Question 10.8: Compute the successor
    of the given node in the inorder traversal
    of the tree
    """
    if node.right is not None:
        cur = node.right

        # recurse down to left-most node
        # in right subtree
        while cur.left:
            cur = cur.left
        return cur

    while node.parent and node.parent.right == node:
        node = node.parent

    return node.parent

def inorder_traversal(root):
    """
    Question 10.9: Implement an
    inorder traversal in O(1) space
    """
    elts = []
    cur = root
    prev = None

    while cur:
        next_node = None
        if cur.parent == prev:
            # came down to cur from prev
            if cur.left:
                next_node = cur.left
            else:
                elts.append(cur.val)
                # done with left, so go up if
                # right is not empty
                # otherwise go up
                next_node = cur.right if cur.right else cur.parent
        elif cur.left == prev:
            # came up to cur from left child
            elts.append(cur.val)
            next_node = cur.right if cur.right else cur.parent
        else:
            # done with both children so move up
            next_node = cur.parent
        prev = cur
        cur = next_node
    return elts

def reconstruct_tree(preorder, inorder):
    """
    Question 10.10
    """
    node_to_inorder_idx = {}
    for idx, elt in enumerate(inorder):
        node_to_inorder_idx[elt] = idx

    return reconstruct_helper(
        preorder, 0, len(preorder),
        0, len(inorder), node_to_inorder_idx,
    )

def reconstruct_helper(
    preorder, preorder_start, preorder_end,
    inorder_start, inorder_end, node_to_inorder_idx,
):
    if preorder_end <= preorder_start or inorder_end <= inorder_start:
        return None

    # first element in preorder is root
    root_inorder_idx = node_to_inorder_idx[preorder[preorder_start]]
    left_subtree_size = root_inorder_idx - inorder_start

    root = TNode(preorder[preorder_start])
    root.left = reconstruct_helper(
        preorder, preorder_start + 1, preorder_start + 1 + left_subtree_size,
        inorder_start, root_inorder_idx, node_to_inorder_idx,
    )
    root.right = reconstruct_helper(
        preorder, preorder_start + 1 + left_subtree_size, preorder_end,
        root_inorder_idx + 1, inorder_end, node_to_inorder_idx,
    )
    return root

def reconstruct_preorder(preorder):
    """
    Question 10.11: Reconstruct binary tree from
    a preorder representation with markers
    """

    subtree_idx = 0
    root, _ = reconstruct_preorder_helper(preorder, subtree_idx)
    return root

def reconstruct_preorder_helper(preorder, subtree_idx):
    """
    Recursive helper to construct subtree
    """
    subtree_key = TNode(preorder[subtree_idx]) if preorder[
        subtree_idx] else None
    subtree_idx += 1
    if subtree_key is None:
        return None, subtree_idx

    left_subtree, left_idx = reconstruct_preorder_helper(preorder, subtree_idx)
    right_subtree, right_idx = reconstruct_preorder_helper(preorder, left_idx)
    subtree_key.left = left_subtree
    subtree_key.right = right_subtree

    return subtree_key, right_idx

def create_leaf_list(root):
    """
    Question 10.12: Create list from leaves
    of binary tree
    """
    if root.left is None and root.right is None:
        return [root.val]

    ls = []
    if root.left is not None:
        ls.extend(create_leaf_list(root.left))
    if root.right is not None:
        ls.extend(create_leaf_list(root.right))
    return ls

def calculate_tree_exterior(root):
    """
    Question 10.13: Calculate exterior of binary
    tree
    """
    if root is None:
        return []
    return [root.val] + calculate_left_exterior(root.left, True) \
        + calculate_right_exterior(root.right, True)

def calculate_left_exterior(root, is_exterior):
    """
    Build left exterior
    """
    if is_exterior:
        ls = [root.val]
        if root.left is not None:
            ls.extend(calculate_left_exterior(root.left, True))
            ls.extend(calculate_left_exterior(root.right, False))
        elif root.right is not None:
            ls.extend(calculate_left_exterior(root.right, True))
        return ls
    elif root.left is None and root.right is None:
        return [root.val]
    return calculate_left_exterior(root.left, False) \
        + calculate_left_exterior(root.right, False)

def calculate_right_exterior(root, is_exterior):
    """
    Build right exterior
    """
    if root is None:
        return []

    if is_exterior:
        ls = []
        if root.right is not None:
            ls.extend(calculate_right_exterior(root.left, False))
            ls.extend(calculate_right_exterior(root.right, True))
        elif root.left is not None:
            ls.extend(calculate_right_exterior(root.left, True))
        ls.append(root.val)
        return ls
    elif root.right is None and root.left is None:
        return [root.val]
    return calculate_right_exterior(root.right, False) \
        + calculate_right_exterior(root.left, False)
<|endoftext|>


<commit_before>from optparse import OptionParser

from txaws import meta
from txaws import version

# XXX Once we start adding script that require conflicting options, we'll need
# multiple parsers and option dispatching...
def parse_options(usage):
    parser = OptionParser(usage, version="%s %s" % (
        meta.display_name, version.txaws))
    parser.add_option(
        "-a", "--access-key", dest="access_key", help="access key ID")
    parser.add_option(
        "-s", "--secret-key", dest="secret_key", help="access secret key")
    parser.add_option(
        "-r", "--region", dest="region", help="US or EU (valid for AWS only)")
    parser.add_option(
        "-U", "--url", dest="url", help="service URL/endpoint")
    parser.add_option(
        "-b", "--bucket", dest="bucket", help="name of the bucket")
    parser.add_option(
        "-o", "--object-name", dest="object_name", help="name of the object")
    parser.add_option(
        "-d", "--object-data", dest="object_data",
        help="content data of the object")
    parser.add_option(
        "--object-file", dest="object_filename",
        help=("the path to the file that will be saved as an object; if "
               "provided, the --object-name and --object-data options are "
               "not necessary"))
    parser.add_option(
        "-c", "--content-type", dest="content_type",
        help="content type of the object")
    options, args = parser.parse_args()
    if not (options.access_key and options.secret_key):
        parser.error(
            "both the access key ID and the secret key must be supplied")
    region = options.region.upper()
    if region and region not in ["US", "EU"]:
        parser.error("region must be one of 'US' or 'EU'")
    return (options, args)
<commit_msg>Fixed the upper call on the region variable.<commit_after>from optparse import OptionParser

from txaws import meta
from txaws import version

# XXX Once we start adding script that require conflicting options, we'll need
# multiple parsers and option dispatching...
def parse_options(usage):
    parser = OptionParser(usage, version="%s %s" % (
        meta.display_name, version.txaws))
    parser.add_option(
        "-a", "--access-key", dest="access_key", help="access key ID")
    parser.add_option(
        "-s", "--secret-key", dest="secret_key", help="access secret key")
    parser.add_option(
        "-r", "--region", dest="region", help="US or EU (valid for AWS only)")
    parser.add_option(
        "-U", "--url", dest="url", help="service URL/endpoint")
    parser.add_option(
        "-b", "--bucket", dest="bucket", help="name of the bucket")
    parser.add_option(
        "-o", "--object-name", dest="object_name", help="name of the object")
    parser.add_option(
        "-d", "--object-data", dest="object_data",
        help="content data of the object")
    parser.add_option(
        "--object-file", dest="object_filename",
        help=("the path to the file that will be saved as an object; if "
               "provided, the --object-name and --object-data options are "
               "not necessary"))
    parser.add_option(
        "-c", "--content-type", dest="content_type",
        help="content type of the object")
    options, args = parser.parse_args()
    if not (options.access_key and options.secret_key):
        parser.error(
            "both the access key ID and the secret key must be supplied")
    region = options.region
    if region and region.upper() not in ["US", "EU"]:
        parser.error("region must be one of 'US' or 'EU'")
    return (options, args)
<|endoftext|>


<commit_before># Copyright 2016 Louis Kirsch. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import tensorflow as tf
import math
import abc

from tensorflow.contrib.layers import xavier_initializer

class SpeechModel:
  def __init__(self, input_loader, input_size, num_classes, learning_rate, learning_rate_decay_factor,
               max_gradient_norm, log_dir, use_relu, run_name, momentum, run_type, language_model):
    """
    Create a new speech model

    Args:
      input_loader: the object that provides input tensors
      input_size: the number of values per time step
      num_classes: the number of output classes (vocabulary_size + 1 for blank label)
      learning_rate: the inital learning rate
      learning_rate_decay_factor: the factor to multiple the learning rate with when it should be decreased
      max_gradient_norm: the maximum gradient norm to apply, otherwise clipping is applied
      log_dir: the directory to log to for use of tensorboard
      use_relu: if True, use relu instead of tanh
      run_name: the name of this run
      momentum: the momentum parameter
      run_type: "train", "dev" or "test"
      language_model: the file path to the language model to use for beam search decoding or None
    """
    self.input_loader = input_loader
    self.input_size = input_size
    self.convolution_count = 0

    self.activation_fnc = tf.nn.relu if use_relu else tf.nn.tanh

    # inputs is of dimension [batch_size, max_time, input_size]
    self.inputs, self.sequence_lengths, self.labels = input_loader.get_inputs()

    # Define non-trainables
    self.global_step = tf.Variable(0, trainable=False)
    self.learning_rate = tf.Variable(float(learning_rate), trainable=False, dtype=tf.float32, name='learning_rate')
    self.learning_rate_decay_op = self.learning_rate.assign(learning_rate_decay_factor * self.learning_rate)

    # Variable summaries
    tf.summary.scalar('learning_rate', self.learning_rate)

    self.logits = self._create_network(num_classes)

    # Generate summary image for logits [batch_size=batch_size, height=num_classes, width=max_time / 2, channels=1]
    tf.summary.image('logits', tf.expand_dims(tf.transpose(self.logits, (1, 2, 0)), 3))
    tf.summary.histogram('logits', self.logits)

    # Define loss and optimizer
    if self.labels is not None:
      with tf.name_scope('training'):
        self.cost = tf.nn.ctc_loss(self.labels, self.logits, self.sequence_lengths // 2)
        self.avg_loss = tf.reduce_mean(self.cost, name='average_loss')
        tf.summary.scalar('loss', self.avg_loss)
        optimizer = tf.train.AdamOptimizer(self.learning_rate, epsilon=1e-3)
        gvs = optimizer.compute_gradients(self.avg_loss)
        gradients, trainables = zip(*gvs)
        clipped_gradients, norm = tf.clip_by_global_norm(gradients, max_gradient_norm, name='clip_gradients')
        self.update = optimizer.apply_gradients(zip(clipped_gradients, trainables),
                                                global_step=self.global_step, name='apply_gradients')

    # Decoding
    with tf.name_scope('decoding'):
      if language_model:
        self.softmaxed = tf.log(tf.nn.softmax(self.logits, name='softmax') + 1e-8) / math.log(10)
        self.decoded, self.log_probabilities = tf.nn.ctc_beam_search_decoder(self.softmaxed,
                                                                             self.sequence_lengths // 2,
                                                                             kenlm_directory_path=language_model,
                                                                             beam_width=100,
                                                                             top_paths=1)
      else:
        self.decoded, self.log_probabilities = tf.nn.ctc_greedy_decoder(self.logits,
                                                                        self.sequence_lengths // 2)

    # Initializing the variables
    self.init = tf.global_variables_initializer()

    # Create saver
    self.saver = tf.train.Saver(tf.global_variables())

    # Create summary writers
    self.merged_summaries = tf.summary.merge_all()
    if run_name:
      run_name += '_'
    self.summary_writer = tf.summary.FileWriter('{}/{}{}'.format(log_dir, run_name, run_type))

  def _convolution(self, value, filter_width, stride, input_channels, out_channels, apply_non_linearity=True):
    """
    Apply a convolutional layer

    Args:
      value: the input tensor to apply the convolution on
      filter_width: the width of the filter (kernel)
      stride: the striding of the filter (kernel)
      input_channels: the number if input channels
      out_channels: the number of output channels
      apply_non_linearity: whether to apply a non linearity

    Returns:
      the output after convolution, added biases and possible non linearity applied

    """

    layer_id = self.convolution_count
    self.convolution_count += 1

    with tf.variable_scope('convolution_layer_{}'.format(layer_id)) as layer:
      # Create variables filter and bias
      filters = tf.get_variable('filters', shape=[filter_width, input_channels, out_channels],
                                dtype=tf.float32, initializer=xavier_initializer())
      bias = tf.Variable(tf.constant(0.0, shape=[out_channels]), name='bias')

      # Apply convolution
      convolution_out = tf.nn.conv1d(value, filters, stride, 'SAME', use_cudnn_on_gpu=True, name='convolution')

      # Create summary
      with tf.name_scope('summaries'):
        # add depth of 1 (=grayscale) leading to shape [filter_width, input_channels, 1, out_channels]
        kernel_with_depth = tf.expand_dims(filters, 2)

        # to tf.image_summary format [batch_size=out_channels, height=filter_width, width=input_channels, channels=1]
        kernel_transposed = tf.transpose(kernel_with_depth, [3, 0, 1, 2])

        # this will display random 3 filters from all the output channels
        tf.summary.image(layer.name + 'filters', kernel_transposed, max_outputs=3)
        tf.summary.histogram(layer.name + 'filters', filters)

        tf.summary.image(layer.name + 'bias', tf.reshape(bias, [1, 1, out_channels, 1]))
        tf.summary.histogram(layer.name + 'bias', bias)

      # Add bias
      convolution_out = tf.nn.bias_add(convolution_out, bias)

      if apply_non_linearity:
        # Add non-linearity
        activations = self.activation_fnc(convolution_out, name='activation')
        tf.summary.histogram(layer.name + 'activation', activations)
        return activations, out_channels
      else:
        return convolution_out, out_channels

  def init_session(self, sess, init_variables=True):
    """
    Initialize a new session for the model.

    Args:
      sess: session to initalize
      init_variables: whether to initialize all variables

    """
    if init_variables:
      sess.run(self.init)

    self.summary_writer.add_graph(sess.graph)

  def step(self, sess, loss=True, update=True, decode=False, return_label=False, summary=False):
    """
    Evaluate the graph, you may update weights, decode audio or generate a summary

    Args:
      sess: tensorflow session
      update: should the network be trained
      loss: should output the avg_loss
      decode: should the decoding be performed and returned
      return_label: should the label be returned
      summary: should the summary be generated

    Returns: avg_loss (optional), decoded (optional), label (optional), update (optional), summary (optional)

    """

    output_feed = []

    if loss:
      output_feed.append(self.avg_loss)

    if decode:
      output_feed.append(self.decoded)

    if return_label:
      output_feed.append(self.labels)

    if update:
      output_feed.append(self.update)

    if summary:
      output_feed.append(self.merged_summaries)

    return sess.run(output_feed, feed_dict=self.input_loader.get_feed_dict())

  @abc.abstractclassmethod
  def _create_network(self, num_classes):
    """
    Should create the network producing the logits to then be used by the loss function

    Args:
      num_classes: specifies the number of classes to generate as logits

    Returns:
      the logits in the form [time, batch_size, num_classes]

    """
    raise NotImplementedError()

class Wav2LetterModel(SpeechModel):

  def __init__(self, input_loader, input_size, num_classes, learning_rate, learning_rate_decay_factor,
               max_gradient_norm, log_dir, use_relu, run_name, momentum, run_type, language_model=None):
    """
    Create a new Wav2Letter model

    Args:
      input_loader: the object that provides input tensors
      input_size: the number of values per time step
      num_classes: the number of output classes (vocabulary_size + 1 for blank label)
      learning_rate: the inital learning rate
      learning_rate_decay_factor: the factor to multiple the learning rate with when it should be decreased
      max_gradient_norm: the maximum gradient norm to apply, otherwise clipping is applied
      log_dir: the directory to log to for use of tensorboard
      use_relu: if True, use relu instead of tanh
      run_name: the name of this run
      momentum: the momentum parameter
      run_type: "train", "dev" or "test"
      language_model: the file path to the language model to use for beam search decoding or None

    """
    super().__init__(input_loader, input_size, num_classes, learning_rate, learning_rate_decay_factor,
                     max_gradient_norm, log_dir, use_relu, run_name, momentum, run_type, language_model)

  def _create_network(self, num_classes):
    # The first layer scales up from input_size channels to 250 channels
    # One striding layer of output size [batch_size, max_time / 2, 250]
    outputs, channels = self._convolution(self.inputs, 48, 2, self.input_size, 250)

    # 7 layers without striding of output size [batch_size, max_time / 2, 250]
    for layer_idx in range(7):
      outputs, channels = self._convolution(outputs, 7, 1, channels, channels)

    # 1 layer with high kernel width and output size [batch_size, max_time / 2, 2000]
    outputs, channels = self._convolution(outputs, 32, 1, channels, channels * 8)

    # 1 fully connected layer of output size [batch_size, max_time / 2, 2000]
    outputs, channels = self._convolution(outputs, 1, 1, channels, channels)

    # 1 fully connected layer of output size [batch_size, max_time / 2, num_classes]
    # We must not apply a non linearity in this last layer
    outputs, channels = self._convolution(outputs, 1, 1, channels, num_classes, False)

    # transpose logits to size [max_time / 2, batch_size, num_classes]
    return tf.transpose(outputs, (1, 0, 2))
<commit_msg>Fix merge_repeated flags for beam search and greedy decoding<commit_after># Copyright 2016 Louis Kirsch. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import tensorflow as tf
import math
import abc

from tensorflow.contrib.layers import xavier_initializer

class SpeechModel:
  def __init__(self, input_loader, input_size, num_classes, learning_rate, learning_rate_decay_factor,
               max_gradient_norm, log_dir, use_relu, run_name, momentum, run_type, language_model):
    """
    Create a new speech model

    Args:
      input_loader: the object that provides input tensors
      input_size: the number of values per time step
      num_classes: the number of output classes (vocabulary_size + 1 for blank label)
      learning_rate: the inital learning rate
      learning_rate_decay_factor: the factor to multiple the learning rate with when it should be decreased
      max_gradient_norm: the maximum gradient norm to apply, otherwise clipping is applied
      log_dir: the directory to log to for use of tensorboard
      use_relu: if True, use relu instead of tanh
      run_name: the name of this run
      momentum: the momentum parameter
      run_type: "train", "dev" or "test"
      language_model: the file path to the language model to use for beam search decoding or None
    """
    self.input_loader = input_loader
    self.input_size = input_size
    self.convolution_count = 0

    self.activation_fnc = tf.nn.relu if use_relu else tf.nn.tanh

    # inputs is of dimension [batch_size, max_time, input_size]
    self.inputs, self.sequence_lengths, self.labels = input_loader.get_inputs()

    # Define non-trainables
    self.global_step = tf.Variable(0, trainable=False)
    self.learning_rate = tf.Variable(float(learning_rate), trainable=False, dtype=tf.float32, name='learning_rate')
    self.learning_rate_decay_op = self.learning_rate.assign(learning_rate_decay_factor * self.learning_rate)

    # Variable summaries
    tf.summary.scalar('learning_rate', self.learning_rate)

    self.logits = self._create_network(num_classes)

    # Generate summary image for logits [batch_size=batch_size, height=num_classes, width=max_time / 2, channels=1]
    tf.summary.image('logits', tf.expand_dims(tf.transpose(self.logits, (1, 2, 0)), 3))
    tf.summary.histogram('logits', self.logits)

    # Define loss and optimizer
    if self.labels is not None:
      with tf.name_scope('training'):
        self.cost = tf.nn.ctc_loss(self.labels, self.logits, self.sequence_lengths // 2)
        self.avg_loss = tf.reduce_mean(self.cost, name='average_loss')
        tf.summary.scalar('loss', self.avg_loss)
        optimizer = tf.train.AdamOptimizer(self.learning_rate, epsilon=1e-3)
        gvs = optimizer.compute_gradients(self.avg_loss)
        gradients, trainables = zip(*gvs)
        clipped_gradients, norm = tf.clip_by_global_norm(gradients, max_gradient_norm, name='clip_gradients')
        self.update = optimizer.apply_gradients(zip(clipped_gradients, trainables),
                                                global_step=self.global_step, name='apply_gradients')

    # Decoding
    with tf.name_scope('decoding'):
      if language_model:
        self.softmaxed = tf.log(tf.nn.softmax(self.logits, name='softmax') + 1e-8) / math.log(10)
        self.decoded, self.log_probabilities = tf.nn.ctc_beam_search_decoder(self.softmaxed,
                                                                             self.sequence_lengths // 2,
                                                                             kenlm_directory_path=language_model,
                                                                             beam_width=100,
                                                                             merge_repeated=False,
                                                                             top_paths=1)
      else:
        self.decoded, self.log_probabilities = tf.nn.ctc_greedy_decoder(self.logits,
                                                                        self.sequence_lengths // 2,
                                                                        merge_repeated=True)

    # Initializing the variables
    self.init = tf.global_variables_initializer()

    # Create saver
    self.saver = tf.train.Saver(tf.global_variables())

    # Create summary writers
    self.merged_summaries = tf.summary.merge_all()
    if run_name:
      run_name += '_'
    self.summary_writer = tf.summary.FileWriter('{}/{}{}'.format(log_dir, run_name, run_type))

  def _convolution(self, value, filter_width, stride, input_channels, out_channels, apply_non_linearity=True):
    """
    Apply a convolutional layer

    Args:
      value: the input tensor to apply the convolution on
      filter_width: the width of the filter (kernel)
      stride: the striding of the filter (kernel)
      input_channels: the number if input channels
      out_channels: the number of output channels
      apply_non_linearity: whether to apply a non linearity

    Returns:
      the output after convolution, added biases and possible non linearity applied

    """

    layer_id = self.convolution_count
    self.convolution_count += 1

    with tf.variable_scope('convolution_layer_{}'.format(layer_id)) as layer:
      # Create variables filter and bias
      filters = tf.get_variable('filters', shape=[filter_width, input_channels, out_channels],
                                dtype=tf.float32, initializer=xavier_initializer())
      bias = tf.Variable(tf.constant(0.0, shape=[out_channels]), name='bias')

      # Apply convolution
      convolution_out = tf.nn.conv1d(value, filters, stride, 'SAME', use_cudnn_on_gpu=True, name='convolution')

      # Create summary
      with tf.name_scope('summaries'):
        # add depth of 1 (=grayscale) leading to shape [filter_width, input_channels, 1, out_channels]
        kernel_with_depth = tf.expand_dims(filters, 2)

        # to tf.image_summary format [batch_size=out_channels, height=filter_width, width=input_channels, channels=1]
        kernel_transposed = tf.transpose(kernel_with_depth, [3, 0, 1, 2])

        # this will display random 3 filters from all the output channels
        tf.summary.image(layer.name + 'filters', kernel_transposed, max_outputs=3)
        tf.summary.histogram(layer.name + 'filters', filters)

        tf.summary.image(layer.name + 'bias', tf.reshape(bias, [1, 1, out_channels, 1]))
        tf.summary.histogram(layer.name + 'bias', bias)

      # Add bias
      convolution_out = tf.nn.bias_add(convolution_out, bias)

      if apply_non_linearity:
        # Add non-linearity
        activations = self.activation_fnc(convolution_out, name='activation')
        tf.summary.histogram(layer.name + 'activation', activations)
        return activations, out_channels
      else:
        return convolution_out, out_channels

  def init_session(self, sess, init_variables=True):
    """
    Initialize a new session for the model.

    Args:
      sess: session to initalize
      init_variables: whether to initialize all variables

    """
    if init_variables:
      sess.run(self.init)

    self.summary_writer.add_graph(sess.graph)

  def step(self, sess, loss=True, update=True, decode=False, return_label=False, summary=False):
    """
    Evaluate the graph, you may update weights, decode audio or generate a summary

    Args:
      sess: tensorflow session
      update: should the network be trained
      loss: should output the avg_loss
      decode: should the decoding be performed and returned
      return_label: should the label be returned
      summary: should the summary be generated

    Returns: avg_loss (optional), decoded (optional), label (optional), update (optional), summary (optional)

    """

    output_feed = []

    if loss:
      output_feed.append(self.avg_loss)

    if decode:
      output_feed.append(self.decoded)

    if return_label:
      output_feed.append(self.labels)

    if update:
      output_feed.append(self.update)

    if summary:
      output_feed.append(self.merged_summaries)

    return sess.run(output_feed, feed_dict=self.input_loader.get_feed_dict())

  @abc.abstractclassmethod
  def _create_network(self, num_classes):
    """
    Should create the network producing the logits to then be used by the loss function

    Args:
      num_classes: specifies the number of classes to generate as logits

    Returns:
      the logits in the form [time, batch_size, num_classes]

    """
    raise NotImplementedError()

class Wav2LetterModel(SpeechModel):

  def __init__(self, input_loader, input_size, num_classes, learning_rate, learning_rate_decay_factor,
               max_gradient_norm, log_dir, use_relu, run_name, momentum, run_type, language_model=None):
    """
    Create a new Wav2Letter model

    Args:
      input_loader: the object that provides input tensors
      input_size: the number of values per time step
      num_classes: the number of output classes (vocabulary_size + 1 for blank label)
      learning_rate: the inital learning rate
      learning_rate_decay_factor: the factor to multiple the learning rate with when it should be decreased
      max_gradient_norm: the maximum gradient norm to apply, otherwise clipping is applied
      log_dir: the directory to log to for use of tensorboard
      use_relu: if True, use relu instead of tanh
      run_name: the name of this run
      momentum: the momentum parameter
      run_type: "train", "dev" or "test"
      language_model: the file path to the language model to use for beam search decoding or None

    """
    super().__init__(input_loader, input_size, num_classes, learning_rate, learning_rate_decay_factor,
                     max_gradient_norm, log_dir, use_relu, run_name, momentum, run_type, language_model)

  def _create_network(self, num_classes):
    # The first layer scales up from input_size channels to 250 channels
    # One striding layer of output size [batch_size, max_time / 2, 250]
    outputs, channels = self._convolution(self.inputs, 48, 2, self.input_size, 250)

    # 7 layers without striding of output size [batch_size, max_time / 2, 250]
    for layer_idx in range(7):
      outputs, channels = self._convolution(outputs, 7, 1, channels, channels)

    # 1 layer with high kernel width and output size [batch_size, max_time / 2, 2000]
    outputs, channels = self._convolution(outputs, 32, 1, channels, channels * 8)

    # 1 fully connected layer of output size [batch_size, max_time / 2, 2000]
    outputs, channels = self._convolution(outputs, 1, 1, channels, channels)

    # 1 fully connected layer of output size [batch_size, max_time / 2, num_classes]
    # We must not apply a non linearity in this last layer
    outputs, channels = self._convolution(outputs, 1, 1, channels, num_classes, False)

    # transpose logits to size [max_time / 2, batch_size, num_classes]
    return tf.transpose(outputs, (1, 0, 2))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import socket

from bs4 import BeautifulSoup
import urllib
# import urllib2
import requests
import time
import xlwt

if __name__ == "__main__":
    start = time.clock()
    # f=urllib2.urlopen(url, timeout=5).read()
    # soup=BeautifulSoup(html)
    # tags=soup.find_all('a')
    file = open("data-detail.txt", encoding='utf8')

# 写入Excel
    wb = xlwt.Workbook("data_out.xls")
    sheet = wb.add_sheet("data-out")
    sheet.write(0, 0, '下载网址')
    sheet.write(0, 1, '标题')
    sheet.write(0, 2, '来源')
    sheet.write(0, 3, '引用')
    sheet.write(0, 4, '作者')
    sheet.write(0, 5, '摘要')

    lines = file.readlines()
    txt_num = 1
    lin_num = 1
    for line in lines:
        object = line.split('\t')
        print(object)
        # file_name = './data/out_' + str(txt_num) + '.txt'
        file_name = './data/out.txt'
        paper_url = object[0]
        attempts = 0
        success = False
        while attempts < 10 and not success:
            try:
                html = urllib.request.urlopen(paper_url).read()
                soup = BeautifulSoup(html, 'html.parser')
                socket.setdefaulttimeout(10)  # 设置10秒后连接超时
                success = True
            except:
                attempts += 1
                print("第"+str(attempts)+"次重试！！")
                if attempts == 10:
                    break

        title = soup.find_all('div', style="text-align:center; width:740px; font-size: 28px;color: #0000a0; font-weight:bold; font-family:'宋体';")
        abstract = soup.find_all('div', style="text-align:left;word-break:break-all")
        author = soup.findAll('div', style='text-align:center; width:740px; height:30px;')
        for item in author:
            author = item.get_text()
        # print(item)
        #fuck = open(file_name, 'a', encoding='utf-8')
        tmp = ''
        for thing in abstract:
            a = thing.strings
            for string in a:
                tmp = tmp + string
            txt_num += 1
        result = tmp.split(' ')
        tstr = ''
        for t in result:
            test = t.split('\n')
            # print(test)
            if test != '\t' and test != '\n' and test != '\r' and test != '':
                for i in test:
                    if len(i) > 1:
                        item = i.split('\r')
                        for j in item:
                            object = j.split('\t')
                            for k in object:
                                tstr += k
        line = line.strip('\n')
        line = line + '\t' + str(author) + '\t' + str(tstr) + '\n'
        outstring = line.split('\t')
        for i in range(len(outstring)):
            sheet.write(lin_num, i, outstring[i])
        print('写入第'+str(lin_num)+'行')
        lin_num += 1
        wb.save('data_out.xls')
        #fuck.write(line)
        #fuck.close()
    file.close()
    end = time.clock()
    print('Running time: %s Seconds' % (end - start))<commit_msg>添加爬取参考文献<commit_after># -*- coding: utf-8 -*-
import socket

from bs4 import BeautifulSoup
import urllib
import time
import xlwt

def spider_paper():
    start = time.clock()
    # f=urllib2.urlopen(url, timeout=5).read()
    # soup=BeautifulSoup(html)
    # tags=soup.find_all('a')
    file = open("data-detail.txt", encoding='utf8')

# 写入Excel
    wb = xlwt.Workbook("data_out.xls")
    sheet = wb.add_sheet("data-out")
    sheet.write(0, 0, '下载网址')
    sheet.write(0, 1, '标题')
    sheet.write(0, 2, '来源')
    sheet.write(0, 3, '引用')
    sheet.write(0, 4, '作者')
    sheet.write(0, 5, '摘要')
    sheet.write(0, 6, '参考文献')

    lines = file.readlines()
    txt_num = 1
    lin_num = 1
    for line in lines:
        object = line.split('\t')
        # file_name = './data/out_' + str(txt_num) + '.txt'
        file_name = './data/out.txt'
        paper_url = object[0]
        attempts = 0
        success = False
        while attempts < 20 and not success:
            try:
                html = urllib.request.urlopen(paper_url).read()
                soup = BeautifulSoup(html, 'html.parser')
                socket.setdefaulttimeout(10)  # 设置10秒后连接超时
                success = True
            except:
                attempts += 1
                print("第"+str(attempts)+"次重试！！")
                if attempts == 10:
                    break

        title = soup.find_all('div', style="text-align:center; width:740px; font-size: 28px;color: #0000a0; font-weight:bold; font-family:'宋体';")
        abstract = soup.find_all('div', style="text-align:left;word-break:break-all")
        author = soup.findAll('div', style='text-align:center; width:740px; height:30px;')
        for item in author:
            author = item.get_text()
        # print(item)
        #fuck = open(file_name, 'a', encoding='utf-8')
        tmp = ''
        for thing in abstract:
            a = thing.strings
            for string in a:
                tmp = tmp + string
            txt_num += 1
        result = tmp.split(' ')
        tstr = ''
        for t in result:
            test = t.split('\n')
            # print(test)
            if test != '\t' and test != '\n' and test != '\r' and test != '':
                for i in test:
                    if len(i) > 1:
                        item = i.split('\r')
                        for j in item:
                            object = j.split('\t')
                            for k in object:
                                tstr += k

        #参考文献
        ifreferen = soup.find_all('td', class_='b14', rowspan='2')
        ref = ''
        for i in range(len(ifreferen)):
            if ('【参考文献】' in ifreferen[i].get_text()):
                referenceList = soup.find_all('div', id='div_Ref')  # 参考文献列表
                if len(referenceList) == 0:
                    referenceList = soup.find_all('div', class_='div_Ref')
                referenceList = referenceList[i]
                for tdref in referenceList.find_all('td', width='676'):
                    # refitem = tdref.findAll('a', target="_blank")
                    refitem = tdref.a.get("href")
                    refitem = refitem.strip()
                    #print(refitem)
                    ref = ref + refitem + ' ,'

        line = line.strip('\n')
        line = line + '\t' + str(author) + '\t' + str(tstr) + '\t' + str(ref) + '\n'
        #print(line)
        outstring = line.split('\t')
        for i in range(len(outstring)):
            sheet.write(lin_num, i, outstring[i])
        print('写入第'+str(lin_num)+'行')
        lin_num += 1
        wb.save('data_out.xls')
        #fuck.write(line)
        #fuck.close()
    file.close()
    end = time.clock()
    print('Running time: %s Seconds' % (end - start))
<|endoftext|>