<commit_before>from django.test import TestCase
from django.test.utils import override_settings
from django.test.client import RequestFactory

from django.core.urlresolvers import reverse
from django.template import Template, Context

from django.contrib.auth.models import AnonymousUser

from django.conf import get_user_model
User = get_user_model()

from django_dynamic_fixture import N, G

from django_webtest import WebTest

from ..models import WishlistItem

from ..context_processors import wishlist_items

from .models import TestItemModel

@override_settings(WISHLIST_ITEM_MODEL='tests.TestItemModel')
class WishlistUnitTests(TestCase):
    """ Unit tests for wishlists. """

    def test_save(self):
        """ Test saving a WishlistItem. """

        # Create
        item = N(WishlistItem)

        # Validate
        item.clean()

        # Save
        item.save()

    def test_unicode(self):
        """ Test __unicode__ """
        item = G(WishlistItem)

        self.assertEquals(unicode(item), unicode(item.item))

    def test_absolute_url(self):
        """ Test get_absolute_url() """
        item = G(WishlistItem)

        self.assertEquals(
            item.get_absolute_url(),
            item.item.get_absolute_url()
        )

    def test_manager(self):
        """ Test custom user manager """

        # Create two users and wishlist items
        user1 = G(User)
        user2 = G(User)

        item1 = G(WishlistItem, user=user1)
        item2 = G(WishlistItem, user=user2)

        # Only this item should be available
        self.assertEquals(
            WishlistItem.objects.for_user(user=user1).count(),
            1
        )

        self.assertEquals(
            WishlistItem.objects.for_user(user=user1)[0],
            item1
        )

        self.assertEquals(
            WishlistItem.objects.for_user(user=user2).count(),
            1
        )

        self.assertEquals(
            WishlistItem.objects.for_user(user=user2)[0],
            item2
        )

    def test_context_processor(self):
        """ Test wishlist_items context processor. """

        # Create one item for testing
        item = G(WishlistItem)

        # Request factory and request
        factory = RequestFactory()
        request = factory.get('/')

        # Anonymous user
        request.user = AnonymousUser()

        # No context for anonymous users
        result = wishlist_items(request)
        self.assertEquals(result, {})

        # User set, should return items for user
        request.user = item.user
        result = wishlist_items(request)

        self.assertEquals(len(result['wishlist_items']), 1)

        self.assertEquals(
            result['wishlist_items'][0],
            WishlistItem.objects.for_user(user=item.user)[0]
        )

    def test_add_form(self):
        """ Test wishlist_add_form template tag. """

        # Create one wished item for testing
        wished_item = G(TestItemModel)

        # Get URL for add view
        add_view = reverse('wishlist_add')

        # Create user for testing
        user = G(User)

        # Setup and render template
        template = Template(
            '{% load wishlist_tags %}{% wishlist_add_form item %}'
        )
        context = Context(dict(item=wished_item, user=user))
        rendered = template.render(context)

        # Test output
        self.assertHTMLEqual(
            '<form action="{0}" method="post">'
            '<input type="hidden" name="item" value="{1}">'
            '<input type="submit" value="Add">'
            '</form>'.format(add_view, wished_item.pk),
            rendered
        )

    def test_remove_form(self):
        """ Test wishlist_remove_form template tag. """

        # Create one item for testing
        wishlist_item = G(WishlistItem)

        # Remove url
        remove_view = reverse(
            'wishlist_remove', kwargs=dict(pk=wishlist_item.pk)
        )

        # Setup and render template
        template = Template(
            '{% load wishlist_tags %}{% wishlist_remove_form item %}'
        )
        context = Context(
            dict(item=wishlist_item.item, user=wishlist_item.user)
        )
        rendered = template.render(context)

        # Test output
        self.assertHTMLEqual(
            '<form action="{0}" method="post">'
            '<input type="submit" value="Remove">'
            '</form>'.format(
                remove_view
            ), rendered
        )

@override_settings(WISHLIST_ITEM_MODEL='tests.TestItemModel')
class WishlistFunctionalTests(WebTest):
    """ Functional/integration tests for views. """

    def test_login_required(self):
        """ Assure login is required. """

        view = reverse('wishlist_add')

        result = self.app.get(view)

        self.assertEquals(result.status_int, 302)
        self.assertIn('login', result.url)

    def test_add(self):
        """ Test adding an item. """

        # Get URL for add view
        add_view = reverse('wishlist_add')
        list_view = reverse('wishlist')

        # Create an item to add
        wished_item = G(TestItemModel, slug='test')

        # Create a user and login
        user = G(User)

        # Request add view
        add_page = self.app.get(add_view, user=user)

        # Fill in form and post
        add_form = add_page.form
        add_form['item'] = wished_item.pk
        add_result = add_form.submit()

        # Assert WishlistItem has been created
        self.assertEquals(WishlistItem.objects.count(), 1)

        item = WishlistItem.objects.get()
        self.assertEquals(item.user, user)
        self.assertEquals(item.item_id, wished_item.pk)

        # Assert redirect location
        self.assertTrue(add_result.location.endswith(list_view))

        # Test redirect after adding
        result = add_result.follow()

        # Test messages after adding
        self.assertContains(result, unicode(item))
        self.assertContains(result, u'added to the wishlist')

    def test_add_twice(self):
        """
        Test adding an item twice.

        Should add a warning message and not generate an exception.
        """

        # Get URL for add view
        add_view = reverse('wishlist_add')
        list_view = reverse('wishlist')

        # Create an item to add
        wished_item = G(TestItemModel, slug='test')

        # Create a user and login
        user = G(User)

        # Request add view
        add_page = self.app.get(add_view, user=user)

        # Fill in form and post
        add_form = add_page.form
        add_form['item'] = wished_item.pk
        add_result = add_form.submit()

        # Assert WishlistItem has been created
        self.assertEquals(WishlistItem.objects.count(), 1)

        # Test redirect after adding
        result = add_result.follow()
        self.assertContains(result, u'added to the wishlist')

        # Post again
        add_result = add_form.submit()

        # Assert WishlistItem still in wishlist
        self.assertEquals(WishlistItem.objects.count(), 1)

        # Assert redirect location
        self.assertTrue(add_result.location.endswith(list_view))

        # Test redirect after adding
        result = add_result.follow()

        # Test redirect after adding
        self.assertContains(result, u'already in the wishlist')

    def test_remove_confirm(self):
        """ Test removal confirm form. """

        # Create wishlist item
        item = G(WishlistItem)

        # Get URL for remove view
        remove_view = reverse('wishlist_remove', kwargs=dict(pk=item.pk))

        # Request view
        page = self.app.get(remove_view, user=item.user)
        self.assertContains(page, 'Confirm')

        # Post form
        page.form.submit()

        # Assert WishlistItem has been removed
        self.assertEquals(WishlistItem.objects.count(), 0)

    def test_list(self):
        """ Test list view. """

        # Create wishlist item
        item = G(WishlistItem)

        # Get URL for list view
        list_view = reverse('wishlist')

        # Request view
        list_page = self.app.get(list_view, user=item.user)

        self.assertContains(list_page, unicode(item))
        self.assertContains(list_page, item.get_absolute_url())
        self.assertContains(list_page, '1 item in wishlist')

        # Add another item for same user
        item2 = G(WishlistItem, user=item.user)

        # Request view
        list_page = self.app.get(list_view, user=item.user)

        self.assertContains(list_page, unicode(item))
        self.assertContains(list_page, item.get_absolute_url())
        self.assertContains(list_page, unicode(item2))
        self.assertContains(list_page, item2.get_absolute_url())
        self.assertContains(list_page, '2 items in wishlist')

    def test_remove(self):
        """ Test removing an item. """

        # Create wishlist item
        item = G(WishlistItem)

        # Get URL for list view
        list_view = reverse('wishlist')

        # Request view
        list_page = self.app.get(list_view, user=item.user)

        # Find and post remove form
        remove_form = list_page.form
        remove_result = remove_form.submit()

        # Assert WishlistItem has been removed
        self.assertEquals(WishlistItem.objects.count(), 0)

        # Perform redirect after adding
        result = remove_result.follow()

        # Test messages after adding
        self.assertContains(result, unicode(item))
        self.assertContains(result, u'removed from the wishlist')

    def test_permission(self):
        """ List for one user should be empty for another. """

        # Create wishlist item
        item = G(WishlistItem)

        # Get URL for list view
        list_view = reverse('wishlist')

        # Request view
        list_page = self.app.get(list_view, user=item.user)

        # Item for creating user
        self.assertContains(list_page, unicode(item))

        # Request view with another user
        list_page = self.app.get(list_view, user=G(User))

        # No item for other user
        self.assertNotContains(list_page, unicode(item))

    def test_clear(self):
        """ Test clearing the wishlist. """

        # Begin at list view with single item
        # Create wishlist item
        item = G(WishlistItem)

        # Get URL for list view
        list_view = reverse('wishlist')

        # Request view
        list_page = self.app.get(list_view, user=item.user)

        # Click clear button
        clear_page = list_page.click('Clear wishlist')
        self.assertContains(clear_page, 'Confirm')

        # Post form
        clear_page.form.submit()

        # Assert WishlistItem has been removed
        self.assertEquals(WishlistItem.objects.count(), 0)
<commit_msg>User model now comes from elsewhere.<commit_after>from django.test import TestCase
from django.test.utils import override_settings
from django.test.client import RequestFactory

from django.core.urlresolvers import reverse
from django.template import Template, Context

from django.contrib.auth.models import AnonymousUser

from django.contrib.auth import get_user_model
User = get_user_model()

from django_dynamic_fixture import N, G

from django_webtest import WebTest

from ..models import WishlistItem

from ..context_processors import wishlist_items

from .models import TestItemModel

@override_settings(WISHLIST_ITEM_MODEL='tests.TestItemModel')
class WishlistUnitTests(TestCase):
    """ Unit tests for wishlists. """

    def test_save(self):
        """ Test saving a WishlistItem. """

        # Create
        item = N(WishlistItem)

        # Validate
        item.clean()

        # Save
        item.save()

    def test_unicode(self):
        """ Test __unicode__ """
        item = G(WishlistItem)

        self.assertEquals(unicode(item), unicode(item.item))

    def test_absolute_url(self):
        """ Test get_absolute_url() """
        item = G(WishlistItem)

        self.assertEquals(
            item.get_absolute_url(),
            item.item.get_absolute_url()
        )

    def test_manager(self):
        """ Test custom user manager """

        # Create two users and wishlist items
        user1 = G(User)
        user2 = G(User)

        item1 = G(WishlistItem, user=user1)
        item2 = G(WishlistItem, user=user2)

        # Only this item should be available
        self.assertEquals(
            WishlistItem.objects.for_user(user=user1).count(),
            1
        )

        self.assertEquals(
            WishlistItem.objects.for_user(user=user1)[0],
            item1
        )

        self.assertEquals(
            WishlistItem.objects.for_user(user=user2).count(),
            1
        )

        self.assertEquals(
            WishlistItem.objects.for_user(user=user2)[0],
            item2
        )

    def test_context_processor(self):
        """ Test wishlist_items context processor. """

        # Create one item for testing
        item = G(WishlistItem)

        # Request factory and request
        factory = RequestFactory()
        request = factory.get('/')

        # Anonymous user
        request.user = AnonymousUser()

        # No context for anonymous users
        result = wishlist_items(request)
        self.assertEquals(result, {})

        # User set, should return items for user
        request.user = item.user
        result = wishlist_items(request)

        self.assertEquals(len(result['wishlist_items']), 1)

        self.assertEquals(
            result['wishlist_items'][0],
            WishlistItem.objects.for_user(user=item.user)[0]
        )

    def test_add_form(self):
        """ Test wishlist_add_form template tag. """

        # Create one wished item for testing
        wished_item = G(TestItemModel)

        # Get URL for add view
        add_view = reverse('wishlist_add')

        # Create user for testing
        user = G(User)

        # Setup and render template
        template = Template(
            '{% load wishlist_tags %}{% wishlist_add_form item %}'
        )
        context = Context(dict(item=wished_item, user=user))
        rendered = template.render(context)

        # Test output
        self.assertHTMLEqual(
            '<form action="{0}" method="post">'
            '<input type="hidden" name="item" value="{1}">'
            '<input type="submit" value="Add">'
            '</form>'.format(add_view, wished_item.pk),
            rendered
        )

    def test_remove_form(self):
        """ Test wishlist_remove_form template tag. """

        # Create one item for testing
        wishlist_item = G(WishlistItem)

        # Remove url
        remove_view = reverse(
            'wishlist_remove', kwargs=dict(pk=wishlist_item.pk)
        )

        # Setup and render template
        template = Template(
            '{% load wishlist_tags %}{% wishlist_remove_form item %}'
        )
        context = Context(
            dict(item=wishlist_item.item, user=wishlist_item.user)
        )
        rendered = template.render(context)

        # Test output
        self.assertHTMLEqual(
            '<form action="{0}" method="post">'
            '<input type="submit" value="Remove">'
            '</form>'.format(
                remove_view
            ), rendered
        )

@override_settings(WISHLIST_ITEM_MODEL='tests.TestItemModel')
class WishlistFunctionalTests(WebTest):
    """ Functional/integration tests for views. """

    def test_login_required(self):
        """ Assure login is required. """

        view = reverse('wishlist_add')

        result = self.app.get(view)

        self.assertEquals(result.status_int, 302)
        self.assertIn('login', result.url)

    def test_add(self):
        """ Test adding an item. """

        # Get URL for add view
        add_view = reverse('wishlist_add')
        list_view = reverse('wishlist')

        # Create an item to add
        wished_item = G(TestItemModel, slug='test')

        # Create a user and login
        user = G(User)

        # Request add view
        add_page = self.app.get(add_view, user=user)

        # Fill in form and post
        add_form = add_page.form
        add_form['item'] = wished_item.pk
        add_result = add_form.submit()

        # Assert WishlistItem has been created
        self.assertEquals(WishlistItem.objects.count(), 1)

        item = WishlistItem.objects.get()
        self.assertEquals(item.user, user)
        self.assertEquals(item.item_id, wished_item.pk)

        # Assert redirect location
        self.assertTrue(add_result.location.endswith(list_view))

        # Test redirect after adding
        result = add_result.follow()

        # Test messages after adding
        self.assertContains(result, unicode(item))
        self.assertContains(result, u'added to the wishlist')

    def test_add_twice(self):
        """
        Test adding an item twice.

        Should add a warning message and not generate an exception.
        """

        # Get URL for add view
        add_view = reverse('wishlist_add')
        list_view = reverse('wishlist')

        # Create an item to add
        wished_item = G(TestItemModel, slug='test')

        # Create a user and login
        user = G(User)

        # Request add view
        add_page = self.app.get(add_view, user=user)

        # Fill in form and post
        add_form = add_page.form
        add_form['item'] = wished_item.pk
        add_result = add_form.submit()

        # Assert WishlistItem has been created
        self.assertEquals(WishlistItem.objects.count(), 1)

        # Test redirect after adding
        result = add_result.follow()
        self.assertContains(result, u'added to the wishlist')

        # Post again
        add_result = add_form.submit()

        # Assert WishlistItem still in wishlist
        self.assertEquals(WishlistItem.objects.count(), 1)

        # Assert redirect location
        self.assertTrue(add_result.location.endswith(list_view))

        # Test redirect after adding
        result = add_result.follow()

        # Test redirect after adding
        self.assertContains(result, u'already in the wishlist')

    def test_remove_confirm(self):
        """ Test removal confirm form. """

        # Create wishlist item
        item = G(WishlistItem)

        # Get URL for remove view
        remove_view = reverse('wishlist_remove', kwargs=dict(pk=item.pk))

        # Request view
        page = self.app.get(remove_view, user=item.user)
        self.assertContains(page, 'Confirm')

        # Post form
        page.form.submit()

        # Assert WishlistItem has been removed
        self.assertEquals(WishlistItem.objects.count(), 0)

    def test_list(self):
        """ Test list view. """

        # Create wishlist item
        item = G(WishlistItem)

        # Get URL for list view
        list_view = reverse('wishlist')

        # Request view
        list_page = self.app.get(list_view, user=item.user)

        self.assertContains(list_page, unicode(item))
        self.assertContains(list_page, item.get_absolute_url())
        self.assertContains(list_page, '1 item in wishlist')

        # Add another item for same user
        item2 = G(WishlistItem, user=item.user)

        # Request view
        list_page = self.app.get(list_view, user=item.user)

        self.assertContains(list_page, unicode(item))
        self.assertContains(list_page, item.get_absolute_url())
        self.assertContains(list_page, unicode(item2))
        self.assertContains(list_page, item2.get_absolute_url())
        self.assertContains(list_page, '2 items in wishlist')

    def test_remove(self):
        """ Test removing an item. """

        # Create wishlist item
        item = G(WishlistItem)

        # Get URL for list view
        list_view = reverse('wishlist')

        # Request view
        list_page = self.app.get(list_view, user=item.user)

        # Find and post remove form
        remove_form = list_page.form
        remove_result = remove_form.submit()

        # Assert WishlistItem has been removed
        self.assertEquals(WishlistItem.objects.count(), 0)

        # Perform redirect after adding
        result = remove_result.follow()

        # Test messages after adding
        self.assertContains(result, unicode(item))
        self.assertContains(result, u'removed from the wishlist')

    def test_permission(self):
        """ List for one user should be empty for another. """

        # Create wishlist item
        item = G(WishlistItem)

        # Get URL for list view
        list_view = reverse('wishlist')

        # Request view
        list_page = self.app.get(list_view, user=item.user)

        # Item for creating user
        self.assertContains(list_page, unicode(item))

        # Request view with another user
        list_page = self.app.get(list_view, user=G(User))

        # No item for other user
        self.assertNotContains(list_page, unicode(item))

    def test_clear(self):
        """ Test clearing the wishlist. """

        # Begin at list view with single item
        # Create wishlist item
        item = G(WishlistItem)

        # Get URL for list view
        list_view = reverse('wishlist')

        # Request view
        list_page = self.app.get(list_view, user=item.user)

        # Click clear button
        clear_page = list_page.click('Clear wishlist')
        self.assertContains(clear_page, 'Confirm')

        # Post form
        clear_page.form.submit()

        # Assert WishlistItem has been removed
        self.assertEquals(WishlistItem.objects.count(), 0)
<|endoftext|>


<commit_before>import pytest
import requests
import os

http_port = os.environ.get("TOMCAT_HTTP_PORT", 8080)

@pytest.mark.parametrize("name,version", [
    ("openjdk-8-jre", "8"),
    ("tomcat7", "7"),
])
def test_packages(host, name, version):
    pkg = host.package(name)
    assert pkg.is_installed
    assert pkg.version.startswith(version)

def test_tomcat_installed(host):
    service = host.service("tomcat7")
    #assert service.is_running
    assert service.is_enabled

    assert host.socket("tcp://0.0.0.0:{}".format(http_port)).is_listening

    res = requests.get("http://localhost:{}/".format(http_port))
    assert res.status_code == 200

def test_gitbucket(host):
    war = host.file("/var/lib/tomcat7/webapps/gitbucket.war")
    assert war.exists
    app_dir = host.file("/var/lib/tomcat7/webapps/gitbucket")
    assert app_dir.exists
    assert app_dir.is_directory
    assert app_dir.user = "tomcat7"
    assert passwd.mode >= 0o600

    res = requests.get("http://localhost:{}/gitbucket".format(http_port))
    assert res.status_code == 200
    assert res.text.find("<title>GitBucket</title>") > 0
<commit_msg>fix typo<commit_after>import pytest
import requests
import os

http_port = os.environ.get("TOMCAT_HTTP_PORT", 8080)

@pytest.mark.parametrize("name,version", [
    ("openjdk-8-jre", "8"),
    ("tomcat7", "7"),
])
def test_packages(host, name, version):
    pkg = host.package(name)
    assert pkg.is_installed
    assert pkg.version.startswith(version)

def test_tomcat_installed(host):
    service = host.service("tomcat7")
    #assert service.is_running
    assert service.is_enabled

    assert host.socket("tcp://0.0.0.0:{}".format(http_port)).is_listening

    res = requests.get("http://localhost:{}/".format(http_port))
    assert res.status_code == 200

def test_gitbucket(host):
    war = host.file("/var/lib/tomcat7/webapps/gitbucket.war")
    assert war.exists
    app_dir = host.file("/var/lib/tomcat7/webapps/gitbucket")
    assert app_dir.exists
    assert app_dir.is_directory
    assert app_dir.user == "tomcat7"
    assert passwd.mode >= 0o600

    res = requests.get("http://localhost:{}/gitbucket".format(http_port))
    assert res.status_code == 200
    assert res.text.find("<title>GitBucket</title>") > 0
<|endoftext|>


<commit_before>__author__ = 'Riley Flynn (nint8835)'
__version__ = "03.30.2016.1"
<commit_msg>Updated version number<commit_after>__author__ = 'Riley Flynn (nint8835)'
__version__ = "03.30.2016.2"
<|endoftext|>


<commit_before>import os
import random
import string
import sys

# add the parent parent directory to sys.path
# XXX this is very hackish :(
import os,sys,inspect
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.append(parentdir)
parentparentdir = os.path.dirname(parentdir)
sys.path.append(parentparentdir)

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
import django
django.setup()

from tosca.engine import XOSTosca
from core.models import User

class BaseToscaTest(object):
    username = "padmin@vicci.org"
    base_yaml = \
"""tosca_definitions_version: tosca_simple_yaml_1_0

description: tosca test case

imports:
   - custom_types/xos.yaml

topology_template:
  node_templates:
"""

    def __init__(self):
        self.runtest()

    def make_nodetemplate(self, name, type, props={}, reqs=[], caps={}, artifacts={}):
        yml = "    %s:\n      type: %s\n"  % (name, type)
        if props:
            yml = yml + "      properties:\n"
            for (k,v) in props.items():
                yml = yml + "          %s: %s\n" % (k, v)

        if reqs:
            yml = yml + "      requirements:\n"
            i=0
            for (name,relat) in reqs:
                yml = yml + "        - req%d:\n" % i
                yml = yml + "              node: %s\n" % name
                yml = yml + "              relationship: %s\n" % relat
                i = i + 1

        if caps:
            yml = yml + "      capabilities:\n"
            for (cap,capdict) in caps.items():
               yml = yml + "        %s:\n" % cap
               yml = yml + "          properties:\n"
               for (k,v) in capdict.items():
                   yml = yml + "            %s: %s\n" % (k,v)

        if artifacts:
            yml = yml + "      artifacts:\n"
            for (k,v) in artifacts.items():
                yml = yml + "        %s: %s\n" % (k,v)

        return yml

    def make_compute(self, slice, name, caps={}, props={}, reqs=[], num_cpus="1", disk_size="10 GB", mem_size="4 MB"):
        reqs = reqs[:]
        caps = caps.copy()

        caps.update( {"host": {"num_cpus": num_cpus, "disk_size": disk_size, "mem_size": mem_size},
                      "os": {"architecture": "x86_64", "type": "linux", "distribution": "rhel", "version": "6.5"}} )
        reqs.append( (slice, "tosca.relationships.MemberOfSlice") )

        return self.make_nodetemplate(name, "tosca.nodes.Compute",
                                      caps= caps,
                                      props = props,
                                      reqs= reqs)

    def make_random_string(self,desired_len):
        return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(desired_len))

    def assert_noobj(self, cls, name):
        objs = cls.objects.filter(name=name)
        assert(not objs)

    def assert_obj(self, cls, name, **kwargs):
        obj = cls.objects.get(name=name)
        assert(obj)
        for (k,v) in kwargs.items():
            if (getattr(obj,k,None) != v):
                print "Object %s property '%s' is '%s' and should be '%s'" % (obj, k, getattr(obj,k,None), v)
                assert(False)
        return obj

    def try_to_delete(self, cls, **kwargs):
        objs = cls.objects.filter(**kwargs)
        for obj in objs:
            obj.delete(purge=True)

    def execute(self, yml):
        u = User.objects.get(email=self.username)

        #print self.base_yaml+yml

        xt = XOSTosca(self.base_yaml+yml, parent_dir=parentdir, log_to_console=False)
        xt.execute(u)

    def destroy(self, yml):
        u = User.objects.get(email=self.username)

        #print self.base_yaml+yml

        xt = XOSTosca(self.base_yaml+yml, parent_dir=parentdir, log_to_console=False)
        xt.destroy(u)

    def runtest(self):
        for test in self.tests:
            print "running", test
            self.cleanup()
            try:
                getattr(self,test)()
            finally:
                self.cleanup()

    def cleanup(self):
        pass
<commit_msg>add purge option to try_to_delete<commit_after>import os
import random
import string
import sys

# add the parent parent directory to sys.path
# XXX this is very hackish :(
import os,sys,inspect
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.append(parentdir)
parentparentdir = os.path.dirname(parentdir)
sys.path.append(parentparentdir)

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
import django
django.setup()

from tosca.engine import XOSTosca
from core.models import User

class BaseToscaTest(object):
    username = "padmin@vicci.org"
    base_yaml = \
"""tosca_definitions_version: tosca_simple_yaml_1_0

description: tosca test case

imports:
   - custom_types/xos.yaml

topology_template:
  node_templates:
"""

    def __init__(self):
        self.runtest()

    def make_nodetemplate(self, name, type, props={}, reqs=[], caps={}, artifacts={}):
        yml = "    %s:\n      type: %s\n"  % (name, type)
        if props:
            yml = yml + "      properties:\n"
            for (k,v) in props.items():
                yml = yml + "          %s: %s\n" % (k, v)

        if reqs:
            yml = yml + "      requirements:\n"
            i=0
            for (name,relat) in reqs:
                yml = yml + "        - req%d:\n" % i
                yml = yml + "              node: %s\n" % name
                yml = yml + "              relationship: %s\n" % relat
                i = i + 1

        if caps:
            yml = yml + "      capabilities:\n"
            for (cap,capdict) in caps.items():
               yml = yml + "        %s:\n" % cap
               yml = yml + "          properties:\n"
               for (k,v) in capdict.items():
                   yml = yml + "            %s: %s\n" % (k,v)

        if artifacts:
            yml = yml + "      artifacts:\n"
            for (k,v) in artifacts.items():
                yml = yml + "        %s: %s\n" % (k,v)

        return yml

    def make_compute(self, slice, name, caps={}, props={}, reqs=[], num_cpus="1", disk_size="10 GB", mem_size="4 MB"):
        reqs = reqs[:]
        caps = caps.copy()

        caps.update( {"host": {"num_cpus": num_cpus, "disk_size": disk_size, "mem_size": mem_size},
                      "os": {"architecture": "x86_64", "type": "linux", "distribution": "rhel", "version": "6.5"}} )
        reqs.append( (slice, "tosca.relationships.MemberOfSlice") )

        return self.make_nodetemplate(name, "tosca.nodes.Compute",
                                      caps= caps,
                                      props = props,
                                      reqs= reqs)

    def make_random_string(self,desired_len):
        return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(desired_len))

    def assert_noobj(self, cls, name):
        objs = cls.objects.filter(name=name)
        assert(not objs)

    def assert_obj(self, cls, name, **kwargs):
        obj = cls.objects.get(name=name)
        assert(obj)
        for (k,v) in kwargs.items():
            if (getattr(obj,k,None) != v):
                print "Object %s property '%s' is '%s' and should be '%s'" % (obj, k, getattr(obj,k,None), v)
                assert(False)
        return obj

    def try_to_delete(self, cls, purge=True, **kwargs):
        for obj in cls.objects.filter(**kwargs):
            obj.delete(purge=purge)

        if purge:
            for obj in cls.deleted_objects.filter(**kwargs):
                obj.delete(purge=True)

    def execute(self, yml):
        u = User.objects.get(email=self.username)

        #print self.base_yaml+yml

        xt = XOSTosca(self.base_yaml+yml, parent_dir=parentdir, log_to_console=False)
        xt.execute(u)

    def destroy(self, yml):
        u = User.objects.get(email=self.username)

        #print self.base_yaml+yml

        xt = XOSTosca(self.base_yaml+yml, parent_dir=parentdir, log_to_console=False)
        xt.destroy(u)

    def runtest(self):
        for test in self.tests:
            print "running", test
            self.cleanup()
            try:
                getattr(self,test)()
            finally:
                self.cleanup()

    def cleanup(self):
        pass
<|endoftext|>


<commit_before>"""
Feed-forward neural network with word dropout, following
Iyyer et al. 2015,
Deep Unordered Composition Rivals Syntactic Methods for Text Classification
https://www.cs.umd.edu/~miyyer/pubs/2015_acl_dan.pdf
"""
from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division

from collections import defaultdict
from pathlib import Path
import plac
import numpy.random

from thinc.neural.nn import NeuralNet
from thinc.extra.eg import Example

def read_data(data_dir):
    for subdir, label in (('pos', 1), ('neg', 0)):
        i = 0
        for filename in (data_dir / subdir).iterdir():
            text = filename.open().read()
            if len(text) >= 10:
                yield text, label
                i += 1

def partition(examples, split_size):
    examples = list(examples)
    numpy.random.shuffle(examples)
    n_docs = len(examples)
    split = int(n_docs * split_size)
    return examples[:split], examples[split:]

def minibatch(data, bs=24):
    for i in range(0, len(data), bs):
        yield data[i:i+bs]

def preprocess(text):
    tokens = []
    for line in text.split('\n'):
        line = line.strip().replace('.', '').replace(',', '')
        line = line.replace(';', '').replace('
', ' ')
        line = line.replace(':', '').replace('"', '')
        line = line.replace('(', '').replace(')', '')
        line = line.replace('!', '').replace('*', '')
        line = line.replace(' - ', ' ').replace(' -- ', '')
        line = line.replace('?', '')
        tokens.extend(line.lower().split())
    return tokens

class Extractor(object):
    def __init__(self, nr_embed, dropout=0.3):
        self.nr_embed = nr_embed
        self.dropout = dropout
        self.vocab = {}

    def __call__(self, text, dropout=True, bigrams=False):
        doc = preprocess(text)
        dropout = self.dropout if dropout is True else 0.0
        bow = defaultdict(float)
        all_words = defaultdict(float)
        prev = None
        for word in doc:
            id_ = self.vocab.setdefault(word, len(self.vocab) + 1)
            if numpy.random.random() >= dropout and word.isalpha():
                bow[id_] += 1
                if bigrams and prev is not None:
                    bi = self.vocab.setdefault(prev+'_'+word, len(self.vocab) + 1)
                    bow[bi] += 1
                    all_words[bi] += 1
                prev = word
            else:
                prev = None
            all_words[id_] += 1
        if sum(bow.values()) < 1:
            bow = all_words
        # Normalize for frequency and adjust for dropout
        total = sum(bow.values())
        total *= 1-dropout
        for word, freq in bow.items():
            bow[word] = float(freq) / total
        return bow

class DenseAveragedNetwork(NeuralNet):
    '''A feed-forward neural network, where:

    * Input is an embedding layer averaged from the words in a document
    * Widths of all layers are the same, including the input (embedding) layer
    * ReLu non-linearities are applied to the hidden layers
    * Softmax is applied to the output
    * Weights updated with Adagrad
    * Weights initialized with He initialization
    * Dropout is applied at the token level
    '''
    def __init__(self, n_classes, width, depth, get_bow, rho=1e-5, eta=0.005,
                 eps=1e-6, update_step='sgd'):
        nn_shape = tuple([width] + [width] * depth + [n_classes])
        NeuralNet.__init__(self, nn_shape, embed=((width,), (0,)),
                           rho=rho, eta=eta, update_step=update_step)
        self.get_bow = get_bow
        self.eg = Example(nr_class=self.nr_class, widths=self.widths)

    def train(self, text, label):
        self.eg.reset()
        self.eg.features = self.get_feats(text)
        self.eg.costs = [i != label for i in range(self.eg.nr_class)]
        loss = self.update(self.eg)
        return loss

    def predict(self, text):
        self.eg.reset()
        self.eg.features = self.get_feats(text, dropout=False)
        return self(self.eg)

    def get_feats(self, text, dropout=True):
        word_ids = self.get_bow(text, dropout=dropout)
        return {(0, word_id): freq for (word_id, freq) in word_ids.items()}

    def save(self):
        raise NotImplementedError

    def load(self):
        raise NotImplementedError

@plac.annotations(
    data_dir=("Data directory", "positional", None, Path),
    vectors_loc=("Path to pre-trained vectors", "positional", None, Path),
    n_iter=("Number of iterations (epochs)", "option", "i", int),
    width=("Size of hidden layers", "option", "H", int),
    depth=("Depth", "option", "d", int),
    dropout=("Drop-out rate", "option", "r", float),
    rho=("Regularization penalty", "option", "p", float),
    eta=("Learning rate", "option", "e", float),
    bias=("Initialize biases to", "option", "B", float),
    batch_size=("Batch size", "option", "b", int),
    solver=("Solver", "option", "s", str),
)
def main(data_dir, vectors_loc=None, depth=2, width=300, n_iter=5,
         batch_size=24, dropout=0.5, rho=1e-5, eta=0.005, bias=0.0, solver='sgd'):
    n_classes = 2
    print("Initializing model")
    model = DenseAveragedNetwork(n_classes, width, depth, Extractor(width, dropout),
                                 update_step=solver, rho=rho, eta=eta, eps=1e-6)
    print(model.widths)
    print(model.nr_weight)
    print("Read data")
    train_data, dev_data = partition(read_data(data_dir / 'train'), 0.8)
    print("Begin training")
    prev_best = 0
    best_weights = None
    numpy.random.seed(0)
    for i, (w, b) in enumerate(model.layers):
        print("Layer %d means:" % i, sum(w)/len(w), sum(b)/len(b))

    prev_score = 0.0
    for epoch in range(n_iter):
        numpy.random.shuffle(train_data)
        train_loss = 0.0
        for text, label in train_data:
            train_loss += model.train(text, label)
        score = sum(model.predict(x).guess == y for x, y in dev_data) / len(dev_data)
        print(epoch, train_loss, score,
              sum(model.weights) / model.nr_weight)
        if prev_score >= score:
            prev_score = sum(model.predict(x).guess == y for x, y in dev_data) / len(dev_data)
        else:
            prev_score = score
    print("Evaluating")
    eval_data = list(read_data(data_dir / 'test'))
    n_correct = sum(model.predict(x).guess == y for x, y in eval_data)
    print(n_correct / len(eval_data))
    print("After averaging")
    model.end_training()
    eval_data = list(read_data(data_dir / 'test'))
    n_correct = sum(model.predict(x).guess == y for x, y in eval_data)
    print(n_correct / len(eval_data))

if __name__ == '__main__':
    #import cProfile
    #import pstats
    #cProfile.runctx("main(Path('/Users/matt/repos/sentiment_tutorial/data/aclImdb'))", globals(), locals(), "Profile.prof")
    #s = pstats.Stats("Profile.prof")
    #s.strip_dirs().sort_stats("time").print_stats(100)

    plac.call(main)
<commit_msg>Add averaged perceptron example to text classifier<commit_after>"""
Feed-forward neural network with word dropout, following
Iyyer et al. 2015,
Deep Unordered Composition Rivals Syntactic Methods for Text Classification
https://www.cs.umd.edu/~miyyer/pubs/2015_acl_dan.pdf
"""
from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division

from itertools import imap
from collections import defaultdict
from pathlib import Path
import plac
import numpy.random

from thinc.neural.nn import NeuralNet
from thinc.linear.avgtron import AveragedPerceptron
from thinc.extra.eg import Example

def read_data(data_dir):
    for subdir, label in (('pos', 1), ('neg', 0)):
        i = 0
        for filename in (data_dir / subdir).iterdir():
            text = filename.open().read()
            if len(text) >= 10:
                yield text, label
                i += 1

def partition(examples, split_size):
    examples = list(examples)
    numpy.random.shuffle(examples)
    n_docs = len(examples)
    split = int(n_docs * split_size)
    return examples[:split], examples[split:]

def minibatch(data, bs=24):
    for i in range(0, len(data), bs):
        yield data[i:i+bs]

def preprocess(text):
    tokens = []
    for line in text.split('\n'):
        line = line.strip().replace('.', '').replace(',', '')
        line = line.replace(';', '').replace('
', ' ')
        line = line.replace(':', '').replace('"', '')
        line = line.replace('(', '').replace(')', '')
        line = line.replace('!', '').replace('*', '')
        line = line.replace(' - ', ' ').replace(' -- ', '')
        line = line.replace('?', '')
        tokens.extend(line.lower().split())
    return tokens

class Extractor(object):
    def __init__(self, dropout=0.3, bigrams=False):
        self.dropout = dropout
        self.bigrams = bigrams
        self.vocab = {}

    def __call__(self, text, dropout=True):
        doc = preprocess(text)
        dropout = self.dropout if dropout is True else 0.0
        bow = defaultdict(float)
        all_words = defaultdict(float)
        prev = None
        inc = (1./(1-dropout))
        for word in doc:
            id_ = self.vocab.setdefault(word, len(self.vocab) + 1)
            if numpy.random.random() >= dropout and word.isalpha():
                bow[id_] += inc
                if self.bigrams and prev is not None:
                    bi = self.vocab.setdefault(prev+'_'+word, len(self.vocab) + 1)
                    bow[bi] += inc
                    all_words[bi] += inc
                prev = word
            else:
                prev = None
            all_words[id_] += inc
        if sum(bow.values()) < 1:
            bow = all_words
        return bow

class DenseAveragedNetwork(NeuralNet):
    '''A feed-forward neural network, where:

    * Input is an embedding layer averaged from the words in a document
    * Widths of all layers are the same, including the input (embedding) layer
    * ReLu non-linearities are applied to the hidden layers
    * Softmax is applied to the output
    * Weights updated with Adagrad
    * Weights initialized with He initialization
    * Dropout is applied at the token level
    '''
    def __init__(self, n_classes, width, depth, get_bow, rho=1e-5, eta=0.005,
                 eps=1e-6, batch_norm=False, update_step='sgd'):
        unigram_width = int(width/2)
        bigram_width = int(width/2)
        nn_shape = tuple([unigram_width + bigram_width] + [width] * depth + [n_classes])
        NeuralNet.__init__(self, nn_shape, embed=((width,bigram_width), (0,1)),
                           rho=rho, eta=eta, update_step=update_step,
                           batch_norm=batch_norm)
        self.get_bow = get_bow

    def Eg(self, text, label=None):
        bow = self.get_feats(text, dropout=bool(label is not None))
        eg = Example(nr_class=self.nr_class, nr_feat=len(bow))
        eg.costs = [i != label for i in range(self.nr_class)]
        eg.features = bow
        return eg

    def get_feats(self, text, dropout=True):
        bow = self.get_bow(text, dropout=dropout)
        output = {}
        for word_id, freq in bow.items():
            output[(0, word_id)] = freq
        return output

    def save(self):
        raise NotImplementedError

    def load(self):
        raise NotImplementedError

class BOWTron(AveragedPerceptron):
    def __init__(self, n_classes, get_bow, *args, **kwargs):
        AveragedPerceptron.__init__(self, tuple())
        self.nr_class = n_classes
        self.get_bow = get_bow

    @property
    def nr_weight(self):
        return 1

    @property
    def weights(self):
        return [self.mem.size]

    def Eg(self, text, label=None):
        bow = self.get_feats(text, dropout=bool(label is not None))
        eg = Example(nr_class=self.nr_class, nr_feat=len(bow))
        eg.costs = [i != label for i in range(self.nr_class)]
        eg.features = bow
        return eg

    def get_feats(self, text, dropout=True):
        bow = self.get_bow(text, dropout=dropout)
        output = {}
        for word_id, freq in bow.items():
            output[(0, word_id)] = freq
        return output

    def save(self):
        raise NotImplementedError

    def load(self):
        raise NotImplementedError

@plac.annotations(
    data_dir=("Data directory", "positional", None, Path),
    vectors_loc=("Path to pre-trained vectors", "positional", None, Path),
    n_iter=("Number of iterations (epochs)", "option", "i", int),
    width=("Size of hidden layers", "option", "H", int),
    depth=("Depth", "option", "d", int),
    dropout=("Drop-out rate", "option", "r", float),
    rho=("Regularization penalty", "option", "p", float),
    eta=("Learning rate", "option", "e", float),
    batch_norm=("Use batch normalization", "flag", "B"),
    batch_size=("Batch size", "option", "b", int),
    solver=("Solver", "option", "s", str),
)
def main(data_dir, vectors_loc=None, depth=2, width=300, n_iter=5,
         batch_size=24, dropout=0.5, rho=1e-5, eta=0.005, batch_norm=False, solver='sgd'):
    n_classes = 2
    print("Initializing model")
    #model = DenseAveragedNetwork(n_classes, width, depth,
    #                             Extractor(width, dropout, bigrams=True),
    #                             update_step=solver, rho=rho, eta=eta, eps=1e-6,
    #                             batch_norm=batch_norm)
    model = BOWTron(2, Extractor(dropout, bigrams=True))
    print("Read data")
    train_data, dev_data = partition(read_data(data_dir / 'train'), 0.8)
    print("Begin training")
    prev_best = 0
    best_weights = None
    numpy.random.seed(0)

    prev_score = 0.0
    try:
        for epoch in range(n_iter):
            numpy.random.shuffle(train_data)
            train_loss = 0.0
            for text, label in train_data:
                eg = model.Eg(text, label)
                train_loss += model.update(eg)
            nr_correct = sum(model(model.Eg(x)).guess == y for x, y in dev_data)
            print(epoch, train_loss, nr_correct / len(dev_data),
                  sum(model.weights) / model.nr_weight)
    except KeyboardInterrupt:
        print("Stopping")
    print("Evaluating")
    eval_data = list(read_data(data_dir / 'test'))
    n_correct = sum(model(model.Eg(x)).guess == y for x, y in eval_data)
    print(n_correct / len(eval_data))
    print("After averaging")
    model.end_training()
    eval_data = list(read_data(data_dir / 'test'))
    n_correct = sum(model(model.Eg(x)).guess == y for x, y in eval_data)
    print(n_correct / len(eval_data))

if __name__ == '__main__':
    plac.call(main)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import shutil
from pwd import getpwnam
from grp import getgrnam

from setuptools import setup, find_packages
from pkg_resources import Requirement, resource_filename

CPSVERSION = '2.0.0'

CONFDIR = '/etc/cpsdirector'
LOGDIR = '/var/log/cpsdirector'

if not os.geteuid() == 0:
    CONFDIR = 'cpsdirectorconf'

long_description = """
ConPaaS: an integrated runtime environment for elastic Cloud applications
=========================================================================
"""
setup(name='cpsdirector',
      version=CPSVERSION,
      description='ConPaaS director',
      author='ConPaaS team',
      author_email='info@conpaas.eu',
      url='http://www.conpaas.eu/',
      download_url='http://www.conpaas.eu/download/',
      license='BSD',
      # packages=[ 'cpsdirector', ],
      packages=find_packages(),
      include_package_data=True,
      zip_safe=False,
      package_data={ 'cpsdirector': [ 'ConPaaS.tar.gz',] },
      data_files=[ ( CONFDIR, [ 'director.cfg.example', 'director.cfg.multicloud-example', 'ConPaaS.tar.gz' ] ), ],
      scripts=[ 'cpsadduser.py', 'director.wsgi', 'cpsconf.py', 'cpscheck.py', 'add-user-columns-to-db.sh' ],
      install_requires=[ 'cpslib', 'flask-sqlalchemy', 'apache-libcloud', 'netaddr', 'flask-openid' ],
      dependency_links=[ 'http://www.conpaas.eu/dl/cpslib-%s.tar.gz' % CPSVERSION, ],)

if __name__ == "__main__" and sys.argv[1] == "install":
    # overwrite /etc/cpsdirector/{config,scripts}
    for what in 'config', 'scripts':
        targetdir = os.path.join(CONFDIR, what)
        if os.path.isdir(targetdir):
            shutil.rmtree(targetdir)

        shutil.copytree(os.path.join('conpaas', what), targetdir)

    if not os.path.exists(os.path.join(CONFDIR, "director.cfg")):
        # copy director.cfg.example under CONFDIR/director.cfg
        conffile = resource_filename(Requirement.parse("cpsdirector"),
            "director.cfg.example")
        shutil.copyfile(conffile, os.path.join(CONFDIR, "director.cfg"))

    # create 'certs' dir
    if not os.path.exists(os.path.join(CONFDIR, "certs")):
        os.mkdir(os.path.join(CONFDIR, "certs"))

    # create user data dir
    if not os.path.exists(os.path.join(CONFDIR, "data")):
        os.mkdir(os.path.join(CONFDIR, "data"))

    # create log dir and debugging log file
    if not os.path.exists(LOGDIR):
        os.mkdir(LOGDIR)
    open(os.path.join(LOGDIR, "debugging.log"), 'a').close()
    os.chmod(os.path.join(LOGDIR, "debugging.log"), 0777)

    # set www-data as the owner of CONFDIR
    try:
        os.chown(CONFDIR, getpwnam('www-data').pw_uid,
            getgrnam('www-data').gr_gid)
    except OSError:
        print "W: 'chown www-data:www-data %s' failed" % CONFDIR

    # set www-data as the owner of CONFDIR/data
    try:
        os.chown('%s/data' % CONFDIR, getpwnam('www-data').pw_uid,
            getgrnam('www-data').gr_gid)
    except OSError:
        print "W: 'chown www-data:www-data %s' failed" % CONFDIR + '/data'

<commit_msg>director: fix problems when installing on newer systems<commit_after>#!/usr/bin/env python

import os
import sys
import shutil
from pwd import getpwnam
from grp import getgrnam

from setuptools import setup, find_packages
from pkg_resources import Requirement, resource_filename

CPSVERSION = '2.0.0'

CONFDIR = '/etc/cpsdirector'
LOGDIR = '/var/log/cpsdirector'
PYTHON_EGGS = '/var/www/.python-eggs'

if not os.geteuid() == 0:
    CONFDIR = 'cpsdirectorconf'

long_description = """
ConPaaS: an integrated runtime environment for elastic Cloud applications
=========================================================================
"""
setup(name='cpsdirector',
      version=CPSVERSION,
      description='ConPaaS director',
      author='ConPaaS team',
      author_email='info@conpaas.eu',
      url='http://www.conpaas.eu/',
      download_url='http://www.conpaas.eu/download/',
      license='BSD',
      # packages=[ 'cpsdirector', ],
      packages=find_packages(),
      include_package_data=True,
      zip_safe=False,
      package_data={ 'cpsdirector': [ 'ConPaaS.tar.gz',] },
      data_files=[ ( CONFDIR, [ 'director.cfg.example', 'director.cfg.multicloud-example', 'ConPaaS.tar.gz' ] ), ],
      scripts=[ 'cpsadduser.py', 'director.wsgi', 'cpsconf.py', 'cpscheck.py', 'add-user-columns-to-db.sh' ],
      install_requires=[ 'cpslib', 'flask-sqlalchemy', 'apache-libcloud', 'netaddr', 'flask-openid', 'Cheetah' ],
      dependency_links=[ 'http://www.conpaas.eu/dl/cpslib-%s.tar.gz' % CPSVERSION, ],)

if __name__ == "__main__" and sys.argv[1] == "install":
    # overwrite /etc/cpsdirector/{config,scripts}
    for what in 'config', 'scripts':
        targetdir = os.path.join(CONFDIR, what)
        if os.path.isdir(targetdir):
            shutil.rmtree(targetdir)

        shutil.copytree(os.path.join('conpaas', what), targetdir)

    if not os.path.exists(os.path.join(CONFDIR, "director.cfg")):
        # copy director.cfg.example under CONFDIR/director.cfg
        conffile = resource_filename(Requirement.parse("cpsdirector"),
            "director.cfg.example")
        shutil.copyfile(conffile, os.path.join(CONFDIR, "director.cfg"))

    # create 'certs' dir
    if not os.path.exists(os.path.join(CONFDIR, "certs")):
        os.mkdir(os.path.join(CONFDIR, "certs"))

    # create user data dir
    if not os.path.exists(os.path.join(CONFDIR, "data")):
        os.mkdir(os.path.join(CONFDIR, "data"))

    # create log dir and debugging log file
    if not os.path.exists(LOGDIR):
        os.mkdir(LOGDIR)
    open(os.path.join(LOGDIR, "debugging.log"), 'a').close()
    os.chmod(os.path.join(LOGDIR, "debugging.log"), 0777)

    # set www-data as the owner of CONFDIR
    try:
        os.chown(CONFDIR, getpwnam('www-data').pw_uid,
            getgrnam('www-data').gr_gid)
    except OSError:
        print "W: 'chown www-data:www-data %s' failed" % CONFDIR

    # set www-data as the owner of CONFDIR/data
    try:
        os.chown('%s/data' % CONFDIR, getpwnam('www-data').pw_uid,
            getgrnam('www-data').gr_gid)
    except OSError:
        print "W: 'chown www-data:www-data %s' failed" % CONFDIR + '/data'

    # create and set permissions for the .python-eggs dir
    if not os.path.exists(PYTHON_EGGS):
        os.mkdir(PYTHON_EGGS)
    try:
        os.chown(PYTHON_EGGS, getpwnam('www-data').pw_uid,
            getgrnam('www-data').gr_gid)
    except OSError:
        print "W: 'chown www-data:www-data %s' failed" % PYTHON_EGGS
<|endoftext|>


<commit_before>#    Copyright 2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=no-name-in-module,import-error,no-member

import os
import subprocess
from distutils.version import LooseVersion

# pylint: disable=wrong-import-position,ungrouped-imports
import_error_str = ''
try:
    import IPython
except ImportError as import_error:
    IPython = None
    # Importing IPython can fail for a variety of reasons, report the actual
    # failure unless it's just that the package is not present
    if import_error.message.startswith("No module named"):  # pylint: disable=E1101
        import_error_str = 'ipynb_exporter requires ipython package to be installed'
    else:
        import_error_str = import_error.message  # pylint: disable=redefined-variable-type

# The current code generates notebooks version 3
NBFORMAT_VERSION = 3

if IPython:
    if LooseVersion('6.0.0') > LooseVersion(IPython.__version__) >= LooseVersion('4.0.0'):
        import nbformat
        from jupyter_client.manager import KernelManager

        def read_notebook(notebook_in):  # pylint: disable=function-redefined
            return nbformat.reads(notebook_in, NBFORMAT_VERSION)  # pylint: disable=E1101

        def write_notebook(notebook, fout):  # pylint: disable=function-redefined
            nbformat.write(notebook, fout)  # pylint: disable=E1101

        NotebookNode = nbformat.NotebookNode  # pylint: disable=E1101

        IPYTHON_NBCONVERT_HTML = ['jupyter', 'nbconvert', '--to html']
        IPYTHON_NBCONVERT_PDF = ['jupyter', 'nbconvert', '--to pdf']

    elif LooseVersion('4.0.0') > LooseVersion(IPython.__version__) >= LooseVersion('3.0.0'):
        from IPython.kernel import KernelManager
        import IPython.nbformat

        def read_notebook(notebook_in):  # pylint: disable=function-redefined
            return IPython.nbformat.reads(notebook_in, NBFORMAT_VERSION)  # pylint: disable=E1101

        def write_notebook(notebook, fout):  # pylint: disable=function-redefined
            IPython.nbformat.write(notebook, fout)  # pylint: disable=E1101

        NotebookNode = IPython.nbformat.NotebookNode  # pylint: disable=E1101

        IPYTHON_NBCONVERT_HTML = ['ipython', 'nbconvert', '--to=html']
        IPYTHON_NBCONVERT_PDF = ['ipython', 'nbconvert', '--to=pdf']
    elif LooseVersion('3.0.0') > LooseVersion(IPython.__version__) >= LooseVersion('2.0.0'):
        from IPython.kernel import KernelManager
        import IPython.nbformat.v3

        def read_notebook(notebook_in):  # pylint: disable=function-redefined
            return IPython.nbformat.v3.reads_json(notebook_in)  # pylint: disable=E1101

        def write_notebook(notebook, fout):  # pylint: disable=function-redefined
            IPython.nbformat.v3.nbjson.JSONWriter().write(notebook, fout)  # pylint: disable=E1101

        NotebookNode = IPython.nbformat.v3.NotebookNode  # pylint: disable=E1101

        IPYTHON_NBCONVERT_HTML = ['ipython', 'nbconvert', '--to=html']
        IPYTHON_NBCONVERT_PDF = ['ipython', 'nbconvert', '--to=latex',
                                 '--post=PDF']
    else:
        # Unsupported IPython version
        import_error_str = 'Unsupported IPython version {}'.format(IPython.__version__)

def parse_valid_output(msg):
    """Parse a valid result from an execution of a cell in an ipython kernel"""
    msg_type = msg["msg_type"]
    if msg_type == 'error':
        msg_type = 'pyerr'
    elif msg_type == 'execute_result':
        msg_type = 'pyout'

    content = msg["content"]
    out = NotebookNode(output_type=msg_type)

    if msg_type == "stream":
        out.stream = content["name"]
        try:
            out.text = content['data']
        except KeyError:
            out.text = content['text']
    elif msg_type in ("display_data", "pyout"):
        for mime, data in content["data"].iteritems():
            if mime == "text/plain":
                attr = "text"
            else:
                attr = mime.split("/")[-1]
            setattr(out, attr, data)
    elif msg_type == "pyerr":
        out.ename = content["ename"]
        out.evalue = content["evalue"]
        out.traceback = content["traceback"]
    else:
        raise ValueError("Unknown msg_type {}".format(msg_type))

    return out

def run_cell(kernel_client, cell):
    """Run a cell of a notebook in an ipython kernel and return its output"""
    kernel_client.execute(cell.input)

    input_acknowledged = False
    outs = []
    while True:
        msg = kernel_client.get_iopub_msg()

        if msg["msg_type"] == "status":
            if msg["content"]["execution_state"] == "idle" and input_acknowledged:
                break
        elif msg["msg_type"] in ('pyin', 'execute_input'):
            input_acknowledged = True
        else:
            out = parse_valid_output(msg)
            outs.append(out)

    return outs

def run_notebook(notebook):
    """Run the notebook"""
    kernel_manager = KernelManager()
    kernel_manager.start_kernel(stderr=open(os.devnull, 'w'))
    kernel_client = kernel_manager.client()
    kernel_client.start_channels()

    for sheet in notebook.worksheets:
        for (prompt_number, cell) in enumerate(sheet.cells, 1):
            if cell.cell_type != "code":
                continue

            cell.outputs = run_cell(kernel_client, cell)

            cell.prompt_number = prompt_number
            if cell.outputs and cell.outputs[0]['output_type'] == 'pyout':
                cell.outputs[0]["prompt_number"] = prompt_number

    kernel_manager.shutdown_kernel()

def export_notebook(nbbasename, output_directory, output_format):
    """Generate a PDF or HTML from the ipython notebook

    output_format has to be either 'pdf' or 'html'.  These are the
    only formats currently supported.

    ipython nbconvert claims that the CLI is not stable, so keep this
    function here to be able to cope with inconsistencies

    """

    if output_format == "html":
        ipython_command = IPYTHON_NBCONVERT_HTML
    elif output_format == "pdf":
        ipython_command = IPYTHON_NBCONVERT_PDF
    else:
        raise ValueError("Unknown output format: {}".format(output_format))

    prev_dir = os.getcwd()
    os.chdir(output_directory)

    with open(os.devnull, 'w') as devnull:
        subprocess.check_call(ipython_command + [nbbasename], stderr=devnull)

    os.chdir(prev_dir)
<commit_msg>utils/ipython: handle nbconvert import<commit_after>#    Copyright 2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=no-name-in-module,import-error,no-member

import os
import subprocess
from distutils.version import LooseVersion

# pylint: disable=wrong-import-position,ungrouped-imports
import_error_str = ''
try:
    import IPython
except ImportError as import_error:
    IPython = None
    # Importing IPython can fail for a variety of reasons, report the actual
    # failure unless it's just that the package is not present
    if import_error.message.startswith("No module named"):  # pylint: disable=E1101
        import_error_str = 'ipynb_exporter requires ipython package to be installed'
    else:
        import_error_str = import_error.message  # pylint: disable=redefined-variable-type

# The current code generates notebooks version 3
NBFORMAT_VERSION = 3

if IPython:
    if LooseVersion('6.0.0') > LooseVersion(IPython.__version__) >= LooseVersion('4.0.0'):
        try:
            import nbformat
            from jupyter_client.manager import KernelManager

            def read_notebook(notebook_in):  # pylint: disable=function-redefined
                return nbformat.reads(notebook_in, NBFORMAT_VERSION)  # pylint: disable=E1101

            def write_notebook(notebook, fout):  # pylint: disable=function-redefined
                nbformat.write(notebook, fout)  # pylint: disable=E1101

            NotebookNode = nbformat.NotebookNode  # pylint: disable=E1101

            IPYTHON_NBCONVERT_HTML = ['jupyter', 'nbconvert', '--to html']
            IPYTHON_NBCONVERT_PDF = ['jupyter', 'nbconvert', '--to pdf']
        except ImportError:
            import_error_str = 'Please install "jupyter" as well as "ipython" packages.'

    elif LooseVersion('4.0.0') > LooseVersion(IPython.__version__) >= LooseVersion('3.0.0'):
        from IPython.kernel import KernelManager
        import IPython.nbformat

        def read_notebook(notebook_in):  # pylint: disable=function-redefined
            return IPython.nbformat.reads(notebook_in, NBFORMAT_VERSION)  # pylint: disable=E1101

        def write_notebook(notebook, fout):  # pylint: disable=function-redefined
            IPython.nbformat.write(notebook, fout)  # pylint: disable=E1101

        NotebookNode = IPython.nbformat.NotebookNode  # pylint: disable=E1101

        IPYTHON_NBCONVERT_HTML = ['ipython', 'nbconvert', '--to=html']
        IPYTHON_NBCONVERT_PDF = ['ipython', 'nbconvert', '--to=pdf']
    elif LooseVersion('3.0.0') > LooseVersion(IPython.__version__) >= LooseVersion('2.0.0'):
        from IPython.kernel import KernelManager
        import IPython.nbformat.v3

        def read_notebook(notebook_in):  # pylint: disable=function-redefined
            return IPython.nbformat.v3.reads_json(notebook_in)  # pylint: disable=E1101

        def write_notebook(notebook, fout):  # pylint: disable=function-redefined
            IPython.nbformat.v3.nbjson.JSONWriter().write(notebook, fout)  # pylint: disable=E1101

        NotebookNode = IPython.nbformat.v3.NotebookNode  # pylint: disable=E1101

        IPYTHON_NBCONVERT_HTML = ['ipython', 'nbconvert', '--to=html']
        IPYTHON_NBCONVERT_PDF = ['ipython', 'nbconvert', '--to=latex',
                                 '--post=PDF']
    else:
        # Unsupported IPython version
        import_error_str = 'Unsupported IPython version {}'.format(IPython.__version__)

def parse_valid_output(msg):
    """Parse a valid result from an execution of a cell in an ipython kernel"""
    msg_type = msg["msg_type"]
    if msg_type == 'error':
        msg_type = 'pyerr'
    elif msg_type == 'execute_result':
        msg_type = 'pyout'

    content = msg["content"]
    out = NotebookNode(output_type=msg_type)

    if msg_type == "stream":
        out.stream = content["name"]
        try:
            out.text = content['data']
        except KeyError:
            out.text = content['text']
    elif msg_type in ("display_data", "pyout"):
        for mime, data in content["data"].iteritems():
            if mime == "text/plain":
                attr = "text"
            else:
                attr = mime.split("/")[-1]
            setattr(out, attr, data)
    elif msg_type == "pyerr":
        out.ename = content["ename"]
        out.evalue = content["evalue"]
        out.traceback = content["traceback"]
    else:
        raise ValueError("Unknown msg_type {}".format(msg_type))

    return out

def run_cell(kernel_client, cell):
    """Run a cell of a notebook in an ipython kernel and return its output"""
    kernel_client.execute(cell.input)

    input_acknowledged = False
    outs = []
    while True:
        msg = kernel_client.get_iopub_msg()

        if msg["msg_type"] == "status":
            if msg["content"]["execution_state"] == "idle" and input_acknowledged:
                break
        elif msg["msg_type"] in ('pyin', 'execute_input'):
            input_acknowledged = True
        else:
            out = parse_valid_output(msg)
            outs.append(out)

    return outs

def run_notebook(notebook):
    """Run the notebook"""
    kernel_manager = KernelManager()
    kernel_manager.start_kernel(stderr=open(os.devnull, 'w'))
    kernel_client = kernel_manager.client()
    kernel_client.start_channels()

    for sheet in notebook.worksheets:
        for (prompt_number, cell) in enumerate(sheet.cells, 1):
            if cell.cell_type != "code":
                continue

            cell.outputs = run_cell(kernel_client, cell)

            cell.prompt_number = prompt_number
            if cell.outputs and cell.outputs[0]['output_type'] == 'pyout':
                cell.outputs[0]["prompt_number"] = prompt_number

    kernel_manager.shutdown_kernel()

def export_notebook(nbbasename, output_directory, output_format):
    """Generate a PDF or HTML from the ipython notebook

    output_format has to be either 'pdf' or 'html'.  These are the
    only formats currently supported.

    ipython nbconvert claims that the CLI is not stable, so keep this
    function here to be able to cope with inconsistencies

    """

    if output_format == "html":
        ipython_command = IPYTHON_NBCONVERT_HTML
    elif output_format == "pdf":
        ipython_command = IPYTHON_NBCONVERT_PDF
    else:
        raise ValueError("Unknown output format: {}".format(output_format))

    prev_dir = os.getcwd()
    os.chdir(output_directory)

    with open(os.devnull, 'w') as devnull:
        subprocess.check_call(ipython_command + [nbbasename], stderr=devnull)

    os.chdir(prev_dir)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Protein Engineering Analysis Tool Structure Analysis (PEATSA)
# Copyright (C) 2010 Michael Johnston & Jens Erik Nielsen
#
# Author: Michael Johnston
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Contact information:
# Email: Jens.Nielsen_at_gmail.com
# Normal mail:
# Jens Nielsen
# SBBS, Conway Institute
# University College Dublin
# Dublin 4, Ireland
#
'''Cgi Script for dynamically creating a mutant pdb file used in a PEAT-SA job'''
import cgi, cgitb, tempfile, os, sys
import PEATSA.WebApp as WebApp
import PEATSA.Core as Core

#URL'S have query string jobId=$jobID&mutationCode=$mutationCode
if __name__ == "__main__":

	#Turn on exception logging
	cgitb.enable()
	formData = cgi.FieldStorage()

	#Redirect stdout
	oldout = sys.stdout
	newout = tempfile.NamedTemporaryFile()
	sys.stdout = newout

	#Get Data
	try:
		#Set environment
		configuration = WebApp.UtilityFunctions.DefaultWebAppConfiguration()
		os.environ['HOME'] = configuration.get('PATHS', 'whatif')

		jobId = formData.getvalue('jobId')
		mutationCode = formData.getvalue('mutationCode')

		#Create Mutation set
		mutationSet = Core.Data.MutationSet(code=mutationCode)

		#Get the pdb
		connection = WebApp.UtilityFunctions.ConnectionFromDefaultConfiguration()
		job = WebApp.Data.Job(jobID=jobId, connection=connection)
		structure = job.structure()

		#Write it out
		temp = tempfile.NamedTemporaryFile()
		name = temp.name
		temp.write(structure)
		temp.flush()

		#Create Mutant
		collection = Core.Data.MutantCollection(pdbFile=name,
					location='/tmp/',
					name=name,
					maxOverlap=0.5,
					temporary=False)
		collection.createMutants(mutationList=[mutationSet])

		if len(collection.mutantFiles()) == 0:
			raise Exception, "Modelling Failure - %s" % temp.name

		#Get PDB
		filename = collection.fileForMutant(mutationSet)
		stream = open(filename)
		string = stream.read()
		stream.close()

		#Clean up
		temp.close()
		sys.stdout = oldout
		newout.close()

		#Download
		print "Content-length: %d", len(string)
		print "Content-type: application/pdb"
		print "Content-Disposition: attachment; filename=%s\n\n" % mutationSet.filename(reduced=True)
		print string
	except Exception, data:
		temp.close()
		sys.stdout = oldout
		newout.close()
		url = WebApp.JobSubmission.ConstructErrorURL(domain="MutantCreationErrorDomain",
			description="Encounter a %s exception while attempting to create mutant" % Exception,
			detailedDescription="Exception data - %s. Mutant %s" % (data, mutationCode),
			recoverySuggestion="This may be bug - please report to site administrators")
		#Redirect to the webpage indicated by resultURL
		print "Location: %s\n\n" % url

<commit_msg>Replace fileForMutant with fileForMutationSet<commit_after>#!/usr/bin/env python
#
# Protein Engineering Analysis Tool Structure Analysis (PEATSA)
# Copyright (C) 2010 Michael Johnston & Jens Erik Nielsen
#
# Author: Michael Johnston
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Contact information:
# Email: Jens.Nielsen_at_gmail.com
# Normal mail:
# Jens Nielsen
# SBBS, Conway Institute
# University College Dublin
# Dublin 4, Ireland
#
'''Cgi Script for dynamically creating a mutant pdb file used in a PEAT-SA job'''
import cgi, cgitb, tempfile, os, sys
import PEATSA.WebApp as WebApp
import PEATSA.Core as Core

#URL'S have query string jobId=$jobID&mutationCode=$mutationCode
if __name__ == "__main__":

	#Turn on exception logging
	cgitb.enable()
	formData = cgi.FieldStorage()

	#Redirect stdout
	oldout = sys.stdout
	newout = tempfile.NamedTemporaryFile()
	sys.stdout = newout

	#Get Data
	try:
		#Set environment
		configuration = WebApp.UtilityFunctions.DefaultWebAppConfiguration()
		os.environ['HOME'] = configuration.get('PATHS', 'whatif')

		jobId = formData.getvalue('jobId')
		mutationCode = formData.getvalue('mutationCode')

		#Create Mutation set
		mutationSet = Core.Data.MutationSet(code=mutationCode)

		#Get the pdb
		connection = WebApp.UtilityFunctions.ConnectionFromDefaultConfiguration()
		job = WebApp.Data.Job(jobID=jobId, connection=connection)
		structure = job.structure()

		#Write it out
		temp = tempfile.NamedTemporaryFile()
		name = temp.name
		temp.write(structure)
		temp.flush()

		#Create Mutant
		collection = Core.Data.MutantCollection(pdbFile=name,
					location='/tmp/',
					name=name,
					maxOverlap=0.5,
					temporary=False)
		collection.createMutants(mutationList=[mutationSet])

		if len(collection.mutantFiles()) == 0:
			raise Exception, "Modelling Failure - %s" % temp.name

		#Get PDB
		filename = collection.fileForMutationSet(mutationSet)
		stream = open(filename)
		string = stream.read()
		stream.close()

		#Clean up
		temp.close()
		sys.stdout = oldout
		newout.close()

		#Download
		print "Content-length: %d", len(string)
		print "Content-type: application/pdb"
		print "Content-Disposition: attachment; filename=%s\n\n" % mutationSet.filename(reduced=True)
		print string
	except Exception, data:
		temp.close()
		sys.stdout = oldout
		newout.close()
		url = WebApp.JobSubmission.ConstructErrorURL(domain="MutantCreationErrorDomain",
			description="Encounter a %s exception while attempting to create mutant" % Exception,
			detailedDescription="Exception data - %s. Mutant %s" % (data, mutationCode),
			recoverySuggestion="This may be bug - please report to site administrators")
		#Redirect to the webpage indicated by resultURL
		print "Location: %s\n\n" % url

<|endoftext|>


<commit_before>from __future__ import absolute_import

import sys
import functools

from . import idiokit, values, threadpool, callqueue

# Monkey patch idiokit.Stream to support .has_result()

def has_result(self):
    return self.result().is_set()
idiokit.Stream.has_result = has_result

# Helpers

NULL = idiokit.Event()
NULL.succeed(None)

class _ValueStream(idiokit.Stream):
    _head = values.Value(None)

    def __init__(self, value):
        idiokit.Stream.__init__(self)
        self._value = value

    def pipe_left(self, *args, **keys):
        pass
    pipe_right = pipe_left

    def head(self):
        return self._head

    def result(self):
        return self._value

# threado.Finished

Finished = StopIteration

# threado.Channel

def _channel():
    while True:
        item = yield idiokit.next()
        yield idiokit.send(item)

class Channel(idiokit.Generator):
    def __init__(self):
        idiokit.Generator.__init__(self, _channel())

    def finish(self, *args):
        self.throw(StopIteration, StopIteration(*args), None)

# threado.stream and threado.GeneratorStream

class _Sub(object):
    def __init__(self, wrapped):
        self.wrapped = wrapped

class Inner(object):
    def __init__(self, outer):
        self._outer = outer

    def send(self, *args):
        outer = self._outer
        if outer is not None:
            outer._output.stack(idiokit.send(*args))

    def sub(self, other):
        return _Sub(other)

    def flush(self):
        return NULL

    def thread(self, func, *args, **keys):
        return _ValueStream(threadpool.run(func, *args, **keys))

    def finish(self, *args):
        idiokit.stop(*args)

    def _close(self):
        self._outer = None

@idiokit.stream
def _next_or_result(stream):
    result = yield stream.next()
    idiokit.stop(result)

class GeneratorStream(idiokit.Generator):
    def __init__(self):
        self._started = idiokit.Value()
        self.inner = Inner(self)

        idiokit.Generator.__init__(self, self._wrapped())

    def _transform(self, next):
        if isinstance(next, _Any):
            return next.resolve(self)

        if isinstance(next, _Sub):
            return next.wrapped

        if next is None:
            return NULL

        if next is self.inner:
            next = idiokit.next()
            next.pipe_left(self._messages.head(), self._signals.head())
            next.pipe_right(self._broken.head())
            return idiokit.Event() | next

        if isinstance(next, idiokit.Stream):
            return _next_or_result(next)

        return _Any(*next).resolve(self)

    def _wrapped(self):
        yield _ValueStream(self._started)

        args = None
        throw = False
        gen = self.run()

        try:
            while True:
                if throw:
                    next = gen.throw(*args)
                else:
                    next = gen.send(args)

                next = self._transform(next)
                try:
                    args = yield next
                    throw = False
                except:
                    args = sys.exc_info()
                    throw = True
        finally:
            self.inner._close()

    def start(self):
        self._started.set((False, ()))

    def run(self):
        return ()

class FuncStream(GeneratorStream):
    def __init__(self, func, *args, **keys):
        GeneratorStream.__init__(self)

        self.func = func
        self.args = args
        self.keys = keys

    def run(self):
        return self.func(self.inner, *self.args, **self.keys)

def stream(func):
    @functools.wraps(func)
    def _stream(*args, **keys):
        result = FuncStream(func, *args, **keys)
        result.start()
        return result
    return _stream

# threado.any and "yield left, right" support

def any(left, right):
    return _Any(left, right)

class _Either(values.Value):
    def __init__(self, left, right):
        values.Value.__init__(self)

        self._left = left
        self._right = right

        callqueue.add(self._init, left, right)

    def _init(self, left, right):
        left.listen(self._listen_left)
        right.listen(self._listen_right)

    def _listen_left(self, value):
        if not self.unsafe_is_set():
            self.unsafe_set((False, (self._left,)))
            self._right.unlisten(self._listen_right)

    def _listen_right(self, value):
        if not self.unsafe_is_set():
            self.unsafe_set((False, (self._right,)))
            self._left.unlisten(self._listen_left)

class _Any(object):
    def __init__(self, left, right):
        self.left = left
        self.right = right

    @idiokit.stream
    def resolve(self, context):
        left = self.left
        right = self.right

        if left is right:
            idiokit.stop((yield left))

        if right is context.inner:
            left, right = right, left

        piped = None

        if left is context.inner:
            piped = idiokit.Piped()
            piped.add(context._messages.head())
            piped.add(context._signals.head())
            piped.add(context._broken.head())
            left_head = piped.head()
        else:
            left_head = left.head()
        right_head = right.head()

        try:
            while True:
                head = yield _ValueStream(_Either(left_head, right_head))
                promise = head.unsafe_get()
                if promise is None:
                    if head is left_head:
                        result = yield _ValueStream(left.result())
                    else:
                        result = yield _ValueStream(right.result())
                    break

                consumed, value, next = promise
                consumed.unsafe_set()

                wrapped = values.Value()
                value.listen(lambda x: wrapped.unsafe_set((False, (x,))))
                result = yield _ValueStream(wrapped)
                if result is not None:
                    result = yield _ValueStream(value)
                    break

                if head is left_head:
                    left_head = next
                else:
                    right_head = next
        finally:
            if piped is not None:
                piped.close()

        if head is left_head:
            idiokit.stop(left, result)
        idiokit.stop(right, result)

# Rest of threado.*

dev_null = idiokit.consume
pipe = idiokit.pipe
run = idiokit.main_loop
<commit_msg>idiokit.threado: Deprecated.<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Plot autoencoder and supervised parallel performance in one plot.
"""
#TODO display train loss of parallel where possible (when schedule is at 1 AE E/1 par E)
import argparse
import matplotlib.pyplot as plt
import numpy as np

from scripts.plotting.plot_statistics import make_data_from_files, make_plot_same_y_parallel, get_last_prl_epochs

def parse_input():
    parser = argparse.ArgumentParser(description='Make overview plots of model training.')
    parser.add_argument('autoencoder_model', type=str, help='name of _test.txt files to plot.')
    parser.add_argument('parallel_model', type=str, help='name of _test.txt files to plot.')
    args = parser.parse_args()
    params = vars(args)
    return params

params = parse_input()
test_files = [ params["autoencoder_model"], params["parallel_model"] ]

xlabel="Epoch"
title="Parallel Training"
figsize = (13,8)
#Override default labels (names of the models); must be one for every test file, otherwise default
labels_override=[]
#legend location for the labels and the test/train box
legend_locations=(1, "upper left")
#Override xtick locations; None for automatic
xticks=None
# override line colors; must be one color for every test file, otherwise automatic
colors=["blue", "orange"] # = automatic
#Name of file in the results/dumped_statistics folder to save the numpy array
#with the plot data to; None will skip saving
dump_to_file=None

#Returns ( [[Test_epoch, Test_ydata, Train_epoch, Train_ydata], ...], ylabel_list, default_label_array)
#for every test file
data_from_files, ylabel_list, default_label_array = make_data_from_files(test_files, dump_to_file)

data_autoencoder = data_from_files[0]
data_parallel = data_from_files[1]

#Which epochs from the parallel encoder history to take:
how_many_epochs_each_to_train = np.array([10,]*1+[2,]*5+[1,]*200)

data_parallel_test, data_parallel_train = get_last_prl_epochs(data_autoencoder, data_parallel, how_many_epochs_each_to_train)

fig = make_plot_same_y_parallel(data_autoencoder, data_parallel_train, data_parallel_test, default_label_array, xlabel, ylabel_list,
                 title, legend_locations, labels_override, colors, xticks, figsize)
plt.show(fig)
<commit_msg>Added optional parallel schedule argument<commit_after># -*- coding: utf-8 -*-
"""
Plot autoencoder and supervised parallel performance in one plot.
"""

import argparse

def parse_input():
    parser = argparse.ArgumentParser(description='Make overview plots of model training.')
    parser.add_argument('autoencoder_model', type=str, help='name of _test.txt files to plot.')
    parser.add_argument('parallel_model', type=str, help='name of _test.txt files to plot.')

    parser.add_argument('-c', '--channel', action="store_true", help='Use the channel parallel schedule (1 enc epoch per AE epoch)')
    args = parser.parse_args()
    params = vars(args)
    return params
params = parse_input()

import matplotlib.pyplot as plt
import numpy as np

from scripts.plotting.plot_statistics import make_data_from_files, make_plot_same_y_parallel, get_last_prl_epochs

test_files = [ params["autoencoder_model"], params["parallel_model"] ]

xlabel="Epoch"
title="Parallel Training"
figsize = (13,8)
#Override default labels (names of the models); must be one for every test file, otherwise default
labels_override=[]
#legend location for the labels and the test/train box
legend_locations=(1, "upper left")
#Override xtick locations; None for automatic
xticks=None
# override line colors; must be one color for every test file, otherwise automatic
colors=["blue", "orange"] # = automatic
#Name of file in the results/dumped_statistics folder to save the numpy array
#with the plot data to; None will skip saving
dump_to_file=None

#Which epochs from the parallel encoder history to take:
if params["channel"]==True:
    how_many_epochs_each_to_train = np.ones(100)
else:
    how_many_epochs_each_to_train = np.array([10,]*1+[2,]*5+[1,]*200)
print("Using parallel schedule", how_many_epochs_each_to_train[:12,], "...")

#Returns ( [[Test_epoch, Test_ydata, Train_epoch, Train_ydata], ...], ylabel_list, default_label_array)
#for every test file
data_from_files, ylabel_list, default_label_array = make_data_from_files(test_files, dump_to_file)

data_autoencoder = data_from_files[0]
data_parallel = data_from_files[1]

data_parallel_test, data_parallel_train = get_last_prl_epochs(data_autoencoder, data_parallel, how_many_epochs_each_to_train)

fig = make_plot_same_y_parallel(data_autoencoder, data_parallel_train, data_parallel_test, default_label_array, xlabel, ylabel_list,
                 title, legend_locations, labels_override, colors, xticks, figsize)
plt.show(fig)
<|endoftext|>


<commit_before>import bisect
from datetime import datetime

def dicts_to_series(list_of_dicts):
    """ convert data from yahoo finance """
    ret = []
    for day in list_of_dicts:
        ret.append(Day(day['Open'], day['High'], day['Low'], day['Close'], day['Adj_Close'],
                   datetime.strptime(day['Date'], "%Y-%m-%d"), day['Volume']))
    return Series(series=ret)

def db_to_series(symbol, start, end, db_con):
    """ read data from database """
    #TODO
    pass

class Day(object):
    """ used to store stock information by the day """
    def __init__(self, open, high, low, close, adj_close, date, volume):
        self.open = open
        self.high = high
        self.low = low
        self.volume = volume
        self.adj_close = adj_close
        self.close = close
        self.date = date

    def set_val(self, attr, value):
        """ used to set attribute values after the object has be initialized """
        setattr(self, attr, value)

    def __lt__(self, other):
        """ used to sort """
        return self.date < other.date

    def vectorize(self, features=None):
        """ return a vector of numbers to be used for machine learning """
        #TODO
        pass

class Series(object):
    """ used to represent a series of days """
    def __init__(self, series=None, config=None):
        self.series = []
        if self.series is not None:
            self.series.extend(series)
        self.sort()
        self.index = 0

    def insert(self, day):
        bisect.insort_left(self.series, day)

    def sort(self):
        self.series.sort()

    def run_calculations(self):
        self.calculate_mavg()

    def calculate_mavg(self):
        #TODO
        pass

    #TODO other calculations and signals

    def __iter__(self):
        """ allows for easy iteration ex: for day in series_obj: """
        for day in self.series:
            yield day

<commit_msg>joining in anand's TA work, this is an initial commit and needs some more work<commit_after>import bisect
from ifc.stockData import get_data_for_sym
import pandas as pd
import numpy as np
from datetime import datetime

def dicts_to_df(list_of_dicts):
    """ convert data from yahoo finance """
    df = pd.DataFrame(list_of_dicts)
    df.drop('Symbol', axis=1, inplace=True)
    df['Date'] = pd.to_datetime(df['Date'])
    df.set_index('Date')
    cols = ['High', 'Low', 'Volume', 'Open', 'Close', 'Adj_Close']
    for c in cols:
        df[c] = pd.to_numeric(df[c])
    return df

def get_series(ticker_sym, start, end):
    return Series(dicts_to_df(get_data_for_sym(ticker_sym, start, end)))

class Series(object):
    """ used to represent a series of days """
    def __init__(self, dataframe, config=None):
        self.df = dataframe
        self.max_win = 26
        if config is None: # default settings
            self.mavg = [10, 30]
            self.macd = [(9, 12, 26)]
            self.ema = []
            self.rsi = [14]
        else:
            raise NotImplementedError

    def run_calculations(self):
        """ runs a set of default calculations """
        for w in self.mavg:
            self.calculate_mavg(w)
        #for w in self.macd:
        #    self.calculate_macd(w[0], w[1], w[2])
        for w in self.ema:
            self.calculate_ema(w)
        for w in self.rsi:
            self.calculate_rsi(w)

    def set_max_win(self, val):
        self.max_win = val if val > self.max_win else self.max_win

    def trim_fat(self, window=None):
        temp_win = window
        if temp_win is None:
            temp_win = self.max_win
        self.df = self.df[temp_win:] # drop the first n rows typically containing Nan

    def calculate_mavg(self, window=10, col='Adj_Close'):
        """ calculates a simple moving average """
        self.set_max_win(window)
        name = "mavg_%s" % (window)
        self.df[name] = self.df['Adj_Close'].rolling(window=window).mean()

    def calculate_rsi(self, window=14, col='Adj_Close'):
        self.set_max_win(window)
        delta = self.df[col].diff()
        dUp, dDown = delta.copy(), delta.copy()
        dUp[dUp < 0] = 0
        dDown[dDown > 0] = 0
        RolUp = dUp.rolling(window=window).mean()
        RolDown = dDown.rolling(window=window).mean().abs()
        name = "rsi_%s" % (window)
        RS = RolUp / RolDown
        self.df[name] = 100.0 - (100.0 / (1.0 + RS))

    def calculate_ema(self, window, name=None, col='Adj_Close'):
        self.set_max_win(window)
        if name is None:
            name = "ema_%s" % (window)
        self.df[name] = self.df[col].ewm(span=window, min_periods=window)

    def calculate_macd(self, signal=9, fast=12, slow=26, col='Adj_Close'):
        """ MACD """
        self.set_max_win(window)
        signal_name = "signal_%s" % (signal)
        #self.calculate_ema(signal, signal_name)

        fast_ema = self.df[col].ewm(span=fast, min_periods=fast)
        slow_ema = self.df[col].ewm(span=slow, min_periods=slow)
        name = "macd_%s_%s" % (fast, slow)
        self.df[name] = pd.Series(fast_ema) - pd.Series(slow_ema)
        self.calculate_ema(signal, col=name)

    def calculate_mom(self, col='Adj_Close', window=1):
        """ Momentum Measures the change in price
                Price(t)-Price(t-n)
        """
        self.set_max_win(window)
        name = "mom_%s" % (window)
        self.df[name] = self.df[col] - self.df[col].shift(window)

    def calculate_rocr(self, window=3, col='Adj_Close'):
        """ Rate of Change Compute rate of change
            relative to previous trading intervals
                (Price(t)/Price(t-n))*100
        """
        self.set_max_win(window)
        name = "rocr_%s" % (window)
        self.df[name] = (self.df[col] / self.df[col].shift(window)) * 100

    def calculate_atr(self, window=14):
        """ Average True Range Shows volatility of market
                ATR(t) = ((n-1) * ATR(t-1) + Tr(t)) / n
                where Tr(t) = Max(Abs(High - Low), Abs(High - Close(t - 1)), Abs(Low - Close(t - 1));
        """
        self.set_max_win(window)
        i = 0
        tr_l = [0]
        for i in range(self.df.index[-1]):
            tr = max(self.df.get_value(i + 1, 'High'),
                     self.df.get_value(i, 'Adj_Close')) - min(self.df.get_value(i + 1, 'Low'),
                     self.df.get_value(i, 'Adj_Close'))
            tr_l.append(tr)
        name = 'atr_%s' % (window)
        self.df[name] = pd.Series(tr_l).ewm(span=window, min_periods=window)

    def calculate_mfi(self, window=14):
        """ Money Flow Index Relates typical price with Volume
                100 - (100 / (1 + Money Ratio))
                where Money Ratio=(+Moneyflow / -Moneyflow);
                Moneyflow=Tp*Volume
                Tp = (High + Low + Close)/3
        """
        name = "mfi_%s" % (window)
        tp = (self.df['High'] + self.df['Low'] + self.df['Adj_Close']) / 3
        i = 0
        PosMF = [0]
        while i < self.df.index[-1]:
            if tp[i + 1] > tp[i]:
                PosMF.append(tp[i + 1] * self.df.get_value(i + 1, 'Volume'))
            else:
                PosMF.append(0)
            i = i + 1
        PosMF = pd.Series(PosMF)
        TotMF = tp * self.df['Volume']
        MFR = pd.Series(PosMF / TotMF)
        self.df[name] = pd.Series(pd.rolling_mean(MFR, window))

    def calculate_adx(self, window=14, window_adx=14):
        """ Average Directional Index Discover if trend is developing
                Sum((+DI-(-DI))/(+DI+(-DI))/n
        """
        self.set_max_win(window)
        i = 0
        UpI = []
        DoI = []
        while i + 1 <= self.df.index[-1]:
            UpMove = self.df.get_value(i + 1, 'High') - self.df.get_value(i, 'High')
            DoMove = self.df.get_value(i, 'Low') - self.df.get_value(i + 1, 'Low')
            if UpMove > DoMove and UpMove > 0:
                UpD = UpMove
            else: UpD = 0
            UpI.append(UpD)
            if DoMove > UpMove and DoMove > 0:
                DoD = DoMove
            else: DoD = 0
            DoI.append(DoD)
            i = i + 1
        atr_name = "atr_%s" % (window)
        self.calculate_atr(window)
        PosDI = pd.Series(UpI).ewm(span=window, min_periods=window-1) / self.df[atr_name]
        NegDI = pd.Series(DoI).ewm(span=window, min_periods=window-1) / self.df[atr_name]
        name = "adx_%s_%s" % (window, window_adx)
        self.df[name] = pd.Series(pd.ewma(abs(PosDI - NegDI) / (PosDI + NegDI), span=window_adx, min_periods=window_adx-1))

    def calculate_cci(self, window=12):
        """ Commodity Channel Index Identifies cyclical turns in stock price
                Tp(t) - TpAvg(t,n) / (0.15 * MD(t)) where:
                Tp(t) = (High(t)+Low(t)+Close(t))/3;
                TpAvg(t,n) = Avg(Tp(t)) over [t, t-1, ..., t-n+1];
                MD(t)=Avg(Abs(Tp(t)-TpAvg(t,n)));
        """
        self.set_max_win(window)
        tp = (self.df['High'] + self.df['Low'] + self.df['Adj_Close']) / 3
        name = "cci_%s" % (window)
        self.df[name] = pd.Series((tp - pd.rolling_mean(tp, window)) / pd.rolling_std(tp, window))

    def calculate_obv(self, col='Adj_Close', window=14):
        """ On Balance Volume is a momentum indicator that uses volume flow
                OBV(t) = OBV(t-1) +/-Volume(t)
        """
        self.set_max_win(window)
        i = 0
        obv = [0]
        for i in range(len(self.df.index[-1])):
            if self.df.get_value(i + 1, col) - self.df.get_value(i, col) > 0:
                obv.append(self.df.get_value(i + 1, 'Volume'))
            if self.df.get_value(i + 1, col) - self.df.get_value(i, col) == 0:
                obv.append(0)
            if self.df.get_value(i + 1, col) - self.df.get_value(i, col) < 0:
                obv.append(-self.df.get_value(i + 1, 'Volume'))
        name = "obv_%s" % (window)
        self.df[name] = pd.Series(obv).rolling(window=window).mean()

    def calculate_trix(self, window=15):
        """ Triple Exponential Moving Average Smooth the insignificant movements
                TR(t) / TR(t-1) where
                TR(t) = EMA(EMA(EMA(Price(t)))) over n days period
        """
        self.set_max_win(window)
        ema = self.df['Adj_Close'].ewm(span=window, min_periods=window-1)
        ema = ema.ewm(span=window, min_periods=window-1)
        ema = ema.ewm(span=window, min_periods=window-1)

        i = 0
        roc_l = [0]
        while i + 1 <= self.df.index[-1]:
            roc_l.append((ema[i + 1] - ema[i]) / ema[i])
            i += 1

        name = "trix_%s" % (window)
        self.df[name] = pd.Series(roc_l)

<|endoftext|>