<commit_before># Copyright 2019 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""The fallback skill implements a special type of skill handling
utterances not handled by the intent system.
"""
import operator
from mycroft.metrics import report_timing, Stopwatch
from mycroft.util.log import LOG

from .mycroft_skill import MycroftSkill, get_handler_name

class FallbackSkill(MycroftSkill):
    """Fallbacks come into play when no skill matches an Adapt or closely with
    a Padatious intent.  All Fallback skills work together to give them a
    view of the user's utterance.  Fallback handlers are called in an order
    determined the priority provided when the the handler is registered.

    ========   ========   ================================================
    Priority   Who?       Purpose
    ========   ========   ================================================
       1-4     RESERVED   Unused for now, slot for pre-Padatious if needed
         5     MYCROFT    Padatious near match (conf > 0.8)
      6-88     USER       General
        89     MYCROFT    Padatious loose match (conf > 0.5)
     90-99     USER       Uncaught intents
       100+    MYCROFT    Fallback Unknown or other future use
    ========   ========   ================================================

    Handlers with the numerically lowest priority are invoked first.
    Multiple fallbacks can exist at the same priority, but no order is
    guaranteed.

    A Fallback can either observe or consume an utterance. A consumed
    utterance will not be see by any other Fallback handlers.
    """
    fallback_handlers = {}
    wrapper_map = []  # Map containing (handler, wrapper) tuples

    def __init__(self, name=None, bus=None, use_settings=True):
        super().__init__(name, bus, use_settings)

        #  list of fallback handlers registered by this instance
        self.instance_fallback_handlers = []

    @classmethod
    def make_intent_failure_handler(cls, bus):
        """Goes through all fallback handlers until one returns True"""

        def handler(message):
            # indicate fallback handling start
            bus.emit(message.forward("mycroft.skill.handler.start",
                                     data={'handler': "fallback"}))

            stopwatch = Stopwatch()
            handler_name = None
            with stopwatch:
                for _, handler in sorted(cls.fallback_handlers.items(),
                                         key=operator.itemgetter(0)):
                    try:
                        if handler(message):
                            #  indicate completion
                            handler_name = get_handler_name(handler)
                            bus.emit(message.forward(
                                     'mycroft.skill.handler.complete',
                                     data={'handler': "fallback",
                                           "fallback_handler": handler_name}))
                            break
                    except Exception:
                        LOG.exception('Exception in fallback.')
                else:  # No fallback could handle the utterance
                    bus.emit(message.forward('complete_intent_failure'))
                    warning = "No fallback could handle intent."
                    LOG.warning(warning)
                    #  indicate completion with exception
                    bus.emit(message.forward('mycroft.skill.handler.complete',
                                             data={'handler': "fallback",
                                                   'exception': warning}))

            # Send timing metric
            if message.context.get('ident'):
                ident = message.context['ident']
                report_timing(ident, 'fallback_handler', stopwatch,
                              {'handler': handler_name})

        return handler

    @classmethod
    def _register_fallback(cls, handler, wrapper, priority):
        """Register a function to be called as a general info fallback
        Fallback should receive message and return
        a boolean (True if succeeded or False if failed)

        Lower priority gets run first
        0 for high priority 100 for low priority

        Arguments:
            handler (callable): original handler, used as a reference when
                                removing
            wrapper (callable): wrapped version of handler
            priority (int): fallback priority
        """
        while priority in cls.fallback_handlers:
            priority += 1

        cls.fallback_handlers[priority] = wrapper
        cls.wrapper_map.append((handler, wrapper))

    def register_fallback(self, handler, priority):
        """Register a fallback with the list of fallback handlers and with the
        list of handlers registered by this instance
        """

        def wrapper(*args, **kwargs):
            if handler(*args, **kwargs):
                self.make_active()
                return True
            return False

        self.instance_fallback_handlers.append(handler)
        self._register_fallback(handler, wrapper, priority)

    @classmethod
    def _remove_registered_handler(cls, wrapper_to_del):
        """Remove a registered wrapper.

        Arguments:
            wrapper_to_del (callable): wrapped handler to be removed

        Returns:
            (bool) True if one or more handlers were removed, otherwise False.
        """
        found_handler = False
        for priority, handler in list(cls.fallback_handlers.items()):
            if handler == wrapper_to_del:
                found_handler = True
                del cls.fallback_handlers[priority]

        if not found_handler:
            LOG.warning('No fallback matching {}'.format(wrapper_to_del))
        return found_handler

    @classmethod
    def remove_fallback(cls, handler_to_del):
        """Remove a fallback handler.

        Arguments:
            handler_to_del: reference to handler
        Returns:
            (bool) True if at least one handler was removed, otherwise False
        """
        # Find wrapper from handler or wrapper
        wrapper_to_del = None
        for h, w in cls.wrapper_map:
            if handler_to_del in (h, w):
                wrapper_to_del = w
                break

        if wrapper_to_del:
            cls.wrapper_map.remove((h, w))
            remove_ok = cls._remove_registered_handler(wrapper_to_del)
        else:
            LOG.warning('Could not find matching fallback handler')
            remove_ok = False
        return remove_ok

    def remove_instance_handlers(self):
        """Remove all fallback handlers registered by the fallback skill."""
        self.log.info('Removing all handlers...')
        while len(self.instance_fallback_handlers):
            handler = self.instance_fallback_handlers.pop()
            self.remove_fallback(handler)

    def default_shutdown(self):
        """Remove all registered handlers and perform skill shutdown."""
        self.remove_instance_handlers()
        super(FallbackSkill, self).default_shutdown()
<commit_msg>Add support for ranges of fallbacks<commit_after># Copyright 2019 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""The fallback skill implements a special type of skill handling
utterances not handled by the intent system.
"""
import operator
from mycroft.metrics import report_timing, Stopwatch
from mycroft.util.log import LOG

from .mycroft_skill import MycroftSkill, get_handler_name

class FallbackSkill(MycroftSkill):
    """Fallbacks come into play when no skill matches an Adapt or closely with
    a Padatious intent.  All Fallback skills work together to give them a
    view of the user's utterance.  Fallback handlers are called in an order
    determined the priority provided when the the handler is registered.

    ========   ========   ================================================
    Priority   Who?       Purpose
    ========   ========   ================================================
       1-4     RESERVED   Unused for now, slot for pre-Padatious if needed
         5     MYCROFT    Padatious near match (conf > 0.8)
      6-88     USER       General
        89     MYCROFT    Padatious loose match (conf > 0.5)
     90-99     USER       Uncaught intents
       100+    MYCROFT    Fallback Unknown or other future use
    ========   ========   ================================================

    Handlers with the numerically lowest priority are invoked first.
    Multiple fallbacks can exist at the same priority, but no order is
    guaranteed.

    A Fallback can either observe or consume an utterance. A consumed
    utterance will not be see by any other Fallback handlers.
    """
    fallback_handlers = {}
    wrapper_map = []  # Map containing (handler, wrapper) tuples

    def __init__(self, name=None, bus=None, use_settings=True):
        super().__init__(name, bus, use_settings)

        #  list of fallback handlers registered by this instance
        self.instance_fallback_handlers = []

    @classmethod
    def make_intent_failure_handler(cls, bus):
        """Goes through all fallback handlers until one returns True"""

        def handler(message):
            start, stop = message.data.get('fallback_range', (0, 101))
            # indicate fallback handling start
            LOG.debug('Checking fallbacks in range '
                      '{} - {}'.format(start, stop))
            bus.emit(message.forward("mycroft.skill.handler.start",
                                     data={'handler': "fallback"}))

            stopwatch = Stopwatch()
            handler_name = None
            with stopwatch:
                sorted_handlers = sorted(cls.fallback_handlers.items(),
                                         key=operator.itemgetter(0))
                handlers = [f[1] for f in sorted_handlers
                            if start <= f[0] < stop]
                for handler in handlers:
                    try:
                        if handler(message):
                            # indicate completion
                            status = True
                            handler_name = get_handler_name(handler)
                            bus.emit(message.forward(
                                     'mycroft.skill.handler.complete',
                                     data={'handler': "fallback",
                                           "fallback_handler": handler_name}))
                            break
                    except Exception:
                        LOG.exception('Exception in fallback.')
                else:
                    status = False
                    #  indicate completion with exception
                    warning = 'No fallback could handle intent.'
                    bus.emit(message.forward('mycroft.skill.handler.complete',
                                             data={'handler': "fallback",
                                                   'exception': warning}))
                    if 'fallback_range' not in message.data:
                        # Old system TODO: Remove in 20.08
                        # No fallback could handle the utterance
                        bus.emit(message.forward('complete_intent_failure'))
                        LOG.warning(warning)

            # return if the utterance was handled to the caller
            bus.emit(message.response(data={'handled': status}))

            # Send timing metric
            if message.context.get('ident'):
                ident = message.context['ident']
                report_timing(ident, 'fallback_handler', stopwatch,
                              {'handler': handler_name})

        return handler

    @classmethod
    def _register_fallback(cls, handler, wrapper, priority):
        """Register a function to be called as a general info fallback
        Fallback should receive message and return
        a boolean (True if succeeded or False if failed)

        Lower priority gets run first
        0 for high priority 100 for low priority

        Arguments:
            handler (callable): original handler, used as a reference when
                                removing
            wrapper (callable): wrapped version of handler
            priority (int): fallback priority
        """
        while priority in cls.fallback_handlers:
            priority += 1

        cls.fallback_handlers[priority] = wrapper
        cls.wrapper_map.append((handler, wrapper))

    def register_fallback(self, handler, priority):
        """Register a fallback with the list of fallback handlers and with the
        list of handlers registered by this instance
        """

        def wrapper(*args, **kwargs):
            if handler(*args, **kwargs):
                self.make_active()
                return True
            return False

        self.instance_fallback_handlers.append(handler)
        self._register_fallback(handler, wrapper, priority)

    @classmethod
    def _remove_registered_handler(cls, wrapper_to_del):
        """Remove a registered wrapper.

        Arguments:
            wrapper_to_del (callable): wrapped handler to be removed

        Returns:
            (bool) True if one or more handlers were removed, otherwise False.
        """
        found_handler = False
        for priority, handler in list(cls.fallback_handlers.items()):
            if handler == wrapper_to_del:
                found_handler = True
                del cls.fallback_handlers[priority]

        if not found_handler:
            LOG.warning('No fallback matching {}'.format(wrapper_to_del))
        return found_handler

    @classmethod
    def remove_fallback(cls, handler_to_del):
        """Remove a fallback handler.

        Arguments:
            handler_to_del: reference to handler
        Returns:
            (bool) True if at least one handler was removed, otherwise False
        """
        # Find wrapper from handler or wrapper
        wrapper_to_del = None
        for h, w in cls.wrapper_map:
            if handler_to_del in (h, w):
                wrapper_to_del = w
                break

        if wrapper_to_del:
            cls.wrapper_map.remove((h, w))
            remove_ok = cls._remove_registered_handler(wrapper_to_del)
        else:
            LOG.warning('Could not find matching fallback handler')
            remove_ok = False
        return remove_ok

    def remove_instance_handlers(self):
        """Remove all fallback handlers registered by the fallback skill."""
        self.log.info('Removing all handlers...')
        while len(self.instance_fallback_handlers):
            handler = self.instance_fallback_handlers.pop()
            self.remove_fallback(handler)

    def default_shutdown(self):
        """Remove all registered handlers and perform skill shutdown."""
        self.remove_instance_handlers()
        super(FallbackSkill, self).default_shutdown()
<|endoftext|>


<commit_before>from django.conf.urls import url, include
from haystack.forms import FacetedSearchForm
from haystack.views import FacetedSearchView
from haystack.query import SearchQuerySet
from django.conf.urls import patterns
from isisdata.forms import *
from isisdata.views import IsisSearchView

from . import views

sqs = SearchQuerySet().facet('authors', size=300). \
        facet('type', size=300). \
        facet('publication_date', size=300). \
        facet('persons', size=300). \
        facet('categories', size=300). \
        facet('subjects', size=300). \
        facet('editors', size=300). \
        facet('advisors', size=300). \
        facet('translators', size=300). \
        facet('publishers', size=300). \
        facet('schools', size=300). \
        facet('institutions', size=300). \
        facet('meetings', size=300). \
        facet('periodicals', size=300). \
        facet('book_series', size=300). \
        facet('time_periods', size=300). \
        facet('geographics', size=300). \
        facet('people', size=300). \
        facet('subject_institutions', size=300). \
        facet('serial_publiations', size=300). \
        facet('classification_terms', size=300). \
        facet('concepts', size=300). \
        facet('creative_works', size=300). \
        facet('events', size=300). \
        facet('authority_type', size=300)

urlpatterns = [
    #url(r'^$', views.index, name='index'),
    url(r'^$',
        IsisSearchView(form_class=MyFacetedSearchForm, searchqueryset=sqs),
        name='index'),
    url(r'^(?P<obj_id>[A-Z]+[0-9]+)/$', views.index, name='index'),
    url(r'^authority/(?P<authority_id>[A-Z]+[0-9]+)/$',
        views.authority,
        name='authority'),
    url(r'^citation/(?P<citation_id>[A-Z]+[0-9]+)/$',
        views.citation,
        name='citation'),
    url(r'^(?P<base_view>[A-Za-z]+)/(?P<obj_id>[A-Z]+[0-9]+).json$', views.api_redirect),
    url(r'^search/',
        IsisSearchView(form_class=MyFacetedSearchForm, searchqueryset=sqs),
        name='haystack_search'),
    url(r'^unapi/+$', views.unapi_server_root, name='unapi'),
]
<commit_msg>ISISCB-153 max facet size is 100<commit_after>from django.conf.urls import url, include
from haystack.forms import FacetedSearchForm
from haystack.views import FacetedSearchView
from haystack.query import SearchQuerySet
from django.conf.urls import patterns
from isisdata.forms import *
from isisdata.views import IsisSearchView

from . import views

sqs = SearchQuerySet().facet('authors', size=100). \
        facet('type', size=100). \
        facet('publication_date', size=100). \
        facet('persons', size=100). \
        facet('categories', size=100). \
        facet('subjects', size=100). \
        facet('editors', size=100). \
        facet('advisors', size=100). \
        facet('translators', size=100). \
        facet('publishers', size=100). \
        facet('schools', size=100). \
        facet('institutions', size=100). \
        facet('meetings', size=100). \
        facet('periodicals', size=100). \
        facet('book_series', size=100). \
        facet('time_periods', size=100). \
        facet('geographics', size=100). \
        facet('people', size=100). \
        facet('subject_institutions', size=100). \
        facet('serial_publiations', size=100). \
        facet('classification_terms', size=100). \
        facet('concepts', size=100). \
        facet('creative_works', size=100). \
        facet('events', size=100). \
        facet('authority_type', size=100)

urlpatterns = [
    #url(r'^$', views.index, name='index'),
    url(r'^$',
        IsisSearchView(form_class=MyFacetedSearchForm, searchqueryset=sqs),
        name='index'),
    url(r'^(?P<obj_id>[A-Z]+[0-9]+)/$', views.index, name='index'),
    url(r'^authority/(?P<authority_id>[A-Z]+[0-9]+)/$',
        views.authority,
        name='authority'),
    url(r'^citation/(?P<citation_id>[A-Z]+[0-9]+)/$',
        views.citation,
        name='citation'),
    url(r'^(?P<base_view>[A-Za-z]+)/(?P<obj_id>[A-Z]+[0-9]+).json$', views.api_redirect),
    url(r'^search/',
        IsisSearchView(form_class=MyFacetedSearchForm, searchqueryset=sqs),
        name='haystack_search'),
    url(r'^unapi/+$', views.unapi_server_root, name='unapi'),
]
<|endoftext|>


<commit_before>import os
from tensorflow_datasets.core import registered
import re
import shutil

# Get list of all available versions of datasets
registered_versions = sorted([i for i in registered.iter_dataset_full_names()])

meta_dirs = []
for root, dirs, files in os.walk(os.path.join(os.pardir, "testing/metadata")):
    meta_dirs.append(root)

# Get all paths in proper format.

paths = []
for meta_dir in meta_dirs:
    match = re.search(r'\d.\d.\d', meta_dir)
    try:
            meta_dir == match.group()
            paths.append(os.path.relpath(meta_dir, '..\\testing/metadata'))
    except:
        continue

number_of_meta_removed = 0
for i in range(0,len(paths)):

    if paths[i] not in registered_versions:
        number_of_meta_removed+=1
        shutil.rmtree(os.path.join(os.pardir, "testing/metadata",paths[i]))

print("*"*80)
print("Total number of registered datasets : ", len(registered_versions))
print("Total number of subdirectories in metadata direcotry : ", len(paths))
print("Total number of meta directories removed : ", len(number_of_meta_removed))
print("*"*80)
<commit_msg>Update metadata_cleanup.py<commit_after>import os
from tensorflow_datasets.core import registered
import re
import shutil

# Get list of all available versions of datasets
registered_versions = sorted([i for i in registered.iter_dataset_full_names()])

meta_dirs = []
for root, dirs, files in os.walk(os.path.join(os.pardir, "testing/metadata")):
    meta_dirs.append(root)

# Get all paths in proper format.

paths = []
for meta_dir in meta_dirs:
    match = re.search(r'\d.\d.\d', meta_dir)
    try:
            meta_dir == match.group()
            paths.append(os.path.relpath(meta_dir, '..\\testing/metadata'))
    except:
        continue

number_of_meta_removed = 0
for i in range(0,len(paths)):
    if paths[i] not in registered_versions:
        number_of_meta_removed+=1
        shutil.rmtree(os.path.join(os.pardir, "testing/metadata",paths[i]))

print("*"*80)
print("Total number of registered datasets : ", len(registered_versions))
print("Total number of subdirectories in metadata direcotry : ", len(paths))
print("Total number of meta directories removed : ", len(number_of_meta_removed))
print("*"*80)
<|endoftext|>


<commit_before>from django.test import TestCase
from django.shortcuts import resolve_url as r
from django.contrib.auth import get_user_model
from myideas.core.forms import IdeasForm
from myideas.core.models import Ideas

class HomeTest(TestCase):

    def setUp(self):
        user = get_user_model().objects.create(username='adminapp')
        self.idea = Ideas.objects.create(user=user, title='test app')
        self.response = self.client.get(r('home'))

    def test_get(self):
        """GET 'Home' must return status code 200"""
        self.assertEqual(200, self.response.status_code)

    def test_template(self):
        """'Home' must use template index.html and base.html"""
        self.assertTemplateUsed(self.response, 'index.html')
        self.assertTemplateUsed(self.response, 'base.html')

    def test_login_link(self):
        """base.html navbar must contains login page link"""
        expected = 'href="{}"'.format(r('auth_login'))
        self.assertContains(self.response, expected)

    def test_register_link(self):
        """base.html navbar must contains register page link"""
        expected = 'href="{}"'.format(r('registration_register'))
        self.assertContains(self.response, expected)

    def test_ideas_form_link(self):
        """base.html navbar contains ideas_form link"""
        expected = 'href="{}"'.format(r('ideas_form'))
        self.assertContains(self.response, expected)

    def test_ideas_details_link(self):
        """home contains idea_details links"""
        expected = 'href="{}"'.format(r('idea_details', slug=self.idea.slug))
        self.assertContains(self.response, expected)

class DetailsTest(TestCase):

    def setUp(self):
        user = get_user_model().objects.create(username='adminapp')
        self.idea = Ideas.objects.create(user=user, title='test app')
        self.response = self.client.get(r(self.idea.get_absolute_url()))

    def test_get(self):
        """GET 'Ideas Details' must return status code 200"""
        self.assertEqual(200, self.response.status_code)

    def test_template(self):
        """'Ideas Details' must use template ideas_details.html and base.html"""
        self.assertTemplateUsed(self.response, 'ideas_details.html')
        self.assertTemplateUsed(self.response, 'base.html')

    def test_html(self):
        self.assertContains(self.response, 'adminapp')
        self.assertContains(self.response, 'test app')

class ProfileTest(TestCase):

    def setUp(self):
        user = get_user_model().objects.create(username='adminapp')
        self.idea = Ideas.objects.create(user=user)
        self.response = self.client.get(r('profile', self.idea.user))

    def test_get(self):
        """GET 'User Profile' must return status code 200"""
        self.assertEqual(200, self.response.status_code)

    def test_template(self):
        """'User Profile' must use template profile.html and base.html"""
        self.assertTemplateUsed(self.response, 'profile.html')
        self.assertTemplateUsed(self.response, 'base.html')

    def test_html(self):
        self.assertContains(self.response, 'adminapp')

class IdeaFormTest(TestCase):

    def setUp(self):
        self.response = self.client.get(r('ideas_form'))

    def test_get(self):
        """GET 'Ideas Form' must return status code 200"""
        self.assertEqual(200, self.response.status_code)

    def test_has_form_on_context(self):
        self.assertIsInstance(self.response.context['form'], IdeasForm)

    def test_template(self):
        """'Ideas Form' must use template index.html and base.html"""
        self.assertTemplateUsed(self.response, 'idea_form.html')
        self.assertTemplateUsed(self.response, 'base.html')

class IdeasDetailNotFound(TestCase):

    def setUp(self):
        self.response = self.client.get(r('ideas_details.html', slug='not-found'))

    def test_not_found(self):
        """GET page not found must return status code 404"""
        self.assertEqual(404, self.response.status_code)

    def test_template(self):
        """'page not found' must use template 404.html and base.html"""
        self.assertTemplateUsed(self.response, '404.html')
        self.assertTemplateUsed(self.response, 'base.html')

class RegisterUserAdmin(TestCase):

    def test_registration(self):
        """GET registration must return status code 200 and use template base.html"""
        response = self.client.get(r('registration_register'))
        self.assertEqual(200, response.status_code)
        self.assertTemplateUsed(response,
                                'registration/registration_form.html')
        self.assertTemplateUsed(response, 'base.html')

    def test_login(self):
        """GET login must return status code 200 and use template base.html"""
        response = self.client.get(r('auth_login'))
        self.assertEqual(200, response.status_code)
        self.assertTemplateUsed(response,
                                'registration/login.html')
        self.assertTemplateUsed(response, 'base.html')

    def test_logout(self):
        """GET logout must return status code 200 and use template base.html"""
        response = self.client.get(r('auth_logout'))
        self.assertEqual(200, response.status_code)
        self.assertTemplateUsed(response,
                                'registration/logout.html')
        self.assertTemplateUsed(response, 'base.html')

    def test_get_and_templates_reset_complete(self):
        """GET pass reset must return status code 200 and use template base.html"""
        response = self.client.get(r('auth_password_reset_complete'))
        self.assertEqual(200, response.status_code)
        self.assertTemplateUsed(response,
                                'registration/password_reset_complete.html')
        self.assertTemplateUsed(response, 'base.html')
<commit_msg>update test_views<commit_after>from django.test import TestCase
from django.shortcuts import resolve_url as r
from django.contrib.auth import get_user_model
from myideas.core.forms import IdeasForm
from myideas.core.models import Ideas

class HomeTest(TestCase):

    def setUp(self):
        user = get_user_model().objects.create(username='adminapp')
        self.idea = Ideas.objects.create(user=user, title='test app')
        self.response = self.client.get(r('home'))

    def test_get(self):
        """GET 'Home' must return status code 200"""
        self.assertEqual(200, self.response.status_code)

    def test_template(self):
        """'Home' must use template index.html and base.html"""
        self.assertTemplateUsed(self.response, 'index.html')
        self.assertTemplateUsed(self.response, 'base.html')

    def test_login_link(self):
        """base.html navbar must contains login page link"""
        expected = 'href="{}"'.format(r('auth_login'))
        self.assertContains(self.response, expected)

    def test_register_link(self):
        """base.html navbar must contains register page link"""
        expected = 'href="{}"'.format(r('registration_register'))
        self.assertContains(self.response, expected)

    def test_ideas_form_link(self):
        """base.html navbar contains ideas_form link"""
        expected = 'href="{}"'.format(r('ideas_form'))
        self.assertContains(self.response, expected)

    def test_ideas_details_link(self):
        """home contains idea_details links"""
        expected = 'href="{}"'.format(r('idea_details', slug=self.idea.slug))
        self.assertContains(self.response, expected)

    def test_profile_link(self):
        """home contains profile links"""
        expected = 'href="{}"'.format(r('profile', self.idea.user))
        self.assertContains(self.response, expected)

class DetailsTest(TestCase):

    def setUp(self):
        user = get_user_model().objects.create(username='adminapp')
        self.idea = Ideas.objects.create(user=user, title='test app')
        self.response = self.client.get(r(self.idea.get_absolute_url()))

    def test_get(self):
        """GET 'Ideas Details' must return status code 200"""
        self.assertEqual(200, self.response.status_code)

    def test_template(self):
        """'Ideas Details' must use template ideas_details.html and base.html"""
        self.assertTemplateUsed(self.response, 'ideas_details.html')
        self.assertTemplateUsed(self.response, 'base.html')

    def test_html(self):
        self.assertContains(self.response, 'adminapp')
        self.assertContains(self.response, 'test app')

class ProfileTest(TestCase):

    def setUp(self):
        user = get_user_model().objects.create(username='adminapp')
        self.idea = Ideas.objects.create(user=user)
        self.response = self.client.get(r('profile', self.idea.user))

    def test_get(self):
        """GET 'User Profile' must return status code 200"""
        self.assertEqual(200, self.response.status_code)

    def test_template(self):
        """'User Profile' must use template profile.html and base.html"""
        self.assertTemplateUsed(self.response, 'profile.html')
        self.assertTemplateUsed(self.response, 'base.html')

    def test_html(self):
        self.assertContains(self.response, 'adminapp')

class IdeaFormTest(TestCase):

    def setUp(self):
        self.response = self.client.get(r('ideas_form'))

    def test_get(self):
        """GET 'Ideas Form' must return status code 200"""
        self.assertEqual(200, self.response.status_code)

    def test_has_form_on_context(self):
        self.assertIsInstance(self.response.context['form'], IdeasForm)

    def test_template(self):
        """'Ideas Form' must use template index.html and base.html"""
        self.assertTemplateUsed(self.response, 'idea_form.html')
        self.assertTemplateUsed(self.response, 'base.html')

class IdeasDetailNotFound(TestCase):

    def setUp(self):
        self.response = self.client.get(r('ideas_details.html', slug='not-found'))

    def test_not_found(self):
        """GET page not found must return status code 404"""
        self.assertEqual(404, self.response.status_code)

    def test_template(self):
        """'page not found' must use template 404.html and base.html"""
        self.assertTemplateUsed(self.response, '404.html')
        self.assertTemplateUsed(self.response, 'base.html')

class RegisterUserAdmin(TestCase):

    def test_registration(self):
        """GET registration must return status code 200 and use template base.html"""
        response = self.client.get(r('registration_register'))
        self.assertEqual(200, response.status_code)
        self.assertTemplateUsed(response,
                                'registration/registration_form.html')
        self.assertTemplateUsed(response, 'base.html')

    def test_login(self):
        """GET login must return status code 200 and use template base.html"""
        response = self.client.get(r('auth_login'))
        self.assertEqual(200, response.status_code)
        self.assertTemplateUsed(response,
                                'registration/login.html')
        self.assertTemplateUsed(response, 'base.html')

    def test_logout(self):
        """GET logout must return status code 200 and use template base.html"""
        response = self.client.get(r('auth_logout'))
        self.assertEqual(200, response.status_code)
        self.assertTemplateUsed(response,
                                'registration/logout.html')
        self.assertTemplateUsed(response, 'base.html')

    def test_get_and_templates_reset_complete(self):
        """GET pass reset must return status code 200 and use template base.html"""
        response = self.client.get(r('auth_password_reset_complete'))
        self.assertEqual(200, response.status_code)
        self.assertTemplateUsed(response,
                                'registration/password_reset_complete.html')
        self.assertTemplateUsed(response, 'base.html')
<|endoftext|>


<commit_before>__author__ = "Felipe Zapata"

# ================> Python Standard  and third-party <==========
from itertools import chain
from os.path import join

import h5py
import numpy as np
# ==================> Internal modules <==========
from nac.integrals import (calculateCoupling3Points,
                           compute_overlaps_for_coupling,
                           correct_phases)
from nac.common import (femtosec2au, retrieve_hdf5_data)
from qmworks.hdf5.quantumHDF5 import StoreasHDF5

# Types hint
from typing import (Dict, List, Tuple)

# Numpy type hints
Vector = np.ndarray
Matrix = np.ndarray

# ==============================> Schedule Tasks <=============================

def lazy_couplings(paths_overlaps: List, path_hdf5: str, project_name: str,
                   enumerate_from: int, dt: float) -> List:
    """
    :parameter dt: dynamic integration time
    :type      dt: Float (Femtoseconds)
    """
    # time in atomic units
    dt_au = dt * femtosec2au

    # Compute the dimension of the coupling matrix
    mtx_0 = retrieve_hdf5_data(path_hdf5, paths_overlaps[0][0])
    _, dim = mtx_0.shape

    # Read all the Overlaps
    concat_paths = chain(*paths_overlaps)
    overlaps = np.stack([retrieve_hdf5_data(path_hdf5, ps)
                         for ps in concat_paths])

    # Number of couplings to compute
    nCouplings = overlaps.shape[0] // 2 - 1

    # Compute all the phases
    mtx_phases = compute_phases(overlaps, nCouplings, dim)

    # Compute the couplings using the four matrices previously calculated
    # Together with the phases
    paths_couplings = []

    for i in range(nCouplings):
        # Path were the couplinp is store
        k = i + enumerate_from
        path = join(project_name, 'coupling_{}'.format(k))
        with h5py.File(path_hdf5, 'r+') as f5:
            is_done  = path in f5

        # Skip the computation if the coupling is already done
        if is_done:
            print("Coupling: ", path, " has already been calculated")
        else:
            # Tensor containing the overlaps
            j = 2 * i
            ps = overlaps[j: j + 4, :, :]
            # Correct the Phase of the Molecular orbitals
            fixed_phase_overlaps = correct_phases(
                ps, mtx_phases[i: i + 3, :], dim)

            # Compute the couplings with the phase corrected overlaps
            couplings = calculateCoupling3Points(dt_au, *fixed_phase_overlaps)

            with h5py.File(path_hdf5, 'r+') as f5:
                store = StoreasHDF5(f5, 'cp2k')
                store.funHDF5(path, couplings)

        paths_couplings.append(path)

    return paths_couplings

def compute_phases(overlaps: List, nCouplings: int, dim: int) -> Matrix:
    """
    Compute the phase of the state_i at time t + dt, using the following
    equation:
    phase_i(t+dt) = Sii(t) * phase_i(t)
    """
    # initial references
    references = np.ones(dim)

    # Matrix containing the phases
    mtx_phases = np.empty((nCouplings, dim))
    mtx_phases[0, :] = references

    # Compute the phases at times t + dt using the phases at time t
    for i in range(nCouplings - 1):
        Sji_t = overlaps[2 * i, :, :].reshape(dim, dim)
        phases = np.sign(np.diag(Sji_t)) * references
        mtx_phases[i + 1] = phases
        references = phases

    return mtx_phases

def lazy_overlaps(i: int, project_name: str, path_hdf5: str, dictCGFs: Dict,
                  geometries: Tuple, mo_paths: List, hdf5_trans_mtx: str=None,
                  enumerate_from: int=0, nHOMO: int=None,
                  couplings_range: Tuple=None) -> str:
    """
    Calculate the 4 overlap matrix used to compute the subsequent couplings.
    The overlap matrices are computed using 3 consecutive set of MOs and
    3 consecutive geometries( in atomic units), from a molecular dynamics.

    :param i: nth coupling calculation
    :type i: int
    :param project_name: Name of the project to be executed.
    :type project_name: str
    :paramter dictCGFS: Dictionary from Atomic Label to basis set
    :type     dictCGFS: Dict String [CGF],
              CGF = ([Primitives], AngularMomentum),
              Primitive = (Coefficient, Exponent)
    :parameter geometries: molecular geometries stored as list of
                           namedtuples.
    :type      geometries: ([AtomXYZ], [AtomXYZ], [AtomXYZ])
    :parameter mo_paths: List of paths to the MO in the HDF5
    :type      mo_paths: [str]
    :param hdf5_trans_mtx: path to the transformation matrix in the HDF5 file.
    :type hdf5_trans_mtx: str
    :param enumerate_from: Number from where to start enumerating the folders
    create for each point in the MD
    :type enumerate_from: int
    :param nHOMO: index of the HOMO orbital in the HDF5
    :param couplings_range: range of Molecular orbitals used to compute the
    coupling.

    :returns: path to the Coupling inside the HDF5
    """
    # Path inside the HDF5 where the overlaps are stored
    root = join(project_name, 'overlaps_{}'.format(i + enumerate_from))
    names_matrices = ['mtx_sji_t0', 'mtx_sij_t0']
    overlaps_paths_hdf5 = [join(root, name) for name in names_matrices]

    # Test if the overlap is store in the HDF5 calculate it
    with h5py.File(path_hdf5, 'r') as f5:
        is_done = all(path in f5 for path in overlaps_paths_hdf5)

    # If the Overlaps are not in the HDF5 file compute them
    if is_done:
        print(overlaps_paths_hdf5, " Overlaps are already in the HDF5")
    else:
        # Read the Molecular orbitals from the HDF5
        print("Computing: ", root)
        mos = tuple(map(lambda j:
                        retrieve_hdf5_data(path_hdf5,
                                           mo_paths[i + j][1]), range(2)))

        # Extract a subset of molecular orbitals to compute the coupling
        lowest, highest = compute_range_orbitals(mos[0], nHOMO, couplings_range)
        mos = tuple(map(lambda xs: xs[:, lowest: highest], mos))

        # Read the transformation matrix to convert from Cartesian to
        # Spherical coordinates
        if hdf5_trans_mtx is not None:
            trans_mtx = retrieve_hdf5_data(path_hdf5, hdf5_trans_mtx)

        # Partial application of the function computing the overlap
        overlaps = compute_overlaps_for_coupling(geometries, mos, dictCGFs,
                                                 trans_mtx)

        # Store the matrices in the HDF5 file
        with h5py.File(path_hdf5, 'r+') as f5:
            store = StoreasHDF5(f5, 'cp2k')
            for p, mtx in zip(overlaps_paths_hdf5, overlaps):
                store.funHDF5(p, mtx)

    return overlaps_paths_hdf5

def compute_range_orbitals(mtx: Matrix, nHOMO: int,
                           couplings_range: Tuple) -> Tuple:
    """
    Compute the lowest and highest index used to extract
    a subset of Columns from the MOs
    """
    # If the user does not define the number of HOMOs and LUMOs
    # assume that the first half of the read MO from the HDF5
    # are HOMOs and the last Half are LUMOs.
    _, nOrbitals = mtx.shape
    nHOMO = nHOMO if nHOMO is not None else nOrbitals // 2

    # If the couplings_range variable is not define I assume
    # that the number of LUMOs is equal to the HOMOs.
    if all(x is not None for x in [nHOMO, couplings_range]):
        lowest = nHOMO - couplings_range[0]
        highest = nHOMO + couplings_range[1]
    else:
        lowest = 0
        highest = nOrbitals

    return lowest, highest

def write_hamiltonians(path_hdf5: str, mo_paths: List, path_couplings: List,
                       nPoints: int, path_dir_results: str=None,
                       enumerate_from: int=0, nHOMO: int=None,
                       couplings_range: Tuple=None) -> None:
    """
    Write the real and imaginary components of the hamiltonian using both
    the orbitals energies and the derivative coupling accoring to:
    http://pubs.acs.org/doi/abs/10.1021/ct400641n
    **Units are: Rydbergs**.
    """
    def write_pyxaid_format(arr, fileName):
        np.savetxt(fileName, arr, fmt='%10.5e', delimiter='  ')

    ham_files = []
    for i in range(nPoints):
        j = i + enumerate_from
        path_coupling = path_couplings[i]
        css = retrieve_hdf5_data(path_hdf5, path_coupling)

        # Extract the energy values
        energies = retrieve_hdf5_data(path_hdf5, mo_paths[i][0])
        if all(x is not None for x in [nHOMO, couplings_range]):
            lowest = nHOMO - couplings_range[0]
            highest = nHOMO + couplings_range[1]
            energies = energies[lowest: highest]

        # FileNames
        file_ham_im = join(path_dir_results, 'Ham_{}_im'.format(j))
        file_ham_re = join(path_dir_results, 'Ham_{}_re'.format(j))

        # convert to Rydbergs
        ham_im = 2.0 * css
        ham_re = np.diag(2.0 * energies)

        write_pyxaid_format(ham_im, file_ham_im)
        write_pyxaid_format(ham_re, file_ham_re)
        ham_files.append((file_ham_im, file_ham_re))

    return ham_files
<commit_msg>fixed range in which the phases are computed<commit_after>__author__ = "Felipe Zapata"

# ================> Python Standard  and third-party <==========
from itertools import chain
from os.path import join

import h5py
import numpy as np
# ==================> Internal modules <==========
from nac.integrals import (calculateCoupling3Points,
                           compute_overlaps_for_coupling,
                           correct_phases)
from nac.common import (femtosec2au, retrieve_hdf5_data)
from qmworks.hdf5.quantumHDF5 import StoreasHDF5

# Types hint
from typing import (Dict, List, Tuple)

# Numpy type hints
Vector = np.ndarray
Matrix = np.ndarray

# ==============================> Schedule Tasks <=============================

def lazy_couplings(paths_overlaps: List, path_hdf5: str, project_name: str,
                   enumerate_from: int, dt: float) -> List:
    """
    :parameter dt: dynamic integration time
    :type      dt: Float (Femtoseconds)
    """
    # time in atomic units
    dt_au = dt * femtosec2au

    # Compute the dimension of the coupling matrix
    mtx_0 = retrieve_hdf5_data(path_hdf5, paths_overlaps[0][0])
    _, dim = mtx_0.shape

    # Read all the Overlaps
    concat_paths = chain(*paths_overlaps)
    overlaps = np.stack([retrieve_hdf5_data(path_hdf5, ps)
                         for ps in concat_paths])

    # Number of couplings to compute
    nCouplings = overlaps.shape[0] // 2 - 1

    # Compute all the phases
    mtx_phases = compute_phases(overlaps, nCouplings, dim)

    # Compute the couplings using the four matrices previously calculated
    # Together with the phases
    paths_couplings = []

    for i in range(nCouplings):
        # Path were the couplinp is store
        k = i + enumerate_from
        path = join(project_name, 'coupling_{}'.format(k))
        with h5py.File(path_hdf5, 'r+') as f5:
            is_done  = path in f5

        # Skip the computation if the coupling is already done
        if is_done:
            print("Coupling: ", path, " has already been calculated")
        else:
            # Tensor containing the overlaps
            j = 2 * i
            ps = overlaps[j: j + 4, :, :]
            # Correct the Phase of the Molecular orbitals
            fixed_phase_overlaps = correct_phases(
                ps, mtx_phases[i: i + 3, :], dim)

            # Compute the couplings with the phase corrected overlaps
            couplings = calculateCoupling3Points(dt_au, *fixed_phase_overlaps)

            with h5py.File(path_hdf5, 'r+') as f5:
                store = StoreasHDF5(f5, 'cp2k')
                store.funHDF5(path, couplings)

        paths_couplings.append(path)

    return paths_couplings

def compute_phases(overlaps: List, nCouplings: int, dim: int) -> Matrix:
    """
    Compute the phase of the state_i at time t + dt, using the following
    equation:
    phase_i(t+dt) = Sii(t) * phase_i(t)
    """
    # initial references
    references = np.ones(dim)

    # Matrix containing the phases
    mtx_phases = np.empty((nCouplings + 2, dim))
    mtx_phases[0, :] = references

    # Compute the phases at times t + dt using the phases at time t
    for i in range(nCouplings + 1):
        Sji_t = overlaps[2 * i, :, :].reshape(dim, dim)
        phases = np.sign(np.diag(Sji_t)) * references
        mtx_phases[i + 1] = phases
        references = phases

    return mtx_phases

def lazy_overlaps(i: int, project_name: str, path_hdf5: str, dictCGFs: Dict,
                  geometries: Tuple, mo_paths: List, hdf5_trans_mtx: str=None,
                  enumerate_from: int=0, nHOMO: int=None,
                  couplings_range: Tuple=None) -> str:
    """
    Calculate the 4 overlap matrix used to compute the subsequent couplings.
    The overlap matrices are computed using 3 consecutive set of MOs and
    3 consecutive geometries( in atomic units), from a molecular dynamics.

    :param i: nth coupling calculation
    :type i: int
    :param project_name: Name of the project to be executed.
    :type project_name: str
    :paramter dictCGFS: Dictionary from Atomic Label to basis set
    :type     dictCGFS: Dict String [CGF],
              CGF = ([Primitives], AngularMomentum),
              Primitive = (Coefficient, Exponent)
    :parameter geometries: molecular geometries stored as list of
                           namedtuples.
    :type      geometries: ([AtomXYZ], [AtomXYZ], [AtomXYZ])
    :parameter mo_paths: List of paths to the MO in the HDF5
    :type      mo_paths: [str]
    :param hdf5_trans_mtx: path to the transformation matrix in the HDF5 file.
    :type hdf5_trans_mtx: str
    :param enumerate_from: Number from where to start enumerating the folders
    create for each point in the MD
    :type enumerate_from: int
    :param nHOMO: index of the HOMO orbital in the HDF5
    :param couplings_range: range of Molecular orbitals used to compute the
    coupling.

    :returns: path to the Coupling inside the HDF5
    """
    # Path inside the HDF5 where the overlaps are stored
    root = join(project_name, 'overlaps_{}'.format(i + enumerate_from))
    names_matrices = ['mtx_sji_t0', 'mtx_sij_t0']
    overlaps_paths_hdf5 = [join(root, name) for name in names_matrices]

    # Test if the overlap is store in the HDF5 calculate it
    with h5py.File(path_hdf5, 'r') as f5:
        is_done = all(path in f5 for path in overlaps_paths_hdf5)

    # If the Overlaps are not in the HDF5 file compute them
    if is_done:
        print(overlaps_paths_hdf5, " Overlaps are already in the HDF5")
    else:
        # Read the Molecular orbitals from the HDF5
        print("Computing: ", root)
        mos = tuple(map(lambda j:
                        retrieve_hdf5_data(path_hdf5,
                                           mo_paths[i + j][1]), range(2)))

        # Extract a subset of molecular orbitals to compute the coupling
        lowest, highest = compute_range_orbitals(mos[0], nHOMO, couplings_range)
        mos = tuple(map(lambda xs: xs[:, lowest: highest], mos))

        # Read the transformation matrix to convert from Cartesian to
        # Spherical coordinates
        if hdf5_trans_mtx is not None:
            trans_mtx = retrieve_hdf5_data(path_hdf5, hdf5_trans_mtx)

        # Partial application of the function computing the overlap
        overlaps = compute_overlaps_for_coupling(geometries, mos, dictCGFs,
                                                 trans_mtx)

        # Store the matrices in the HDF5 file
        with h5py.File(path_hdf5, 'r+') as f5:
            store = StoreasHDF5(f5, 'cp2k')
            for p, mtx in zip(overlaps_paths_hdf5, overlaps):
                store.funHDF5(p, mtx)

    return overlaps_paths_hdf5

def compute_range_orbitals(mtx: Matrix, nHOMO: int,
                           couplings_range: Tuple) -> Tuple:
    """
    Compute the lowest and highest index used to extract
    a subset of Columns from the MOs
    """
    # If the user does not define the number of HOMOs and LUMOs
    # assume that the first half of the read MO from the HDF5
    # are HOMOs and the last Half are LUMOs.
    _, nOrbitals = mtx.shape
    nHOMO = nHOMO if nHOMO is not None else nOrbitals // 2

    # If the couplings_range variable is not define I assume
    # that the number of LUMOs is equal to the HOMOs.
    if all(x is not None for x in [nHOMO, couplings_range]):
        lowest = nHOMO - couplings_range[0]
        highest = nHOMO + couplings_range[1]
    else:
        lowest = 0
        highest = nOrbitals

    return lowest, highest

def write_hamiltonians(path_hdf5: str, mo_paths: List, path_couplings: List,
                       nPoints: int, path_dir_results: str=None,
                       enumerate_from: int=0, nHOMO: int=None,
                       couplings_range: Tuple=None) -> None:
    """
    Write the real and imaginary components of the hamiltonian using both
    the orbitals energies and the derivative coupling accoring to:
    http://pubs.acs.org/doi/abs/10.1021/ct400641n
    **Units are: Rydbergs**.
    """
    def write_pyxaid_format(arr, fileName):
        np.savetxt(fileName, arr, fmt='%10.5e', delimiter='  ')

    ham_files = []
    for i in range(nPoints):
        j = i + enumerate_from
        path_coupling = path_couplings[i]
        css = retrieve_hdf5_data(path_hdf5, path_coupling)

        # Extract the energy values
        energies = retrieve_hdf5_data(path_hdf5, mo_paths[i][0])
        if all(x is not None for x in [nHOMO, couplings_range]):
            lowest = nHOMO - couplings_range[0]
            highest = nHOMO + couplings_range[1]
            energies = energies[lowest: highest]

        # FileNames
        file_ham_im = join(path_dir_results, 'Ham_{}_im'.format(j))
        file_ham_re = join(path_dir_results, 'Ham_{}_re'.format(j))

        # convert to Rydbergs
        ham_im = 2.0 * css
        ham_re = np.diag(2.0 * energies)

        write_pyxaid_format(ham_im, file_ham_im)
        write_pyxaid_format(ham_re, file_ham_re)
        ham_files.append((file_ham_im, file_ham_re))

    return ham_files
<|endoftext|>


<commit_before>"""
The basic functions to create usable rockstar scripts to run rockstar on all tipsy
outputs in a directory.

snaps()
  create a snaps.txt file containing all the outputs in the directory
  one number per line

cfg(ncorespernode=32, nnodes=1, ServerInterface='ipogif0', massdef='200c', massdef2=None)
  create a rockstar.submit.cfg file to submit to the rockstar program
  nocorespernode: the number of cores on each node of the computer
  ServerInterface: 'ib0' for most computers except bluewaters which uses 'ipogif0'
  massdef: how to define the virial mass of a halo
  massdef2: if you want another mass defined

mainsubmissionscript(walltime = '24:00:00', email = 'l.sonofanders@gmail.com', machine='stampede', nnodes=1, ncorespernode=32, queue='largemem', rockstardir='/home1/02575/lmanders/code/Rockstar-Galaxies/')
  create the submission script to run rockstar.
  walltime: total amount of wallclock time alloted to the analysis
  email: the email you want messages from the queue sent to
  machine: the machine you will run the process on;
           right now either 'stampede', 'pleiades' or 'bluewaters'
  nnodes: the number of nodes to run on
  ncorespernode: the number of cores per node on the machine
  queue: the queue to submit to
  rockstardir: the directory your rockstar copy lies in

postsubmissionscript(email = 'l.sonofanders@gmail.com', machine = 'stampede', queue = 'largemem'\
, rockstardir='/home1/02575/lmanders/code/Rockstar-Galaxies', ncorespernode=32, walltime='24:00:00')
  create the submission script to do post processing on rockstar and turn its natural outputs
  into .grp and .stat files
  email: the email you want messages from the queue sent to
  machine: the machine you will run the process on;
           right now either 'stampede', 'pleiades', or 'bluewaters'
  queue: the queue to submit to
  rockstardir: the directory your rockstar copy lies in
  ncorespernode: the number of cores per node on the machine
  walltime: total amount of wallclock time alloted to the analysis
"""

import glob
import numpy as np
import struct
from .. import nptipsyreader
import os
from .. import findtipsy

def snaps():
    files = glob.glob('*.iord')
    files.sort()
    snaps = []
    f = open('snaps.txt', 'w')
    for i in files: f.write(i.split('.')[-2] + '\n')
    f.close()

def cfg(ncorespernode=32, nnodes=1, ServerInterface='ipogif0', massdef='200c', massdef2=None):
    filename = ('.').join(glob.glob('*.iord')[0].split('.')[:-2])
    tipsyfile = ('.').join(glob.glob('*.iord')[0].split('.')[:-1])
    tipsy = nptipsyreader.Tipsy(tipsyfile)
    tipsy._read_param()
    dmsol = np.float(tipsy.paramfile['dMsolUnit'])
    dkpc = np.float(tipsy.paramfile['dKpcUnit'])
    f = open(tipsyfile)
    (t, nbodies, ndim, nsph, ndark, nstar) = struct.unpack('>diiiii', f.read(28))
    f.seek(struct.calcsize('>diiiiii')+struct.calcsize('12f4')*nsph)
    (mass, x, y, z, vx, vy, vz, eps, phi) = struct.unpack('>9f4', f.read(36))
    darkmass = mass*dmsol*tipsy.h
    #f.seek(struct.calcsize('>diiiiii')+struct.calcsize('12f4')*nsph+struct.calcsize('9f4')*ndark)
    #(mass,x,y,z,vx,vy,vz,metals,tform,eps,phi) = struct.unpack('>11f4', f.read(44))
    force = eps*dkpc*tipsy.h/1e3
    f.close()

    f = open('rockstar.submit.cfg', 'w')
    f.write('OVERLAP_LENGTH=0.5 \n')
    f.write('PARALLEL_IO=1 \n')
    f.write('NUM_WRITERS=' + str(ncorespernode*nnodes)+' \n')
    f.write('NUM_BLOCKS=1 \n')
    f.write('FILENAME=' + filename +'.<snap> \n')
    f.write('#NUM_SNAPS=1 \n')
    f.write('SNAPSHOT_NAMES=snaps.txt  #One snapshot name per line \n')
    f.write('FORK_READERS_FROM_WRITERS=1 \n')
    f.write('FORK_PROCESSORS_PER_MACHINE=' + str(ncorespernode) + '\n')
    f.write('FILE_FORMAT = "TIPSY" # or "ART" or "ASCII" \n')
    f.write('PARTICLE_MASS ='+ str(darkmass) +'  #Msol/h \n')
    f.write('FORCE_RES = ' + str(force) + '  #Mpc/h \n')
    f.write('# You should specify cosmology parameters only for ASCII formats \n# For GADGET2 and ART, these parameters will be replaced with values from the \n# particle data file \n')
    f.write('h0 =' + str(tipsy.h) + '\n')
    f.write('Ol =' + tipsy.paramfile['dLambda'] + '\n')
    f.write('Om =' + tipsy.paramfile['dOmega0'] + '\n')
    f.write('# For GADGET2, you may need to specify conversion parameters. \n# Rockstar"s internal units are Mpc/h (lengths) and Msun/h (masses) \n')
    f.write('TIPSY_MASS_CONVERSION   = ' + str(dmsol*tipsy.h) + '\n')
    f.write('TIPSY_LENGTH_CONVERSION = '  + str(dkpc*tipsy.h/1e3) + '\n')
    f.write('TIPSY_VELOCITY_CONVERSION = ' + str(tipsy.velunit) + '\n')
    f.write('BOX_SIZE = ' + str(dkpc*tipsy.h/1e3) + '\n')
    f.write('#For ascii files, the file format is assumed to be: \n# X Y Z VX VY VZ ID \n')
    f.write('#For non-periodic boundary conditions only: \n#PERIODIC=0 \n')
    f.write('FULL_PARTICLE_CHUNKS = ' + str(ncorespernode*nnodes)  + '  # Should be the same as NUM_WRITERS above to save all particles \n')
    f.write('PARALLEL_IO_SERVER_INTERFACE = "' + ServerInterface + '"\n')
    f.write('MASS_DEFINITION  = "' + massdef + '"\n')
    if massdef2: f.write('MASS_DEFINITION2 = "' + massdef2+ '"\n')

def mainsubmissionscript(walltime = '24:00:00', email = 'l.sonofanders@gmail.com', machine='stampede', nnodes=1, ncorespernode=32, queue='largemem', rockstardir='/home1/02575/lmanders/code/Rockstar-Galaxies/'):
    sbatchname = findtipsy.find()[0].split('.')[0]
    if (machine == 'pleiades') or (machine == 'bluewaters'):
        filename = 'rockstar.qsub'
        f = open(filename, 'w')

        f.write('#!/bin/bash \n')
        f.write('#PBS -N ' + sbatchname + ' \n')
        if machine == 'pleiades': f.write('#PBS -lselect=' + str(nnodes) + ':ncpus='+str(ncorespernode)+':mpiprocs=1'+ '\n')
        if machine == 'bluewaters':f.write('#PBS -lnodes=' + str(nnodes) + ':ppn='+str(ncorespernode)+'\n')
        f.write('#PBS -m be \n')
        f.write('#PBS -M ' + email +' \n')
        f.write('#PBS -q ' + queue + ' \n')
        f.write('#PBS -l walltime=' + walltime + ' \n')

        f.write('cd $PBS_O_WORKDIR \n')
        f.write('rm auto-rockstar.cfg \n')
        f.write('ulimit -c unlimited \n')

        if machine=='bluewaters': prefix = 'aprun -n ' + str(nnodes) + ' -N 1 '
        else: prefix = ''

        f.write(rockstardir + 'rockstar-galaxies -c rockstar.submit.cfg & \n')
        f.write("perl -e 'sleep 1 while (!(-e "+'"' + "auto-rockstar.cfg"+'"' + "))' \n")
        f.write(prefix + rockstardir + 'rockstar-galaxies -c auto-rockstar.cfg \n')

    if machine == 'stampede':
        filename = 'rockstar.sbatch'
        f = open(filename, 'w')

        f.write('#!/bin/bash \n')
        f.write('#SBATCH -J ' + sbatchname + ' \n')
        f.write('#SBATCH -n ' + str(nnodes) + ' \n')
        f.write('#SBATCH -N ' + str(nnodes) + ' \n')
        f.write('#SBATCH -o ' + sbatchname + '.rockstar.o%j \n')
        f.write('#SBATCH -p ' + queue + ' \n')
        f.write('#SBATCH -t 24:00:00 \n')
        f.write('#SBATCH --mail-user=l.sonofanders@gmail.com \n')
        f.write('#SBATCH --mail-type=ALL \n')
        f.write('#SBATCH -A TG-MCA94P018 \n')
        f.write('## -n: total tasks \n')
        f.write('## -N: nodes \n')

        f.write('cd $SLURM_SUBMIT_DIR \n')
        f.write('rm auto-rockstar.cfg \n')
        f.write('ulimit -c unlimited \n')

        f.write(rockstardir + 'rockstar-galaxies -c rockstar.submit.cfg & \n')
        f.write("perl -e 'sleep 1 while (!(-e "+'"' + "auto-rockstar.cfg"+'"' + "))' \n")
        f.write('ibrun ' + rockstardir + 'rockstar-galaxies -c auto-rockstar.cfg \n')

def postsubmissionscript(email = 'l.sonofanders@gmail.com', machine = 'stampede', queue = 'largemem', rockstardir='/home1/02575/lmanders/code/Rockstar-Galaxies', ncorespernode=32, walltime='24:00:00', nnodes=1):
    tipsyfile = findtipsy.find()[0]
    iordfilepre = ('.').join(tipsyfile.split('.')[:-1])

    sim = nptipsyreader.Tipsy(tipsyfile)
    sim._read_param()
    boxsize = str(float(sim.paramfile['dKpcUnit'])/1000.*sim.h)

    outfiles = glob.glob('out_*.list')

    snapsf = open('snaps.txt')
    snaps = [line.split('\n')[0] for line in snapsf]
    snapsf.close()

    genstatexecline = []

    if machine == 'stampede':
        f = open('rockstar.post.sbatch', 'w')

        f.write('#!/bin/bash \n')
        f.write('#SBATCH -J' + iordfilepre +' \n')
        f.write('#SBATCH -n 1 \n')
        f.write('#SBATCH -N 1 \n')
        f.write('#SBATCH -o ' + iordfilepre + '.rockstar.o%j \n')
        f.write('#SBATCH -p ' + queue + ' \n')
        f.write('#SBATCH -t ' + walltime + ' \n')
        f.write('#SBATCH --mail-suers=' + email + '\n')
        f.write('#SBATCH --mail-type=ALL \n')
        f.write('#SBATCH -A TG-MCA94P018 \n')
        f.write('cd $SLURM_SUBMIT_DIR \n')
        f.write('rm auto-rockstar.cfg \n')
        f.write('ulimit -c unlimited \n')
    if (machine == 'pleiades') or (machine == 'bluewaters'):
        f = open('rockstar.post.qsub', 'w')

        f.write('#!/bin/bash \n')
        f.write('#PBS -N cosmo6.rockstar \n')
        if machine == 'pleiades':f.write('#PBS -lselect=1:ncpus='+str(ncorespernode)+':mpiprocs=1 \n')
        if machine == 'bluewaters':f.write('#PBS -lnodes=1:ppn='+str(ncorespernode)+' \n')
        f.write('#PBS -q ' + queue + ' \n')
        f.write('#PBS -l walltime=24:00:00 \n')
        f.write('#PBS -m be \n')
        f.write('#PBS -M ' + email + '\n')
        f.write('cd $PBS_O_DIR \n')
        f.write('ulimit -c unlimited \n')

    for i in range(len(snaps)):
        parentexecline = (rockstardir + 'util/find_parents out_'+ str(i  ) + '.list ' + boxsize + ' > out_'+ snaps[i] + '.parents \n' )
        f.write(parentexecline)
        genstatexecline.append(rockstardir + 'examples/gen_grp_stats out_' + snaps[i] + '.parents ' + iordfilepre + '.' +snaps[i] + '.iord  halos_' + snaps[i] + '.*.particles \n')

    for i in range(len(genstatexecline)):
        f.write("perl -e 'sleep 1 while (!(-e " + '"' + "out_" + snaps[i] + ".parents"+'"'+"))'" + "\n")
        f.write(genstatexecline[i])

    f.close()
    #    os.system(parentexecline)
    #    os.system(genstatexecline)
<commit_msg>fixed workdir<commit_after>"""
The basic functions to create usable rockstar scripts to run rockstar on all tipsy
outputs in a directory.

snaps()
  create a snaps.txt file containing all the outputs in the directory
  one number per line

cfg(ncorespernode=32, nnodes=1, ServerInterface='ipogif0', massdef='200c', massdef2=None)
  create a rockstar.submit.cfg file to submit to the rockstar program
  nocorespernode: the number of cores on each node of the computer
  ServerInterface: 'ib0' for most computers except bluewaters which uses 'ipogif0'
  massdef: how to define the virial mass of a halo
  massdef2: if you want another mass defined

mainsubmissionscript(walltime = '24:00:00', email = 'l.sonofanders@gmail.com', machine='stampede', nnodes=1, ncorespernode=32, queue='largemem', rockstardir='/home1/02575/lmanders/code/Rockstar-Galaxies/')
  create the submission script to run rockstar.
  walltime: total amount of wallclock time alloted to the analysis
  email: the email you want messages from the queue sent to
  machine: the machine you will run the process on;
           right now either 'stampede', 'pleiades' or 'bluewaters'
  nnodes: the number of nodes to run on
  ncorespernode: the number of cores per node on the machine
  queue: the queue to submit to
  rockstardir: the directory your rockstar copy lies in

postsubmissionscript(email = 'l.sonofanders@gmail.com', machine = 'stampede', queue = 'largemem'\
, rockstardir='/home1/02575/lmanders/code/Rockstar-Galaxies', ncorespernode=32, walltime='24:00:00')
  create the submission script to do post processing on rockstar and turn its natural outputs
  into .grp and .stat files
  email: the email you want messages from the queue sent to
  machine: the machine you will run the process on;
           right now either 'stampede', 'pleiades', or 'bluewaters'
  queue: the queue to submit to
  rockstardir: the directory your rockstar copy lies in
  ncorespernode: the number of cores per node on the machine
  walltime: total amount of wallclock time alloted to the analysis
"""

import glob
import numpy as np
import struct
from .. import nptipsyreader
import os
from .. import findtipsy

def snaps():
    files = glob.glob('*.iord')
    files.sort()
    snaps = []
    f = open('snaps.txt', 'w')
    for i in files: f.write(i.split('.')[-2] + '\n')
    f.close()

def cfg(ncorespernode=32, nnodes=1, ServerInterface='ipogif0', massdef='200c', massdef2=None):
    filename = ('.').join(glob.glob('*.iord')[0].split('.')[:-2])
    tipsyfile = ('.').join(glob.glob('*.iord')[0].split('.')[:-1])
    tipsy = nptipsyreader.Tipsy(tipsyfile)
    tipsy._read_param()
    dmsol = np.float(tipsy.paramfile['dMsolUnit'])
    dkpc = np.float(tipsy.paramfile['dKpcUnit'])
    f = open(tipsyfile)
    (t, nbodies, ndim, nsph, ndark, nstar) = struct.unpack('>diiiii', f.read(28))
    f.seek(struct.calcsize('>diiiiii')+struct.calcsize('12f4')*nsph)
    (mass, x, y, z, vx, vy, vz, eps, phi) = struct.unpack('>9f4', f.read(36))
    darkmass = mass*dmsol*tipsy.h
    #f.seek(struct.calcsize('>diiiiii')+struct.calcsize('12f4')*nsph+struct.calcsize('9f4')*ndark)
    #(mass,x,y,z,vx,vy,vz,metals,tform,eps,phi) = struct.unpack('>11f4', f.read(44))
    force = eps*dkpc*tipsy.h/1e3
    f.close()

    f = open('rockstar.submit.cfg', 'w')
    f.write('OVERLAP_LENGTH=0.5 \n')
    f.write('PARALLEL_IO=1 \n')
    f.write('NUM_WRITERS=' + str(ncorespernode*nnodes)+' \n')
    f.write('NUM_BLOCKS=1 \n')
    f.write('FILENAME=' + filename +'.<snap> \n')
    f.write('#NUM_SNAPS=1 \n')
    f.write('SNAPSHOT_NAMES=snaps.txt  #One snapshot name per line \n')
    f.write('FORK_READERS_FROM_WRITERS=1 \n')
    f.write('FORK_PROCESSORS_PER_MACHINE=' + str(ncorespernode) + '\n')
    f.write('FILE_FORMAT = "TIPSY" # or "ART" or "ASCII" \n')
    f.write('PARTICLE_MASS ='+ str(darkmass) +'  #Msol/h \n')
    f.write('FORCE_RES = ' + str(force) + '  #Mpc/h \n')
    f.write('# You should specify cosmology parameters only for ASCII formats \n# For GADGET2 and ART, these parameters will be replaced with values from the \n# particle data file \n')
    f.write('h0 =' + str(tipsy.h) + '\n')
    f.write('Ol =' + tipsy.paramfile['dLambda'] + '\n')
    f.write('Om =' + tipsy.paramfile['dOmega0'] + '\n')
    f.write('# For GADGET2, you may need to specify conversion parameters. \n# Rockstar"s internal units are Mpc/h (lengths) and Msun/h (masses) \n')
    f.write('TIPSY_MASS_CONVERSION   = ' + str(dmsol*tipsy.h) + '\n')
    f.write('TIPSY_LENGTH_CONVERSION = '  + str(dkpc*tipsy.h/1e3) + '\n')
    f.write('TIPSY_VELOCITY_CONVERSION = ' + str(tipsy.velunit) + '\n')
    f.write('BOX_SIZE = ' + str(dkpc*tipsy.h/1e3) + '\n')
    f.write('#For ascii files, the file format is assumed to be: \n# X Y Z VX VY VZ ID \n')
    f.write('#For non-periodic boundary conditions only: \n#PERIODIC=0 \n')
    f.write('FULL_PARTICLE_CHUNKS = ' + str(ncorespernode*nnodes)  + '  # Should be the same as NUM_WRITERS above to save all particles \n')
    f.write('PARALLEL_IO_SERVER_INTERFACE = "' + ServerInterface + '"\n')
    f.write('MASS_DEFINITION  = "' + massdef + '"\n')
    if massdef2: f.write('MASS_DEFINITION2 = "' + massdef2+ '"\n')

def mainsubmissionscript(walltime = '24:00:00', email = 'l.sonofanders@gmail.com', machine='stampede', nnodes=1, ncorespernode=32, queue='largemem', rockstardir='/home1/02575/lmanders/code/Rockstar-Galaxies/'):
    sbatchname = findtipsy.find()[0].split('.')[0]
    if (machine == 'pleiades') or (machine == 'bluewaters'):
        filename = 'rockstar.qsub'
        f = open(filename, 'w')

        f.write('#!/bin/bash \n')
        f.write('#PBS -N ' + sbatchname + ' \n')
        if machine == 'pleiades': f.write('#PBS -lselect=' + str(nnodes) + ':ncpus='+str(ncorespernode)+':mpiprocs=1'+ '\n')
        if machine == 'bluewaters':f.write('#PBS -lnodes=' + str(nnodes) + ':ppn='+str(ncorespernode)+'\n')
        f.write('#PBS -m be \n')
        f.write('#PBS -M ' + email +' \n')
        f.write('#PBS -q ' + queue + ' \n')
        f.write('#PBS -l walltime=' + walltime + ' \n')

        f.write('cd $PBS_O_WORKDIR \n')
        f.write('rm auto-rockstar.cfg \n')
        f.write('ulimit -c unlimited \n')

        if machine=='bluewaters': prefix = 'aprun -n ' + str(nnodes) + ' -N 1 '
        else: prefix = ''

        f.write(rockstardir + 'rockstar-galaxies -c rockstar.submit.cfg & \n')
        f.write("perl -e 'sleep 1 while (!(-e "+'"' + "auto-rockstar.cfg"+'"' + "))' \n")
        f.write(prefix + rockstardir + 'rockstar-galaxies -c auto-rockstar.cfg \n')

    if machine == 'stampede':
        filename = 'rockstar.sbatch'
        f = open(filename, 'w')

        f.write('#!/bin/bash \n')
        f.write('#SBATCH -J ' + sbatchname + ' \n')
        f.write('#SBATCH -n ' + str(nnodes) + ' \n')
        f.write('#SBATCH -N ' + str(nnodes) + ' \n')
        f.write('#SBATCH -o ' + sbatchname + '.rockstar.o%j \n')
        f.write('#SBATCH -p ' + queue + ' \n')
        f.write('#SBATCH -t 24:00:00 \n')
        f.write('#SBATCH --mail-user=l.sonofanders@gmail.com \n')
        f.write('#SBATCH --mail-type=ALL \n')
        f.write('#SBATCH -A TG-MCA94P018 \n')
        f.write('## -n: total tasks \n')
        f.write('## -N: nodes \n')

        f.write('cd $SLURM_SUBMIT_DIR \n')
        f.write('rm auto-rockstar.cfg \n')
        f.write('ulimit -c unlimited \n')

        f.write(rockstardir + 'rockstar-galaxies -c rockstar.submit.cfg & \n')
        f.write("perl -e 'sleep 1 while (!(-e "+'"' + "auto-rockstar.cfg"+'"' + "))' \n")
        f.write('ibrun ' + rockstardir + 'rockstar-galaxies -c auto-rockstar.cfg \n')

def postsubmissionscript(email = 'l.sonofanders@gmail.com', machine = 'stampede', queue = 'largemem', rockstardir='/home1/02575/lmanders/code/Rockstar-Galaxies', ncorespernode=32, walltime='24:00:00', nnodes=1):
    tipsyfile = findtipsy.find()[0]
    iordfilepre = ('.').join(tipsyfile.split('.')[:-1])

    sim = nptipsyreader.Tipsy(tipsyfile)
    sim._read_param()
    boxsize = str(float(sim.paramfile['dKpcUnit'])/1000.*sim.h)

    outfiles = glob.glob('out_*.list')

    snapsf = open('snaps.txt')
    snaps = [line.split('\n')[0] for line in snapsf]
    snapsf.close()

    genstatexecline = []

    if machine == 'stampede':
        f = open('rockstar.post.sbatch', 'w')

        f.write('#!/bin/bash \n')
        f.write('#SBATCH -J' + iordfilepre +' \n')
        f.write('#SBATCH -n 1 \n')
        f.write('#SBATCH -N 1 \n')
        f.write('#SBATCH -o ' + iordfilepre + '.rockstar.o%j \n')
        f.write('#SBATCH -p ' + queue + ' \n')
        f.write('#SBATCH -t ' + walltime + ' \n')
        f.write('#SBATCH --mail-suers=' + email + '\n')
        f.write('#SBATCH --mail-type=ALL \n')
        f.write('#SBATCH -A TG-MCA94P018 \n')
        f.write('cd $SLURM_SUBMIT_DIR \n')
        f.write('rm auto-rockstar.cfg \n')
        f.write('ulimit -c unlimited \n')
    if (machine == 'pleiades') or (machine == 'bluewaters'):
        f = open('rockstar.post.qsub', 'w')

        f.write('#!/bin/bash \n')
        f.write('#PBS -N cosmo6.rockstar \n')
        if machine == 'pleiades':f.write('#PBS -lselect=1:ncpus='+str(ncorespernode)+':mpiprocs=1 \n')
        if machine == 'bluewaters':f.write('#PBS -lnodes=1:ppn='+str(ncorespernode)+' \n')
        f.write('#PBS -q ' + queue + ' \n')
        f.write('#PBS -l walltime=24:00:00 \n')
        f.write('#PBS -m be \n')
        f.write('#PBS -M ' + email + '\n')
        f.write('cd $PBS_O_WORKDIR \n')
        f.write('ulimit -c unlimited \n')

    for i in range(len(snaps)):
        parentexecline = (rockstardir + 'util/find_parents out_'+ str(i  ) + '.list ' + boxsize + ' > out_'+ snaps[i] + '.parents \n' )
        f.write(parentexecline)
        genstatexecline.append(rockstardir + 'examples/gen_grp_stats out_' + snaps[i] + '.parents ' + iordfilepre + '.' +snaps[i] + '.iord  halos_' + snaps[i] + '.*.particles \n')

    for i in range(len(snaps)):
        #f.write("perl -e 'sleep 1 while (!(-e " + '"' + "out_" + snaps[i] + ".parents"+'"'+"))'" + "\n")
        f.write(genstatexecline[i])
    for i in range(len(snaps)):
        f.write("perl -e 'sleep 1 while (!(-e " + '"' + "out_" + str(i) + ".grp"+'"'+"))'" + "\n")
        f.write("mv out_" + str(i) + ".grp "  + iordfilepre+'.'+snaps[i]+".rockstar.grp \n")
        f.write("perl -e 'sleep 1 while (!(-e " + '"' + "out_" + str(i) + ".stat"+'"'+"))'" + "\n")
        f.write("mv out_" + str(i) + ".stat " + iordfilepre+'.'+snaps[i]+".rockstar.stat \n")
    f.close()
    #    os.system(parentexecline)
    #    os.system(genstatexecline)
<|endoftext|>


<commit_before>from __future__ import absolute_import, division
from dials.array_family import flex # import dependency

class Test(object):

  def __init__(self):
    from os.path import join
    import libtbx.load_env
    try:
      dials_regression = libtbx.env.dist_path('dials_regression')
    except KeyError, e:
      print 'FAIL: dials_regression not configured'
      exit(0)

    self.path = join(
      dials_regression,
      "refinement_test_data",
      "multi_sweep_one_sample",
      "glucose_isomerase",
      "SWEEP1",
      "index",
      "sv_refined_experiments.json")

  def run(self):
    from libtbx import easy_run

    # Call dials.integrate
    easy_run.fully_buffered([
      'dials.plot_scan_varying_crystal',
      self.path
    ]).raise_if_errors()

    print 'OK'

if __name__ == '__main__':
  from dials.test import cd_auto
  with cd_auto(__file__):
    test = Test()
    test.run()
<commit_msg>correct comment<commit_after>from __future__ import absolute_import, division
from dials.array_family import flex # import dependency

class Test(object):

  def __init__(self):
    from os.path import join
    import libtbx.load_env
    try:
      dials_regression = libtbx.env.dist_path('dials_regression')
    except KeyError, e:
      print 'FAIL: dials_regression not configured'
      exit(0)

    self.path = join(
      dials_regression,
      "refinement_test_data",
      "multi_sweep_one_sample",
      "glucose_isomerase",
      "SWEEP1",
      "index",
      "sv_refined_experiments.json")

  def run(self):
    from libtbx import easy_run

    # Call dials.plot_scan_varying_crystal
    easy_run.fully_buffered([
      'dials.plot_scan_varying_crystal',
      self.path
    ]).raise_if_errors()

    print 'OK'

if __name__ == '__main__':
  from dials.test import cd_auto
  with cd_auto(__file__):
    test = Test()
    test.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2014 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# graphupload_prog.py
# Created by Disa Mhembere on 2015-02-27.
# Email: disa@jhu.edu
# Copyright (c) 2015. All rights reserved.

import os
from time import strftime, localtime

from django.http import HttpResponse
from django.conf import settings

from pipeline.utils.util import makeDirIfNone, writeBodyToDisk, saveFileToDisk
from pipeline.utils.util import sendJobBeginEmail, check_email
from pipeline.procs.inv_compute import invariant_compute
from pipeline.utils.zipper import unzip
from pipeline.procs.run_invariants import run_invariants
from pipeline.tasks import task_invariant_compute
from pipeline.utils.util import get_script_prefix

def graph_load_inv_prog(request, webargs):
  if request.method == 'POST' and webargs:

    split_webargs = webargs.split("/")
    to_email = split_webargs[0]
    if (not check_email(to_email)):
      return HttpResponse("ERROR: Incorrect email address format")

    try:
      in_graph_format = split_webargs[1]
      if in_graph_format not in ("graphml", "ncol", "edgelist", "lgl", "pajek", "graphdb", "numpy", "mat"):
        return HttpResponse("ERROR: Unknown graph input format")
      invariants = split_webargs[2:]
      if not invariants:
        return HttpResponse("ERROR: No invariants to compute provided")
    except:
      return HttpResponse("ERROR: Error with input graph format OR invariants chosen")

    data_dir = os.path.join(settings.MEDIA_ROOT, 'tmp', strftime("UploadGraph%a%d%b%Y_%H.%M.%S/", localtime()))
    makeDirIfNone([data_dir])
    uploadedZip = writeBodyToDisk(request.body, data_dir)[0] # Not necessarily a zip

    try: # Assume its a zip first
      unzip(uploadedZip, data_dir) # Unzip the zip
      os.remove(uploadedZip) # Delete the zip
    except:
      print "Non-zip file uploaded ..."

    graph_invariants_loc = os.path.join(data_dir, 'graphInvariants')
    makeDirIfNone([graph_invariants_loc])

    task_invariant_compute.delay(invariants, uploadedZip, graph_invariants_loc,
        data_dir, in_graph_format, to_email)

    sendJobBeginEmail(to_email, invariants)

    return HttpResponse("Successful job submission, please " \
                          "await reception & completion emails at {0}".format(to_email))
  else:
    return HttpResponse("There was an error! If you believe it " \
                          "is on our end please email: {0}".format(settings.DEFAULT_FROM_EMAIL))
<commit_msg>Added some more error handling for a graphuplaod<commit_after>#!/usr/bin/env python

# Copyright 2014 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# graphupload_prog.py
# Created by Disa Mhembere on 2015-02-27.
# Email: disa@jhu.edu
# Copyright (c) 2015. All rights reserved.

import os
from time import strftime, localtime

from django.http import HttpResponse
from django.conf import settings

from pipeline.utils.util import makeDirIfNone, writeBodyToDisk, saveFileToDisk
from pipeline.utils.util import sendJobBeginEmail, check_email
from pipeline.procs.inv_compute import invariant_compute
from pipeline.utils.zipper import unzip
from pipeline.procs.run_invariants import run_invariants
from pipeline.tasks import task_invariant_compute
from pipeline.utils.util import get_script_prefix

def graph_load_inv_prog(request, webargs):
  if request.method == 'POST' and webargs:

    split_webargs = webargs.split("/")
    to_email = split_webargs[0]
    if (not check_email(to_email)):
      return HttpResponse("ERROR: Incorrect email address format")

    try:
      in_graph_format = split_webargs[1]
      if in_graph_format not in ("graphml", "ncol", "edgelist", "lgl", "pajek", "graphdb", "numpy", "mat"):
        return HttpResponse("ERROR: Unknown graph input format")
      invariants = split_webargs[2:]
      for inv in invariants:
        if inv not in settings.VALID_FILE_TYPES:
          return HttpResponse("ERROR: Invariant '{0}' unknown!".format(inv))

      if not invariants:
        return HttpResponse("ERROR: No invariants to compute provided")
    except:
      return HttpResponse("ERROR: Opaque Error with input graph format OR invariants chosen")

    data_dir = os.path.join(settings.MEDIA_ROOT, 'tmp', strftime("UploadGraph%a%d%b%Y_%H.%M.%S/", localtime()))
    makeDirIfNone([data_dir])
    uploadedZip = writeBodyToDisk(request.body, data_dir)[0] # Not necessarily a zip

    try: # Assume its a zip first
      unzip(uploadedZip, data_dir) # Unzip the zip
      os.remove(uploadedZip) # Delete the zip
    except:
      print "Non-zip file uploaded ..."

    graph_invariants_loc = os.path.join(data_dir, 'graphInvariants')
    makeDirIfNone([graph_invariants_loc])

    task_invariant_compute.delay(invariants, uploadedZip, graph_invariants_loc,
        data_dir, in_graph_format, to_email)

    sendJobBeginEmail(to_email, invariants)

    return HttpResponse("Successful job submission, please " \
                          "await reception & completion emails at {0}".format(to_email))
  else:
    return HttpResponse("There was an error! If you believe it " \
                          "is on our end please email: {0}".format(settings.DEFAULT_FROM_EMAIL))
<|endoftext|>


<commit_before>from KalutServer.Exceptions import *
import KalutServer.conf as myconf
from KalutServer.Room import Room
from random import randint
import threading
import time
import Queue
import select
import socket
import sys
import json
import traceback
import ssl
rooms = dict()
srvsoc = socket.socket()

def soc_router():
    global srvsoc
    srvsoc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    srvsoc.bind((myconf.host, myconf.sock_port))
    srvsoc.listen(5)
    while 1:
        try:
            sslcltsock, addr = srvsoc.accept()
            # sslcltsock = ssl.wrap_socket(cltsock, myconf.keyfile, myconf.certfile, server_side=True)
            data = sslcltsock.recv(1024)
        except:
            print 'Server socket closed.'
            break
        try:
            if data:
                req = json.loads(data)
                rint 'Client with name {0} connected, client...'.format(req['name'])
                if int(req['pin']) in rooms.keys() and req['name']:
                    if rooms[req['pin']].status in ('WAITING_FOR_PLAYERS', 'WAITING_FOR_LAST_PLAYER'):
                        sslcltsock.send(json.dumps({'status' : 'ok', 'UID' : str(rooms[req['pin']].game_uid)}))
                        rooms[req['pin']].add_client(sslcltsock, req['name'])
                        if not rooms[req['pin']].running:
                            t = threading.Thread(target=start_room, args=(req['pin'],))
                            t.start()
                    else:
                        sslcltsock.send(json.dumps({'status' : "Room isn't waiting for players."}))
                        sslcltsock.close()
                else:
                    sslcltsock.send(json.dumps({'status' : 'invalid pin.'}))
                    sslcltsock.close()
            else:
                sslcltsock.close()
        except:
            sslcltsock.close()
            print traceback.format_exc()
            break

def start_room(pin):
    room = rooms[pin]
    room.running = True
    print 'starting room with pin of ' + str(pin)
    room.mainloop()

def close_srvsoc():
    global srvsoc
    srvsoc.close()

def are_there_rooms_running():
    '''Returns true of there are no running rooms.'''
    return len(rooms.keys())==0

def start_room_async(pin):
    t = threading.Thread(target=start_room, args=(pin,))
    t.start()

def create_room(uid, pin=0):
    while pin in rooms.keys() or pin==0:
        pin=randint(1000,10000)
    rooms[pin]=Room.Room(pin, uid)
    print 'created room with pin of' + str(pin)
    return pin

def close_room(pin):
    if pin in rooms.keys():
        del rooms[pin]

def close_all_rooms():
    global srvsoc
    for pin, room in rooms.items():
        room.close()
    s = socket.socket()
    s.connect((myconf.host, myconf.sock_port))
    srvsoc.close()
    s.close()
<commit_msg>fix syntax err - working final<commit_after>from KalutServer.Exceptions import *
import KalutServer.conf as myconf
from KalutServer.Room import Room
from random import randint
import threading
import time
import Queue
import select
import socket
import sys
import json
import traceback
import ssl
rooms = dict()
srvsoc = socket.socket()

def soc_router():
    global srvsoc
    srvsoc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    srvsoc.bind((myconf.host, myconf.sock_port))
    srvsoc.listen(5)
    while 1:
        try:
            sslcltsock, addr = srvsoc.accept()
            # sslcltsock = ssl.wrap_socket(cltsock, myconf.keyfile, myconf.certfile, server_side=True)
            data = sslcltsock.recv(1024)
        except:
            print 'Server socket closed.'
            break
        try:
            if data:
                req = json.loads(data)
                print 'Client with name {0} connected, client...'.format(req['name'])
                if int(req['pin']) in rooms.keys() and req['name']:
                    if rooms[req['pin']].status in ('WAITING_FOR_PLAYERS', 'WAITING_FOR_LAST_PLAYER'):
                        sslcltsock.send(json.dumps({'status' : 'ok', 'UID' : str(rooms[req['pin']].game_uid)}))
                        rooms[req['pin']].add_client(sslcltsock, req['name'])
                        if not rooms[req['pin']].running:
                            t = threading.Thread(target=start_room, args=(req['pin'],))
                            t.start()
                    else:
                        sslcltsock.send(json.dumps({'status' : "Room isn't waiting for players."}))
                        sslcltsock.close()
                else:
                    sslcltsock.send(json.dumps({'status' : 'invalid pin.'}))
                    sslcltsock.close()
            else:
                sslcltsock.close()
        except:
            sslcltsock.close()
            print traceback.format_exc()
            break

def start_room(pin):
    room = rooms[pin]
    room.running = True
    print 'starting room with pin of ' + str(pin)
    room.mainloop()

def close_srvsoc():
    global srvsoc
    srvsoc.close()

def are_there_rooms_running():
    '''Returns true of there are no running rooms.'''
    return len(rooms.keys())==0

def start_room_async(pin):
    t = threading.Thread(target=start_room, args=(pin,))
    t.start()

def create_room(uid, pin=0):
    while pin in rooms.keys() or pin==0:
        pin=randint(1000,10000)
    rooms[pin]=Room.Room(pin, uid)
    print 'created room with pin of' + str(pin)
    return pin

def close_room(pin):
    if pin in rooms.keys():
        del rooms[pin]

def close_all_rooms():
    global srvsoc
    for pin, room in rooms.items():
        room.close()
    s = socket.socket()
    s.connect((myconf.host, myconf.sock_port))
    srvsoc.close()
    s.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from .unit_level import UnitLevel
from .battle import Battle
from .battle import InvalidCombatantException
from .battle import CombatStatusException
from .battlemaster import Battlemaster
from .unit import Unit
from .dungeon import Dungeon
from .effect import Effect
from .item import Item
from .unit_builder import UnitBuilder
from .unit_dialogue import UnitDialogue
from .item_builder import ItemBuilder
from .unit_type import UnitType
from .item_generator import ItemGenerator
from .unit_generator import UnitGenerator
from .effect_generator import EffectGenerator
from .db import Database
from .world import World
from .announcer import Announcer
from .player_announcer import PlayerAnnouncer
from .dungeon_announcer import DungeonAnnouncer
from .worldbuilder import Worldbuilder
<commit_msg>Rearranges some thangs<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from .unit_level import UnitLevel
from .unit import Unit
from .dungeon import Dungeon
from .effect import Effect
from .item import Item
from .unit_builder import UnitBuilder
from .unit_dialogue import UnitDialogue
from .item_builder import ItemBuilder
from .unit_type import UnitType
from .item_generator import ItemGenerator
from .unit_generator import UnitGenerator
from .effect_generator import EffectGenerator
from .db import Database
from .announcer import Announcer
from .player_announcer import PlayerAnnouncer
from .dungeon_announcer import DungeonAnnouncer
from .battle import Battle
from .battle import InvalidCombatantException
from .battle import CombatStatusException
from .battlemaster import Battlemaster
from .world import World
from .worldbuilder import Worldbuilder
<|endoftext|>


<commit_before>__author__ = 'Johannes'
#import Adafruit_BBIO.GPIO as GPIO
import DMCC

class Robot:

    def setLeftMotor(self, percent):
        actuel = 0
        if percent == 0:
            print('leftMotor ' + str(percent))
            DMCC.setMotor(0, 1, percent * 100/2)
        if percent > 0:
            while actuel <= percent:
                print('leftMotor ' + str(actuel))
                DMCC.setMotor(0, 1, actuel * 100/2)
                actuel += 1
        if percent < 0:
            while actuel >= percent:
                print('leftMotor ' + str(actuel))
                DMCC.setMotor(0, 1, actuel * 100/2)
                actuel -= 1

    def setRightMotor(self, percent):
        actuel = 0
        if percent == 0:
            print('rightMotor ' + str(percent))
            DMCC.setMotor(0, 2, percent * 100/2)
        if percent > 0:
            while actuel <= percent:
                print('rightMotor ' + str(actuel))
                DMCC.setMotor(0, 2, actuel * 100/2)
                actuel += 1
        if percent < 0:
            while actuel >= percent:
                print('rightMotor ' + str(actuel))
                DMCC.setMotor(0, 2, actuel * 100/2)
                actuel -= 1

    def driveStraight(self):
        self.setLeftMotor(80)
        self.setRightMotor(80)

    def driveLeft(self):
        self.setRightMotor(100)
        self.setLeftMotor(40)

    def driveRight(self):
        self.setLeftMotor(100)
        self.setRightMotor(40)<commit_msg>anfahren mit wartezeiten<commit_after>__author__ = 'Johannes'
#import Adafruit_BBIO.GPIO as GPIO
import DMCC
import time

class Robot:

    def setLeftMotor(self, percent):
        actuel = 0
        if percent == 0:
            print('leftMotor ' + str(percent))
            DMCC.setMotor(0, 1, percent * 100/2)
        if percent > 0:
            while actuel <= percent:
                print('leftMotor ' + str(actuel))
                DMCC.setMotor(0, 1, actuel * 100/2)
                actuel += 1
                time.sleep(0.010)
        if percent < 0:
            while actuel >= percent:
                print('leftMotor ' + str(actuel))
                DMCC.setMotor(0, 1, actuel * 100/2)
                actuel -= 1
                time.sleep(0.010)

    def setRightMotor(self, percent):
        actuel = 0
        if percent == 0:
            print('rightMotor ' + str(percent))
            DMCC.setMotor(0, 2, percent * 100/2)
        if percent > 0:
            while actuel <= percent:
                print('rightMotor ' + str(actuel))
                DMCC.setMotor(0, 2, actuel * 100/2)
                actuel += 1
                time.sleep(0.010)
        if percent < 0:
            while actuel >= percent:
                print('rightMotor ' + str(actuel))
                DMCC.setMotor(0, 2, actuel * 100/2)
                actuel -= 1
                time.sleep(0.010)

    def driveStraight(self):
        self.setLeftMotor(80)
        self.setRightMotor(80)

    def driveLeft(self):
        self.setRightMotor(100)
        self.setLeftMotor(40)

    def driveRight(self):
        self.setLeftMotor(100)
        self.setRightMotor(40)<|endoftext|>