<commit_before>import re

from django.core import urlresolvers
from django.core.urlresolvers import Resolver404, resolve, reverse as _reverse, NoReverseMatch
from django.db import models
from django.http import HttpResponse
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

try:
    from functools import partial
except ImportError:
    from django.utils.functional import curry as partial

from feincms.admin.editor import ItemEditorForm
from feincms.contrib.fields import JSONField

try:
    from threading import local
except ImportError:
    from django.utils._threading_local import local

_local = local()

def retrieve_page_information(page):
    _local.proximity_info = (page.tree_id, page.lft, page.rght, page.level)

OTHER_APPLICATIONCONTENT_SEPARATOR = '/'

def reverse(viewname, urlconf=None, args=None, kwargs=None, prefix=None, *vargs, **vkwargs):
    """
    This reverse replacement adds two new capabilities to the Django reverse method:

    - If reverse is called from inside ``ApplicationContent.process``, it
      automatically prepends the URL of the page the ``ApplicationContent``
      is attached to, thereby allowing ``reverse`` and ``{% url %}`` to
      return correct URLs without hard-coding the application integration
      point into the templates or URLconf files.
    - If the viewname contains a slash, the part before the slash is
      interpreted as the path to an URLconf file. This allows the template
      author to resolve URLs only reachable via an ``ApplicationContent``,
      even inside another application contents' ``process`` method::

          {% url registration.urls/auth_logout %}
    """

    if isinstance(viewname, basestring) and OTHER_APPLICATIONCONTENT_SEPARATOR in viewname:
        # try to reverse an URL inside another applicationcontent
        other_urlconf, other_viewname = viewname.split(OTHER_APPLICATIONCONTENT_SEPARATOR)

        if hasattr(_local, 'urlconf') and other_urlconf == _local.urlconf[0]:
            # We are reversing an URL from our own ApplicationContent
            return _reverse(other_viewname, other_urlconf, args, kwargs, _local.urlconf[1], *vargs, **vkwargs)

        if not hasattr(_local, 'reverse_cache'):
            _local.reverse_cache = {}

        if other_urlconf not in _local.reverse_cache:
            # TODO do not use internal feincms data structures as much
            model_class = ApplicationContent._feincms_content_models[0]
            contents = model_class.objects.filter(
                urlconf_path=other_urlconf).select_related('parent')

            proximity_info = getattr(_local, 'proximity_info', None)

            if proximity_info:
                # Poor man's proximity analysis. Filter by tree_id :-)
                try:
                    content = contents.get(parent__tree_id=proximity_info[0])
                except (model_class.DoesNotExist, model_class.MultipleObjectsReturned):
                    try:
                        content = contents[0]
                    except IndexError:
                        content = None
            else:
                try:
                    content = contents[0]
                except IndexError:
                    content = None
            _local.reverse_cache[other_urlconf] = content
        else:
            content = _local.reverse_cache[other_urlconf]

        if content:
            # Save information from _urlconfs in case we are inside another
            # application contents' ``process`` method currently
            saved_cfg = getattr(_local, 'urlconf', None)

            # Initialize application content reverse hackery for the other application
            _local.urlconf = (other_urlconf, content.parent.get_absolute_url())

            try:
                url = reverse(other_viewname, other_urlconf, args, kwargs, prefix, *vargs, **vkwargs)
            except:
                # We really must not fail here. We absolutely need to remove/restore
                # the _urlconfs information
                url = None

            if saved_cfg:
                _local.urlconf = saved_cfg
            else:
                del _local.urlconf

            # We found an URL somewhere in here... return it. Otherwise, we continue
            # below
            if url:
                return url

    if hasattr(_local, 'urlconf'):
        # Special handling inside ApplicationContent.render; override urlconf
        # and prefix variables so that reverse works as expected.
        urlconf1, prefix1 = _local.urlconf
        try:
            return _reverse(viewname, urlconf1, args, kwargs, prefix1, *vargs, **vkwargs)
        except NoReverseMatch:
            # fall through to calling reverse with default arguments
            pass

    return _reverse(viewname, urlconf, args, kwargs, prefix, *vargs, **vkwargs)
urlresolvers.reverse = reverse

class ApplicationContent(models.Model):
    #: parameters is used to serialize instance-specific data which will be
    # provided to the view code. This allows customization (e.g. "Embed
    # MyBlogApp for blog <slug>")
    parameters = JSONField(null=True, editable=False)

    ALL_APPS_CONFIG = {}

    class Meta:
        abstract = True
        verbose_name = _('application content')
        verbose_name_plural = _('application contents')

    @classmethod
    def initialize_type(cls, APPLICATIONS):
        # Generate a more flexible application configuration structure from
        # the legacy pattern:

        # TODO: Consider changing the input signature to something cleaner, at
        # the cost of a one-time backwards incompatible change

        for i in APPLICATIONS:
            if not 2 <= len(i) <= 3:
                raise ValueError("APPLICATIONS must be provided with tuples containing at least two parameters (urls, name) and an optional extra config dict")

            urls, name = i[0:2]

            if len(i) == 3:
                app_conf = i[2]

                if not isinstance(app_conf, dict):
                    raise ValueError("The third parameter of an APPLICATIONS entry must be a dict or the name of one!")
            else:
                app_conf = {}

            cls.ALL_APPS_CONFIG[urls] = {
                "urls":     urls,
                "name":     name,
                "config":   app_conf
            }

        cls.add_to_class('urlconf_path',
            models.CharField(_('application'), max_length=100, choices=[(c['urls'], c['name']) for c in cls.ALL_APPS_CONFIG.values()])
        )

        class ApplicationContentItemEditorForm(ItemEditorForm):
            app_config    = {}
            custom_fields = {}

            def __init__(self, *args, **kwargs):
                super(ApplicationContentItemEditorForm, self).__init__(*args, **kwargs)

                instance = kwargs.get("instance", None)

                if instance:
                    self.app_config = cls.ALL_APPS_CONFIG[instance.urlconf_path]['config']
                    admin_fields    = self.app_config.get('admin_fields', {})

                    if isinstance(admin_fields, dict):
                        self.custom_fields.update(admin_fields)
                    else:
                        get_fields = urlresolvers.get_callable(admin_fields)
                        self.custom_fields.update(get_fields(self, *args, **kwargs))

                    for k, v in self.custom_fields.items():
                        self.fields[k] = v

            def clean(self, *args, **kwargs):
                cleaned_data = super(ApplicationContentItemEditorForm, self).clean(*args, **kwargs)

                # TODO: Check for newly added instances so we can force a re-validation of their custom fields

                return cleaned_data

            def save(self, commit=True, *args, **kwargs):
                # Django ModelForms return the model instance from save. We'll
                # call save with commit=False first to do any necessary work &
                # get the model so we can set .parameters to the values of our
                # custom fields before calling save(commit=True)

                m = super(ApplicationContentItemEditorForm, self).save(commit=False, *args, **kwargs)

                m.parameters = dict((k, self.cleaned_data[k]) for k in self.custom_fields if k in self.cleaned_data)

                if commit:
                    m.save(**kwargs)

                return m

        #: This provides hooks for us to customize the admin interface for embedded instances:
        cls.feincms_item_editor_form = ApplicationContentItemEditorForm

    def __init__(self, *args, **kwargs):
        super(ApplicationContent, self).__init__(*args, **kwargs)
        self.app_config = self.ALL_APPS_CONFIG.get(self.urlconf_path, {}).get('config', {})

    def render(self, request, **kwargs):
        return getattr(request, "_feincms_applicationcontents", {}).get(self.id, u'')

    def process(self, request):
        page_url = self.parent.get_absolute_url()

        # Get the rest of the URL

        # Provide a way for appcontent items to customize URL processing by
        # altering the perceived path of the page:
        if "path_mapper" in self.app_config:
            path_mapper = urlresolvers.get_callable(self.app_config["path_mapper"])
            path, page_url = path_mapper(
                request.path,
                page_url,
                appcontent_parameters=self.parameters
            )
        else:
            path = re.sub('^' + re.escape(page_url[:-1]), '', request.path)

        # Change the prefix and urlconf for the monkey-patched reverse function ...
        _local.urlconf = (self.urlconf_path, page_url)

        try:
            fn, args, kwargs = resolve(path, self.urlconf_path)
        except (ValueError, Resolver404):
            del _local.urlconf
            raise Resolver404

        #: Variables from the ApplicationContent parameters are added to request
        #  so we can expose them to our templates via the appcontent_parameters
        #  context_processor
        request._feincms_appcontent_parameters.update(self.parameters)

        view_wrapper = self.app_config.get("view_wrapper", None)
        if view_wrapper:
            fn = partial(
                urlresolvers.get_callable(view_wrapper),
                view=fn,
                appcontent_parameters=self.parameters
            )

        try:
            output = fn(request, *args, **kwargs)
        except:
            # We want exceptions to propagate, but we cannot allow the
            # modifications to reverse() to stay here.
            del _local.urlconf
            raise

        # ... and restore it after processing the view
        del _local.urlconf

        if isinstance(output, HttpResponse):
            if output.status_code == 200:
                if not getattr(output, 'standalone', False):
                    request._feincms_applicationcontents[self.id] = mark_safe(output.content.decode('utf-8'))

            return output
        else:
            request._feincms_applicationcontents[self.id] = mark_safe(output)
<commit_msg>ApplicationContent: Clear reverse cache when changing application contents<commit_after>import re

from django.core import urlresolvers
from django.core.urlresolvers import Resolver404, resolve, reverse as _reverse, NoReverseMatch
from django.db import models
from django.http import HttpResponse
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

try:
    from functools import partial
except ImportError:
    from django.utils.functional import curry as partial

from feincms.admin.editor import ItemEditorForm
from feincms.contrib.fields import JSONField

try:
    from threading import local
except ImportError:
    from django.utils._threading_local import local

_local = local()

def retrieve_page_information(page):
    _local.proximity_info = (page.tree_id, page.lft, page.rght, page.level)

OTHER_APPLICATIONCONTENT_SEPARATOR = '/'

def reverse(viewname, urlconf=None, args=None, kwargs=None, prefix=None, *vargs, **vkwargs):
    """
    This reverse replacement adds two new capabilities to the Django reverse method:

    - If reverse is called from inside ``ApplicationContent.process``, it
      automatically prepends the URL of the page the ``ApplicationContent``
      is attached to, thereby allowing ``reverse`` and ``{% url %}`` to
      return correct URLs without hard-coding the application integration
      point into the templates or URLconf files.
    - If the viewname contains a slash, the part before the slash is
      interpreted as the path to an URLconf file. This allows the template
      author to resolve URLs only reachable via an ``ApplicationContent``,
      even inside another application contents' ``process`` method::

          {% url registration.urls/auth_logout %}
    """

    if isinstance(viewname, basestring) and OTHER_APPLICATIONCONTENT_SEPARATOR in viewname:
        # try to reverse an URL inside another applicationcontent
        other_urlconf, other_viewname = viewname.split(OTHER_APPLICATIONCONTENT_SEPARATOR)

        if hasattr(_local, 'urlconf') and other_urlconf == _local.urlconf[0]:
            # We are reversing an URL from our own ApplicationContent
            return _reverse(other_viewname, other_urlconf, args, kwargs, _local.urlconf[1], *vargs, **vkwargs)

        if not hasattr(_local, 'reverse_cache'):
            _local.reverse_cache = {}

        if other_urlconf not in _local.reverse_cache:
            # TODO do not use internal feincms data structures as much
            model_class = ApplicationContent._feincms_content_models[0]
            contents = model_class.objects.filter(
                urlconf_path=other_urlconf).select_related('parent')

            proximity_info = getattr(_local, 'proximity_info', None)

            if proximity_info:
                # Poor man's proximity analysis. Filter by tree_id :-)
                try:
                    content = contents.get(parent__tree_id=proximity_info[0])
                except (model_class.DoesNotExist, model_class.MultipleObjectsReturned):
                    try:
                        content = contents[0]
                    except IndexError:
                        content = None
            else:
                try:
                    content = contents[0]
                except IndexError:
                    content = None
            _local.reverse_cache[other_urlconf] = content
        else:
            content = _local.reverse_cache[other_urlconf]

        if content:
            # Save information from _urlconfs in case we are inside another
            # application contents' ``process`` method currently
            saved_cfg = getattr(_local, 'urlconf', None)

            # Initialize application content reverse hackery for the other application
            _local.urlconf = (other_urlconf, content.parent.get_absolute_url())

            try:
                url = reverse(other_viewname, other_urlconf, args, kwargs, prefix, *vargs, **vkwargs)
            except:
                # We really must not fail here. We absolutely need to remove/restore
                # the _urlconfs information
                url = None

            if saved_cfg:
                _local.urlconf = saved_cfg
            else:
                del _local.urlconf

            # We found an URL somewhere in here... return it. Otherwise, we continue
            # below
            if url:
                return url

    if hasattr(_local, 'urlconf'):
        # Special handling inside ApplicationContent.render; override urlconf
        # and prefix variables so that reverse works as expected.
        urlconf1, prefix1 = _local.urlconf
        try:
            return _reverse(viewname, urlconf1, args, kwargs, prefix1, *vargs, **vkwargs)
        except NoReverseMatch:
            # fall through to calling reverse with default arguments
            pass

    return _reverse(viewname, urlconf, args, kwargs, prefix, *vargs, **vkwargs)
urlresolvers.reverse = reverse

class ApplicationContent(models.Model):
    #: parameters is used to serialize instance-specific data which will be
    # provided to the view code. This allows customization (e.g. "Embed
    # MyBlogApp for blog <slug>")
    parameters = JSONField(null=True, editable=False)

    ALL_APPS_CONFIG = {}

    class Meta:
        abstract = True
        verbose_name = _('application content')
        verbose_name_plural = _('application contents')

    @classmethod
    def initialize_type(cls, APPLICATIONS):
        # Generate a more flexible application configuration structure from
        # the legacy pattern:

        # TODO: Consider changing the input signature to something cleaner, at
        # the cost of a one-time backwards incompatible change

        for i in APPLICATIONS:
            if not 2 <= len(i) <= 3:
                raise ValueError("APPLICATIONS must be provided with tuples containing at least two parameters (urls, name) and an optional extra config dict")

            urls, name = i[0:2]

            if len(i) == 3:
                app_conf = i[2]

                if not isinstance(app_conf, dict):
                    raise ValueError("The third parameter of an APPLICATIONS entry must be a dict or the name of one!")
            else:
                app_conf = {}

            cls.ALL_APPS_CONFIG[urls] = {
                "urls":     urls,
                "name":     name,
                "config":   app_conf
            }

        cls.add_to_class('urlconf_path',
            models.CharField(_('application'), max_length=100, choices=[(c['urls'], c['name']) for c in cls.ALL_APPS_CONFIG.values()])
        )

        class ApplicationContentItemEditorForm(ItemEditorForm):
            app_config    = {}
            custom_fields = {}

            def __init__(self, *args, **kwargs):
                super(ApplicationContentItemEditorForm, self).__init__(*args, **kwargs)

                instance = kwargs.get("instance", None)

                if instance:
                    self.app_config = cls.ALL_APPS_CONFIG[instance.urlconf_path]['config']
                    admin_fields    = self.app_config.get('admin_fields', {})

                    if isinstance(admin_fields, dict):
                        self.custom_fields.update(admin_fields)
                    else:
                        get_fields = urlresolvers.get_callable(admin_fields)
                        self.custom_fields.update(get_fields(self, *args, **kwargs))

                    for k, v in self.custom_fields.items():
                        self.fields[k] = v

            def clean(self, *args, **kwargs):
                cleaned_data = super(ApplicationContentItemEditorForm, self).clean(*args, **kwargs)

                # TODO: Check for newly added instances so we can force a re-validation of their custom fields

                return cleaned_data

            def save(self, commit=True, *args, **kwargs):
                # Django ModelForms return the model instance from save. We'll
                # call save with commit=False first to do any necessary work &
                # get the model so we can set .parameters to the values of our
                # custom fields before calling save(commit=True)

                m = super(ApplicationContentItemEditorForm, self).save(commit=False, *args, **kwargs)

                m.parameters = dict((k, self.cleaned_data[k]) for k in self.custom_fields if k in self.cleaned_data)

                if commit:
                    m.save(**kwargs)

                return m

        #: This provides hooks for us to customize the admin interface for embedded instances:
        cls.feincms_item_editor_form = ApplicationContentItemEditorForm

    def __init__(self, *args, **kwargs):
        super(ApplicationContent, self).__init__(*args, **kwargs)
        self.app_config = self.ALL_APPS_CONFIG.get(self.urlconf_path, {}).get('config', {})

    def render(self, request, **kwargs):
        return getattr(request, "_feincms_applicationcontents", {}).get(self.id, u'')

    def process(self, request):
        page_url = self.parent.get_absolute_url()

        # Get the rest of the URL

        # Provide a way for appcontent items to customize URL processing by
        # altering the perceived path of the page:
        if "path_mapper" in self.app_config:
            path_mapper = urlresolvers.get_callable(self.app_config["path_mapper"])
            path, page_url = path_mapper(
                request.path,
                page_url,
                appcontent_parameters=self.parameters
            )
        else:
            path = re.sub('^' + re.escape(page_url[:-1]), '', request.path)

        # Change the prefix and urlconf for the monkey-patched reverse function ...
        _local.urlconf = (self.urlconf_path, page_url)

        try:
            fn, args, kwargs = resolve(path, self.urlconf_path)
        except (ValueError, Resolver404):
            del _local.urlconf
            raise Resolver404

        #: Variables from the ApplicationContent parameters are added to request
        #  so we can expose them to our templates via the appcontent_parameters
        #  context_processor
        request._feincms_appcontent_parameters.update(self.parameters)

        view_wrapper = self.app_config.get("view_wrapper", None)
        if view_wrapper:
            fn = partial(
                urlresolvers.get_callable(view_wrapper),
                view=fn,
                appcontent_parameters=self.parameters
            )

        try:
            output = fn(request, *args, **kwargs)
        except:
            # We want exceptions to propagate, but we cannot allow the
            # modifications to reverse() to stay here.
            del _local.urlconf
            raise

        # ... and restore it after processing the view
        del _local.urlconf

        if isinstance(output, HttpResponse):
            if output.status_code == 200:
                if not getattr(output, 'standalone', False):
                    request._feincms_applicationcontents[self.id] = mark_safe(output.content.decode('utf-8'))

            return output
        else:
            request._feincms_applicationcontents[self.id] = mark_safe(output)

    def save(self, *args, **kwargs):
        super(ApplicationContent, self).save(*args, **kwargs)
        # Clear reverse() cache
        _local.reverse_cache = {}

    def delete(self, *args, **kwargs):
        super(ApplicationContent, self).delete(*args, **kwargs)
        # Clear reverse() cache
        _local.reverse_cache = {}

<|endoftext|>


<commit_before>import re

from django.core import urlresolvers
from django.core.urlresolvers import Resolver404, resolve, reverse as _reverse, NoReverseMatch
from django.db import models
from django.http import HttpResponse
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

try:
    from threading import local
except ImportError:
    from django.utils._threading_local import local

_local = local()

def retrieve_page_information(page):
    _local.proximity_info = (page.tree_id, page.lft, page.rght, page.level)

OTHER_APPLICATIONCONTENT_SEPARATOR = '/'

def reverse(viewname, urlconf=None, args=None, kwargs=None, prefix=None, *vargs, **vkwargs):
    """
    This reverse replacement adds two new capabilities to the Django reverse method:

    - If reverse is called from inside ``ApplicationContent.process``, it
      automatically prepends the URL of the page the ``ApplicationContent``
      is attached to, thereby allowing ``reverse`` and ``{% url %}`` to
      return correct URLs without hard-coding the application integration
      point into the templates or URLconf files.
    - If the viewname contains a slash, the part before the slash is
      interpreted as the path to an URLconf file. This allows the template
      author to resolve URLs only reachable via an ``ApplicationContent``,
      even inside another application contents' ``process`` method::

          {% url registration.urls/auth_logout %}
    """

    if OTHER_APPLICATIONCONTENT_SEPARATOR in viewname:
        # try to reverse an URL inside another applicationcontent
        other_urlconf, other_viewname = viewname.split(OTHER_APPLICATIONCONTENT_SEPARATOR)

        # TODO do not use internal feincms data structures as much
        model_class = ApplicationContent._feincms_content_models[0]
        contents = model_class.objects.filter(
            urlconf_path=other_urlconf).select_related('parent')

        proximity_info = getattr(_local, 'proximity_info', None)

        if proximity_info:
            # Poor man's proximity analysis. Filter by tree_id :-)
            try:
                content = contents.get(parent__tree_id=proximity_info[0])
            except (model_class.DoesNotExist, model_class.MultipleObjectsReturned):
                try:
                    content = contents[0]
                except IndexError:
                    content = None
        else:
            try:
                content = contents[0]
            except IndexError:
                content = None

        if content:
            # Save information from _urlconfs in case we are inside another
            # application contents' ``process`` method currently
            saved_cfg = getattr(_local, 'urlconf', None)

            # Initialize application content reverse hackery for the other application
            _local.urlconf = (other_urlconf, content.parent.get_absolute_url())

            try:
                url = reverse(other_viewname, other_urlconf, args, kwargs, prefix, *vargs, **vkwargs)
            except:
                # We really must not fail here. We absolutely need to remove/restore
                # the _urlconfs information
                url = None

            if saved_cfg:
                _local.urlconf = saved_cfg
            else:
                del _local.urlconf

            # We found an URL somewhere in here... return it. Otherwise, we continue
            # below
            if url:
                return url

    if hasattr(_local, 'urlconf'):
        # Special handling inside ApplicationContent.render; override urlconf
        # and prefix variables so that reverse works as expected.
        urlconf1, prefix1 = _local.urlconf
        try:
            return _reverse(viewname, urlconf1, args, kwargs, prefix1, *vargs, **vkwargs)
        except NoReverseMatch:
            # fall through to calling reverse with default arguments
            pass

    return _reverse(viewname, urlconf, args, kwargs, prefix, *vargs, **vkwargs)
urlresolvers.reverse = reverse

class ApplicationContent(models.Model):
    class Meta:
        abstract = True
        verbose_name = _('application content')
        verbose_name_plural = _('application contents')

    @classmethod
    def initialize_type(cls, APPLICATIONS):
        cls.add_to_class('urlconf_path', models.CharField(_('application'), max_length=100,
                                                          choices=APPLICATIONS))

    def render(self, request, **kwargs):
        return request._feincms_applicationcontents.get(self.id, u'')

    def process(self, request):
        # prepare storage for rendered application contents
        if not hasattr(request, '_feincms_applicationcontents'):
            request._feincms_applicationcontents = {}
            request._feincms_applicationcontents_fragments = {}

        page_url = self.parent.get_absolute_url()

        # Get the rest of the URL
        path = re.sub('^' + re.escape(page_url[:-1]), '', request.path)

        # Change the prefix and urlconf for the monkey-patched reverse function ...
        _local.urlconf = (self.urlconf_path, page_url)

        try:
            fn, args, kwargs = resolve(path, self.urlconf_path)
        except (ValueError, Resolver404):
            # Silent failure if resolving failed
            del _local.urlconf
            return

        try:
            output = fn(request, *args, **kwargs)
        except:
            # We want exceptions to propagate, but we cannot allow the
            # modifications to reverse() to stay here.
            del _local.urlconf
            raise

        # ... and restore it after processing the view
        del _local.urlconf

        if isinstance(output, HttpResponse):
            if output.status_code == 200:
                request._feincms_applicationcontents[self.id] = mark_safe(output.content.decode('utf-8'))

            return output
        else:
            request._feincms_applicationcontents[self.id] = mark_safe(output)
<commit_msg>Patched reverse: Do not iterate over a function object, it does not work<commit_after>import re

from django.core import urlresolvers
from django.core.urlresolvers import Resolver404, resolve, reverse as _reverse, NoReverseMatch
from django.db import models
from django.http import HttpResponse
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

try:
    from threading import local
except ImportError:
    from django.utils._threading_local import local

_local = local()

def retrieve_page_information(page):
    _local.proximity_info = (page.tree_id, page.lft, page.rght, page.level)

OTHER_APPLICATIONCONTENT_SEPARATOR = '/'

def reverse(viewname, urlconf=None, args=None, kwargs=None, prefix=None, *vargs, **vkwargs):
    """
    This reverse replacement adds two new capabilities to the Django reverse method:

    - If reverse is called from inside ``ApplicationContent.process``, it
      automatically prepends the URL of the page the ``ApplicationContent``
      is attached to, thereby allowing ``reverse`` and ``{% url %}`` to
      return correct URLs without hard-coding the application integration
      point into the templates or URLconf files.
    - If the viewname contains a slash, the part before the slash is
      interpreted as the path to an URLconf file. This allows the template
      author to resolve URLs only reachable via an ``ApplicationContent``,
      even inside another application contents' ``process`` method::

          {% url registration.urls/auth_logout %}
    """

    if isinstance(viewname, basestring) and OTHER_APPLICATIONCONTENT_SEPARATOR in viewname:
        # try to reverse an URL inside another applicationcontent
        other_urlconf, other_viewname = viewname.split(OTHER_APPLICATIONCONTENT_SEPARATOR)

        # TODO do not use internal feincms data structures as much
        model_class = ApplicationContent._feincms_content_models[0]
        contents = model_class.objects.filter(
            urlconf_path=other_urlconf).select_related('parent')

        proximity_info = getattr(_local, 'proximity_info', None)

        if proximity_info:
            # Poor man's proximity analysis. Filter by tree_id :-)
            try:
                content = contents.get(parent__tree_id=proximity_info[0])
            except (model_class.DoesNotExist, model_class.MultipleObjectsReturned):
                try:
                    content = contents[0]
                except IndexError:
                    content = None
        else:
            try:
                content = contents[0]
            except IndexError:
                content = None

        if content:
            # Save information from _urlconfs in case we are inside another
            # application contents' ``process`` method currently
            saved_cfg = getattr(_local, 'urlconf', None)

            # Initialize application content reverse hackery for the other application
            _local.urlconf = (other_urlconf, content.parent.get_absolute_url())

            try:
                url = reverse(other_viewname, other_urlconf, args, kwargs, prefix, *vargs, **vkwargs)
            except:
                # We really must not fail here. We absolutely need to remove/restore
                # the _urlconfs information
                url = None

            if saved_cfg:
                _local.urlconf = saved_cfg
            else:
                del _local.urlconf

            # We found an URL somewhere in here... return it. Otherwise, we continue
            # below
            if url:
                return url

    if hasattr(_local, 'urlconf'):
        # Special handling inside ApplicationContent.render; override urlconf
        # and prefix variables so that reverse works as expected.
        urlconf1, prefix1 = _local.urlconf
        try:
            return _reverse(viewname, urlconf1, args, kwargs, prefix1, *vargs, **vkwargs)
        except NoReverseMatch:
            # fall through to calling reverse with default arguments
            pass

    return _reverse(viewname, urlconf, args, kwargs, prefix, *vargs, **vkwargs)
urlresolvers.reverse = reverse

class ApplicationContent(models.Model):
    class Meta:
        abstract = True
        verbose_name = _('application content')
        verbose_name_plural = _('application contents')

    @classmethod
    def initialize_type(cls, APPLICATIONS):
        cls.add_to_class('urlconf_path', models.CharField(_('application'), max_length=100,
                                                          choices=APPLICATIONS))

    def render(self, request, **kwargs):
        return request._feincms_applicationcontents.get(self.id, u'')

    def process(self, request):
        # prepare storage for rendered application contents
        if not hasattr(request, '_feincms_applicationcontents'):
            request._feincms_applicationcontents = {}
            request._feincms_applicationcontents_fragments = {}

        page_url = self.parent.get_absolute_url()

        # Get the rest of the URL
        path = re.sub('^' + re.escape(page_url[:-1]), '', request.path)

        # Change the prefix and urlconf for the monkey-patched reverse function ...
        _local.urlconf = (self.urlconf_path, page_url)

        try:
            fn, args, kwargs = resolve(path, self.urlconf_path)
        except (ValueError, Resolver404):
            # Silent failure if resolving failed
            del _local.urlconf
            return

        try:
            output = fn(request, *args, **kwargs)
        except:
            # We want exceptions to propagate, but we cannot allow the
            # modifications to reverse() to stay here.
            del _local.urlconf
            raise

        # ... and restore it after processing the view
        del _local.urlconf

        if isinstance(output, HttpResponse):
            if output.status_code == 200:
                request._feincms_applicationcontents[self.id] = mark_safe(output.content.decode('utf-8'))

            return output
        else:
            request._feincms_applicationcontents[self.id] = mark_safe(output)
<|endoftext|>


<commit_before>SEG_NON_POSITIVE_MESSAGE = "The segment size must be a positive integer"
OVERLAP_LARGE_MESSAGE = "The segment size must be greater than " \
    "the overlap of words between chunks"
PROP_NEGATIVE_MESSAGE = "The proportional size of the last segment must be " \
    "zero or greater"
OVERLAP_NEGATIVE_MESSAGE = "The overlap between chunks must be zero or greater"
NON_POSITIVE_SEGMENT_MESSAGE = "the segment size should be positive"
NEG_OVERLAP_LAST_PROP_MESSAGE = "the overlap or last segment proportion" \
                                "should not be negative"
LARGER_CHUNK_SIZE_MESSAGE = "the segment size should be larger " \
    "than the overlap size"
NON_NEGATIVE_INDEX_MESSAGE = "the index should be larger than or equal to zero"
MATRIX_DIMENSION_UNEQUAL_MESSAGE = "the dimension of the matrix " \
    "should be equal"
EMPTY_LIST_MESSAGE = "the list should not be empty"
WINDOW_SIZE_LARGE_MESSAGE = "The window size must be less than or equal to" \
                            " the length of the given document"
INVALID_CUTTING_TYPE_MESSAGE = "the cutting type should be letters, lines, " \
    "number, words or milestone"
EMPTY_NP_ARRAY_MESSAGE = "the numpy array should not be empty"
<commit_msg>Specified error message<commit_after>SEG_NON_POSITIVE_MESSAGE = "The segment size must be a positive integer"
OVERLAP_LARGE_MESSAGE = "The segment size must be greater than " \
    "the overlap of words between chunks"
PROP_NEGATIVE_MESSAGE = "The proportional size of the last segment must be " \
    "zero or greater"
OVERLAP_NEGATIVE_MESSAGE = "The overlap between chunks must be zero or greater"
NON_POSITIVE_SEGMENT_MESSAGE = "the segment size should be positive"
NEG_OVERLAP_LAST_PROP_MESSAGE = "the overlap or last segment proportion" \
                                "should not be negative"
LARGER_CHUNK_SIZE_MESSAGE = "the segment size should be larger " \
    "than the overlap size"
NON_NEGATIVE_INDEX_MESSAGE = "the index should be larger than or equal to zero"
MATRIX_DIMENSION_UNEQUAL_MESSAGE = "the dimension of the matrix " \
    "should be equal"
EMPTY_LIST_MESSAGE = "the list should not be empty"
WINDOW_SIZE_LARGE_MESSAGE = "The window size must be less than or equal to" \
                            " the length of the given document"
INVALID_CUTTING_TYPE_MESSAGE = "the cutting type should be letters, lines, " \
    "number, words or milestone"
EMPTY_NP_ARRAY_MESSAGE = "The input numpy array should not be empty, since" \
    "the input file should not be empty."
<|endoftext|>


<commit_before>"""
@brief
@file Batch file use to automate some of the tasks (setup, unit tests, help, pypi).

.. versionadded:: 1.1
"""

#################
#: stop if error
#################
windows_error = "if %errorlevel% neq 0 exit /b %errorlevel%"

#################
#: prefix
#################
windows_prefix = """
if "%1"=="" goto default_value_python:
set pythonexe=%1
goto start_script:

:default_value_python:
set pythonexe=__PY34_X64__\\python
:start_script:
"""

#################
#: prefix 27
#################
windows_prefix_27 = """
if "%1"=="" goto default_value_python:
set pythonexe27=%1
goto start_script:

:default_value_python:
set pythonexe27=__PY27_X64__\\python
:start_script:
"""

#################
#: run unit test 27
#################
windows_unittest27 = """
set PYTHONPATH=
cd dist_module27\\_unittests

for /d %%d in (ut_*) do %pythonexe27%\\..\\Scripts\\nosetests.exe -w %%d

""" + windows_error + "\ncd ..\.."

#################
#: call the setup
#################
windows_setup = "%pythonexe% setup.py"
jenkins_windows_setup = "%jenkinspythonexe% setup.py"

#################
#: build script for Windows
#################
windows_build = """
IF EXIST dist del /Q dist\\*.*

set virtual_env_suffix=%2

if "%1"=="" goto default_value:
set pythonexe=%1
%pythonexe% setup.py write_version
goto custom_python:

:default_value:
IF NOT EXIST __PY34__ GOTO checkinstall64:

:checkinstall:
IF EXIST __PY34__vir GOTO nexta:
mkdir __PY34__vir

:nexta:
IF EXIST __PY34__vir\\install GOTO fullsetupa:
__PY34__\\Scripts\\virtualenv __PY34__vir\\install --system-site-packages
if %errorlevel% neq 0 exit /b %errorlevel%

:fullsetupa:
echo #######################################################0
__PY34__vir\\install\\Scripts\\python -u setup.py install
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################1

:checkinstall64:
IF EXIST __PY34_X64__vir GOTO nextb:
mkdir __PY34_X64__vir

:nextb:
IF EXIST __PY34_X64__vir\\install GOTO fullsetupb:
__PY34_X64__\\Scripts\\virtualenv __PY34_X64__vir\\install --system-site-packages
if %errorlevel% neq 0 exit /b %errorlevel%

:fullsetupb:
echo #######################################################2
__PY34_X64__vir\\install\\Scripts\\python -u setup.py install
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################3

:setup34:
IF NOT EXIST __PY34__ GOTO utpy34_64:
set pythonexe=__PY34__\\python
%pythonexe% setup.py write_version
%pythonexe% setup.py clean_pyd
%pythonexe% setup.py build bdist_wininst --plat-name=win-amd64
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################4

:utpy34_64:
set pythonexe=__PY34_X64__\\python
:custom_python:
echo ###----################################################5
if not exist ..\\virtual mkdir ..\\virtual
set virtual_env_py=..\\virtual\\__MODULE__
if not exist %pythonexe%\\..\\Scripts\\virtualenv.exe goto conda_virtual_env:

if exist %virtual_env_py%_vir rmdir /Q /S %virtual_env_py%_vir
mkdir %virtual_env_py%_vir

if exist %virtual_env_py%_vir%virtual_env_suffix%\\python goto with_virtual:
set KEEPPATH=PATH
set PATH=%pythonexe%\\..;%PATH%
%pythonexe%\\..\\Scripts\\virtualenv --system-site-packages %virtual_env_py%_vir%virtual_env_suffix%
set PATH=%KEEPPATH%
if %errorlevel% neq 0 exit /b %errorlevel%
:with_virtual:
set pythonexe=%virtual_env_py%_vir%virtual_env_suffix%\\Scripts\\python
set pythonpip=%virtual_env_py%_vir%virtual_env_suffix%\\Scripts\\pip
goto requirements:

:conda_virtual_env:

if exist %virtual_env_py%_condavir rmdir /Q /S %virtual_env_py%_condavir

if exist %virtual_env_py%_condavir\\python goto with_virtual_conda:
%pythonexe%\\..\\Scripts\\conda create -p %virtual_env_py%_condavir --clone %pythonexe%\\.. --offline
if %errorlevel% neq 0 exit /b %errorlevel%
:with_virtual_conda:
set pythonexe=%virtual_env_py%_condavir\\python
set pythonpip=%virtual_env_py%_condavir\\Scripts\\pip

:requirements:
echo #######################################################_auto_setup_dep.py
%pythonexe% build\\auto_setup\\auto_setup_dep.py install
if %errorlevel% neq 0 exit /b %errorlevel%

echo #######################################################_requirements_begin
echo %pythonpip%
__REQUIREMENTS__
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################_requirements_end

%pythonexe% setup.py write_version
echo #######################################################_clean
%pythonexe% -u setup.py clean_space
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################_unit
%pythonexe% -u setup.py unittests
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################6

%pythonexe% setup.py clean_pyd
%pythonexe% setup.py sdist --formats=gztar,zip --verbose
if %errorlevel% neq 0 exit /b %errorlevel%
%pythonexe% setup.py bdist_wininst --plat-name=win-amd64
if %errorlevel% neq 0 exit /b %errorlevel%
%pythonexe% setup.py bdist_msi
if %errorlevel% neq 0 exit /b %errorlevel%
%pythonexe% setup.py bdist_wheel
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################7

:documentation:
%pythonexe% -u setup.py build_sphinx
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################8

:copyfiles:
if not exist dist\\html mkdir dist\\html
xcopy /E /C /I /Y _doc\\sphinxdoc\\build\\html dist\\html
if exist _doc\\sphinxdoc\\build\\latex xcopy /E /C /I /Y _doc\\sphinxdoc\\build\\latex\\*.pdf dist\\html
if %errorlevel% neq 0 exit /b %errorlevel%

:end:
rem we copy the wheel on a local folder to let a pypiserver take it
if not exist ..\\local_pypi_server mkdir ..\\local_pypi_server
copy /Y dist\\*.whl ..\\local_pypi_server
"""

#################
#: notebooks
#################
windows_notebook = """
if "%1"=="" goto default_value:
set pythonexe=%1
goto nextn:

:default_value:
set pythonexe=__PY34_X64__

:nextn:
set path=%path%;%pythonexe%;%pythonexe%\\Scripts
ipython3 notebook --notebook-dir=_doc\\notebooks --matplotlib=inline
"""

#################
#: publish a module
#################
windows_publish = """
%pythonexe% setup.py rotate --match=.zip --keep=1
%pythonexe% setup.py rotate --match=.tar.gz --keep=3
rem %pythonexe% setup.py sdist register
%pythonexe% setup.py sdist --formats=gztar upload
"""

#################
#: publish the documentation
#################
windows_publish_doc = """
%pythonexe% setup.py upload_docs --upload-dir=dist/html
"""

#################
#: run a pypi server
#################
windows_pypi = """
set pythonexe=__PY34_X64__

:custom_python:
if "%2"=="" goto default_port:
set portpy=%2
goto run:

:default_port:
set portpy=__PORT__

:run:
%pythonexe%\Scripts\pypi-server.exe -u -p %portpy% --disable-fallback ..\\local_pypi_server
"""

#################
#: script for Jenkins
#################
windows_jenkins = "set jenkinspythonexe=__PYTHON__\n" + jenkins_windows_setup + " build_script\n" + \
    windows_error + "\nauto_unittest_setup_help.bat %jenkinspythonexe% __SUFFIX__\n" + \
    windows_error

#################
#: script for Jenkins
#################
windows_jenkins_27 = [
    "set jenkinspythonexe=__PYTHON__\n" + jenkins_windows_setup + " build_script\n" +
    windows_error + "\nauto_setup_copy27.bat %jenkinspythonexe%\n" +
    windows_error,
    "set jenkinspythonexe=__PYTHON27__\n\n" +
    "\n\n__REQUIREMENTS__\n\n" +
    "\nauto_cmd_run27.bat %jenkinspythonexe%\n" + windows_error,
    "set jenkinspythonexe=__PYTHON27__\n\n" +
    "\n\n__REQUIREMENTS__\n\n" +
    "\nauto_cmd_build27.bat %jenkinspythonexe%\n" + windows_error]

##################
#: auto setup
##################
setup_script_dependency_py = """
import sys
from distutils.core import setup, Extension
import distutils.sysconfig as SH
from setuptools import find_packages

project_var_name = "dependencies___MODULE__"
versionPython = "%s.%s" % (sys.version_info.major, sys.version_info.minor)
path = "Lib/site-packages/" + project_var_name

setup(
    name=project_var_name,
    version=versionPython,
    install_requires=[
        "numpy",
        "dateutils",
        "IPython",
        "matplotlib",
        "sphinx",
        "pandas",
        "docutils", ],
)
"""
#########################
#:
#########################

copy_dist_to_local_pypi = """
if not exist ..\local_pypi_server mkdir ..\local_pypi_server
copy /Y dist\*.whl ..\local_pypi_server
"""
<commit_msg>fix another path issue introduced for WinPython<commit_after>"""
@brief
@file Batch file use to automate some of the tasks (setup, unit tests, help, pypi).

.. versionadded:: 1.1
"""

#################
#: stop if error
#################
windows_error = "if %errorlevel% neq 0 exit /b %errorlevel%"

#################
#: prefix
#################
windows_prefix = """
if "%1"=="" goto default_value_python:
set pythonexe=%1
goto start_script:

:default_value_python:
set pythonexe=__PY34_X64__\\python
:start_script:
"""

#################
#: prefix 27
#################
windows_prefix_27 = """
if "%1"=="" goto default_value_python:
set pythonexe27=%1
goto start_script:

:default_value_python:
set pythonexe27=__PY27_X64__\\python
:start_script:
"""

#################
#: run unit test 27
#################
windows_unittest27 = """
set PYTHONPATH=
cd dist_module27\\_unittests

for /d %%d in (ut_*) do %pythonexe27%\\..\\Scripts\\nosetests.exe -w %%d

""" + windows_error + "\ncd ..\.."

#################
#: call the setup
#################
windows_setup = "%pythonexe% setup.py"
jenkins_windows_setup = "%jenkinspythonexe% setup.py"

#################
#: build script for Windows
#################
windows_build = """
IF EXIST dist del /Q dist\\*.*

set virtual_env_suffix=%2

if "%1"=="" goto default_value:
set pythonexe=%1
%pythonexe% setup.py write_version
goto custom_python:

:default_value:
IF NOT EXIST __PY34__ GOTO checkinstall64:

:checkinstall:
IF EXIST __PY34__vir GOTO nexta:
mkdir __PY34__vir

:nexta:
IF EXIST __PY34__vir\\install GOTO fullsetupa:
__PY34__\\Scripts\\virtualenv __PY34__vir\\install --system-site-packages
if %errorlevel% neq 0 exit /b %errorlevel%

:fullsetupa:
echo #######################################################0
__PY34__vir\\install\\Scripts\\python -u setup.py install
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################1

:checkinstall64:
IF EXIST __PY34_X64__vir GOTO nextb:
mkdir __PY34_X64__vir

:nextb:
IF EXIST __PY34_X64__vir\\install GOTO fullsetupb:
__PY34_X64__\\Scripts\\virtualenv __PY34_X64__vir\\install --system-site-packages
if %errorlevel% neq 0 exit /b %errorlevel%

:fullsetupb:
echo #######################################################2
__PY34_X64__vir\\install\\Scripts\\python -u setup.py install
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################3

:setup34:
IF NOT EXIST __PY34__ GOTO utpy34_64:
set pythonexe=__PY34__\\python
%pythonexe% setup.py write_version
%pythonexe% setup.py clean_pyd
%pythonexe% setup.py build bdist_wininst --plat-name=win-amd64
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################4

:utpy34_64:
set pythonexe=__PY34_X64__\\python
:custom_python:
echo ###----################################################5
if not exist ..\\virtual mkdir ..\\virtual
set virtual_env_py=..\\virtual\\__MODULE__
if not exist %pythonexe%\\..\\Scripts\\virtualenv.exe goto conda_virtual_env:

if exist %virtual_env_py%_vir rmdir /Q /S %virtual_env_py%_vir
mkdir %virtual_env_py%_vir

if exist %virtual_env_py%_vir%virtual_env_suffix%\\python goto with_virtual:
set KEEPPATH=%PATH%
set PATH=%pythonexe%\\..;%PATH%
%pythonexe%\\..\\Scripts\\virtualenv --system-site-packages %virtual_env_py%_vir%virtual_env_suffix%
set PATH=%KEEPPATH%
if %errorlevel% neq 0 exit /b %errorlevel%
:with_virtual:
set pythonexe=%virtual_env_py%_vir%virtual_env_suffix%\\Scripts\\python
set pythonpip=%virtual_env_py%_vir%virtual_env_suffix%\\Scripts\\pip
goto requirements:

:conda_virtual_env:

if exist %virtual_env_py%_condavir rmdir /Q /S %virtual_env_py%_condavir

if exist %virtual_env_py%_condavir\\python goto with_virtual_conda:
%pythonexe%\\..\\Scripts\\conda create -p %virtual_env_py%_condavir --clone %pythonexe%\\.. --offline
if %errorlevel% neq 0 exit /b %errorlevel%
:with_virtual_conda:
set pythonexe=%virtual_env_py%_condavir\\python
set pythonpip=%virtual_env_py%_condavir\\Scripts\\pip

:requirements:
echo #######################################################_auto_setup_dep.py
%pythonexe% build\\auto_setup\\auto_setup_dep.py install
if %errorlevel% neq 0 exit /b %errorlevel%

echo #######################################################_requirements_begin
echo %pythonpip%
__REQUIREMENTS__
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################_requirements_end

%pythonexe% setup.py write_version
echo #######################################################_clean
%pythonexe% -u setup.py clean_space
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################_unit
%pythonexe% -u setup.py unittests
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################6

%pythonexe% setup.py clean_pyd
%pythonexe% setup.py sdist --formats=gztar,zip --verbose
if %errorlevel% neq 0 exit /b %errorlevel%
%pythonexe% setup.py bdist_wininst --plat-name=win-amd64
if %errorlevel% neq 0 exit /b %errorlevel%
%pythonexe% setup.py bdist_msi
if %errorlevel% neq 0 exit /b %errorlevel%
%pythonexe% setup.py bdist_wheel
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################7

:documentation:
%pythonexe% -u setup.py build_sphinx
if %errorlevel% neq 0 exit /b %errorlevel%
echo #######################################################8

:copyfiles:
if not exist dist\\html mkdir dist\\html
xcopy /E /C /I /Y _doc\\sphinxdoc\\build\\html dist\\html
if exist _doc\\sphinxdoc\\build\\latex xcopy /E /C /I /Y _doc\\sphinxdoc\\build\\latex\\*.pdf dist\\html
if %errorlevel% neq 0 exit /b %errorlevel%

:end:
rem we copy the wheel on a local folder to let a pypiserver take it
if not exist ..\\local_pypi_server mkdir ..\\local_pypi_server
copy /Y dist\\*.whl ..\\local_pypi_server
"""

#################
#: notebooks
#################
windows_notebook = """
if "%1"=="" goto default_value:
set pythonexe=%1
goto nextn:

:default_value:
set pythonexe=__PY34_X64__

:nextn:
set path=%path%;%pythonexe%;%pythonexe%\\Scripts
ipython3 notebook --notebook-dir=_doc\\notebooks --matplotlib=inline
"""

#################
#: publish a module
#################
windows_publish = """
%pythonexe% setup.py rotate --match=.zip --keep=1
%pythonexe% setup.py rotate --match=.tar.gz --keep=3
rem %pythonexe% setup.py sdist register
%pythonexe% setup.py sdist --formats=gztar upload
"""

#################
#: publish the documentation
#################
windows_publish_doc = """
%pythonexe% setup.py upload_docs --upload-dir=dist/html
"""

#################
#: run a pypi server
#################
windows_pypi = """
set pythonexe=__PY34_X64__

:custom_python:
if "%2"=="" goto default_port:
set portpy=%2
goto run:

:default_port:
set portpy=__PORT__

:run:
%pythonexe%\Scripts\pypi-server.exe -u -p %portpy% --disable-fallback ..\\local_pypi_server
"""

#################
#: script for Jenkins
#################
windows_jenkins = "set jenkinspythonexe=__PYTHON__\n" + jenkins_windows_setup + " build_script\n" + \
    windows_error + "\nauto_unittest_setup_help.bat %jenkinspythonexe% __SUFFIX__\n" + \
    windows_error

#################
#: script for Jenkins
#################
windows_jenkins_27 = [
    "set jenkinspythonexe=__PYTHON__\n" + jenkins_windows_setup + " build_script\n" +
    windows_error + "\nauto_setup_copy27.bat %jenkinspythonexe%\n" +
    windows_error,
    "set jenkinspythonexe=__PYTHON27__\n\n" +
    "\n\n__REQUIREMENTS__\n\n" +
    "\nauto_cmd_run27.bat %jenkinspythonexe%\n" + windows_error,
    "set jenkinspythonexe=__PYTHON27__\n\n" +
    "\n\n__REQUIREMENTS__\n\n" +
    "\nauto_cmd_build27.bat %jenkinspythonexe%\n" + windows_error]

##################
#: auto setup
##################
setup_script_dependency_py = """
import sys
from distutils.core import setup, Extension
import distutils.sysconfig as SH
from setuptools import find_packages

project_var_name = "dependencies___MODULE__"
versionPython = "%s.%s" % (sys.version_info.major, sys.version_info.minor)
path = "Lib/site-packages/" + project_var_name

setup(
    name=project_var_name,
    version=versionPython,
    install_requires=[
        "numpy",
        "dateutils",
        "IPython",
        "matplotlib",
        "sphinx",
        "pandas",
        "docutils", ],
)
"""
#########################
#:
#########################

copy_dist_to_local_pypi = """
if not exist ..\local_pypi_server mkdir ..\local_pypi_server
copy /Y dist\*.whl ..\local_pypi_server
"""
<|endoftext|>


<commit_before>from epumgmt.api.exceptions import *
from epumgmt.main.em_core_load import get_cloudinit

from threading import Thread

THREADS_PER_BATCH = 20

class FetchThread(Thread):

    def __init__ (self, worker, c, m, scpcmd):
        Thread.__init__(self)
        self.worker = worker
        self.iid = worker.instanceid # convenience
        self.scpcmd = scpcmd
        self.c = c
        self.m = m
        self.error = None

    def run(self):
        try:
            self.m.runlogs.fetch_logs(self.scpcmd)
            self.c.log.info("Fetched logs from '%s'" % self.iid)
        except Exception,e:
            self.c.log.error("error retrieving logs from '%s'" % self.iid)
            self.error = e

def fetch_all(p, c, m, run_name, cloudinitd):
    """Fetch log files from any VM instance that is part of the run
    that we know about.
    """
    if c.trace:
        c.log.debug("fetch_all()")

    run_vms = _get_runvms_required(c, m, run_name, cloudinitd)

    threads = []
    for vm in run_vms:
        scpcmd = m.runlogs.get_scp_command_str(c, vm, cloudinitd)
        if not scpcmd:
            continue
        threads.append(FetchThread(vm, c, m, scpcmd))

    txt = "%d service" % len(run_vms)
    if len(run_vms) != 1:
        txt += "s"
    c.log.info("Beginning to logfetch %s" % txt)

    done = False
    idx = 0
    while not done:
        current_batch = threads[idx:idx+THREADS_PER_BATCH]
        idx += THREADS_PER_BATCH
        if idx > len(threads):
            done = True

        for thr in current_batch:
            thr.start()

        for thr in current_batch:
            thr.join()

    error_count = 0
    for thr in threads:
        if thr.error:
            error_count += 1
            msg = "** Issue with %s:\n" % thr.worker.instanceid
            msg += str(thr.error)
            c.log.error("\n\n%s\n" % msg)

    if error_count > 1:
        c.log.info("All fetched with %d errors (%s)" % (error_count, txt))
    elif error_count == 1:
        c.log.info("All fetched with 1 error (%s)" % txt)
    else:
        c.log.info("All fetched (%s)" % txt)

def fetch_by_vm_id(p, c, m, run_name, instanceid):
    """Fetch log files from the VM instance in this run with a particular ID

    p,c,m are seen everywhere: parameters, common, modules
    """
    c.log.debug("fetch_by_vm_id()")

    run_vms = _get_runvms_required(c, m, run_name, None)

    vm = None
    for avm in run_vms:
        if avm.instanceid == instanceid:
            vm = avm
            break
    if not vm:
        raise IncompatibleEnvironment("Cannot find an active VM associated with run '%s' with the instance id '%s'" % (run_name, instanceid))

    _fetch_one_vm(p, c, m, run_name, vm)

def fetch_by_service_name(p, c, m, run_name, servicename, cloudinitd=None):
    """Fetch log files from the VM instance(s) in this run that were started
    with the role of this service name.

    For example, when the "sleeper" EPU infrastructure is launched, you
    can fetch from nodes that were launched for this only.  If you want to
    retrieve the logs from any worker VM of an HA service that epumgmt
    has found out about, you must use the suffix of the HA service name.
    See: epumgmt.api.RunVM.WORKER_SUFFIX

    p,c,m are seen everywhere: parameters, common, modules
    """
    if c.trace:
        c.log.debug("fetch_by_service_name()")

    run_vms = _get_runvms_required(c, m, run_name, None)

    vms = []
    for avm in run_vms:
        if avm.service_type == servicename:
            vms.append(avm)

    if not len(vms):
        raise IncompatibleEnvironment("Cannot find any active VMs associated with run '%s' with the service type/name '%s'" % (run_name, servicename))

    for vm in vms:
        _fetch_one_vm(p, c, m, run_name, vm, cloudinitd=cloudinitd)

# -----------------------------------------------------------------

def _get_runvms_required(c, m, run_name, cloudinitd):
    run_vms = m.persistence.get_run_vms_or_none(run_name)
    if not run_vms or len(run_vms) == 0:
        raise IncompatibleEnvironment("Cannot find any VMs associated with run '%s'" % run_name)

    if cloudinitd:
        m.remote_svc_adapter.initialize(m, run_name, cloudinitd)
        if m.remote_svc_adapter.is_channel_open():
            c.log.info("Getting status from the EPU controllers, to filter out non-running workers from log fetch")
        else:
            c.log.warn("Cannot get worker status: there is no channel open to the EPU controllers")

    return run_vms

def _fetch_one_vm(p, c, m, run_name, vm, cloudinitd=None):
    c.log.info("fetching logs from '%s' instance '%s' (run '%s')" % (vm.service_type, vm.instanceid, run_name))
    if not cloudinitd:
        cloudinitd = get_cloudinit(p, c, m, run_name)
    scpcmd = m.runlogs.get_scp_command_str(c, vm, cloudinitd)
    m.runlogs.fetch_logs(scpcmd)
<commit_msg>Filter terminated VM workers from logfetch for ooiion-231<commit_after>from epumgmt.api.exceptions import *
from epumgmt.main.em_core_load import get_cloudinit
import epumgmt.defaults.epustates as epustates
from em_core_status import _find_state_from_events as find_state_from_events

from threading import Thread

THREADS_PER_BATCH = 20

class FetchThread(Thread):

    def __init__ (self, worker, c, m, scpcmd):
        Thread.__init__(self)
        self.worker = worker
        self.iid = worker.instanceid # convenience
        self.scpcmd = scpcmd
        self.c = c
        self.m = m
        self.error = None

    def run(self):
        try:
            self.m.runlogs.fetch_logs(self.scpcmd)
            self.c.log.info("Fetched logs from '%s'" % self.iid)
        except Exception,e:
            self.c.log.error("error retrieving logs from '%s'" % self.iid)
            self.error = e

def fetch_all(p, c, m, run_name, cloudinitd):
    """Fetch log files from any VM instance that is part of the run
    that we know about.
    """
    if c.trace:
        c.log.debug("fetch_all()")

    run_vms = _get_runvms_required(c, m, run_name, cloudinitd)

    threads = []
    for vm in run_vms:
        scpcmd = m.runlogs.get_scp_command_str(c, vm, cloudinitd)
        if not scpcmd:
            continue
        threads.append(FetchThread(vm, c, m, scpcmd))

    txt = "%d service" % len(run_vms)
    if len(run_vms) != 1:
        txt += "s"
    c.log.info("Beginning to logfetch %s" % txt)

    done = False
    idx = 0
    while not done:
        current_batch = threads[idx:idx+THREADS_PER_BATCH]
        idx += THREADS_PER_BATCH
        if idx > len(threads):
            done = True

        for thr in current_batch:
            thr.start()

        for thr in current_batch:
            thr.join()

    error_count = 0
    for thr in threads:
        if thr.error:
            error_count += 1
            msg = "** Issue with %s:\n" % thr.worker.instanceid
            msg += str(thr.error)
            c.log.error("\n\n%s\n" % msg)

    if error_count > 1:
        c.log.info("All fetched with %d errors (%s)" % (error_count, txt))
    elif error_count == 1:
        c.log.info("All fetched with 1 error (%s)" % txt)
    else:
        c.log.info("All fetched (%s)" % txt)

def fetch_by_vm_id(p, c, m, run_name, instanceid):
    """Fetch log files from the VM instance in this run with a particular ID

    p,c,m are seen everywhere: parameters, common, modules
    """
    c.log.debug("fetch_by_vm_id()")

    run_vms = _get_runvms_required(c, m, run_name, None)

    vm = None
    for avm in run_vms:
        if avm.instanceid == instanceid:
            vm = avm
            break
    if not vm:
        raise IncompatibleEnvironment("Cannot find an active VM associated with run '%s' with the instance id '%s'" % (run_name, instanceid))

    _fetch_one_vm(p, c, m, run_name, vm)

def fetch_by_service_name(p, c, m, run_name, servicename, cloudinitd=None):
    """Fetch log files from the VM instance(s) in this run that were started
    with the role of this service name.

    For example, when the "sleeper" EPU infrastructure is launched, you
    can fetch from nodes that were launched for this only.  If you want to
    retrieve the logs from any worker VM of an HA service that epumgmt
    has found out about, you must use the suffix of the HA service name.
    See: epumgmt.api.RunVM.WORKER_SUFFIX

    p,c,m are seen everywhere: parameters, common, modules
    """
    if c.trace:
        c.log.debug("fetch_by_service_name()")

    run_vms = _get_runvms_required(c, m, run_name, None)

    vms = []
    for avm in run_vms:
        if avm.service_type == servicename:
            vms.append(avm)

    if not len(vms):
        raise IncompatibleEnvironment("Cannot find any active VMs associated with run '%s' with the service type/name '%s'" % (run_name, servicename))

    for vm in vms:
        _fetch_one_vm(p, c, m, run_name, vm, cloudinitd=cloudinitd)

# -----------------------------------------------------------------

def _get_runvms_required(c, m, run_name, cloudinitd):
    run_vms = m.persistence.get_run_vms_or_none(run_name)
    if not run_vms or len(run_vms) == 0:
        raise IncompatibleEnvironment("Cannot find any VMs associated with run '%s'" % run_name)

    if cloudinitd:
        m.remote_svc_adapter.initialize(m, run_name, cloudinitd)
        if m.remote_svc_adapter.is_channel_open():
            c.log.info("Getting status from the EPU controllers, to filter out non-running workers from log fetch")
        else:
            c.log.warn("Cannot get worker status: there is no channel open to the EPU controllers")

    run_vms = filter(lambda x: find_state_from_events(x) != epustates.TERMINATED, run_vms)

    return run_vms

def _fetch_one_vm(p, c, m, run_name, vm, cloudinitd=None):
    c.log.info("fetching logs from '%s' instance '%s' (run '%s')" % (vm.service_type, vm.instanceid, run_name))
    if not cloudinitd:
        cloudinitd = get_cloudinit(p, c, m, run_name)
    scpcmd = m.runlogs.get_scp_command_str(c, vm, cloudinitd)
    m.runlogs.fetch_logs(scpcmd)
<|endoftext|>


<commit_before>import copy
import os
import shutil
import threading
import xml.etree.ElementTree as ET
from xml.dom import minidom
from tempfile import gettempdir as systmp
from time import sleep
from random import randint

import rospy
from lg_msg_defs.msg import ApplicationState, WindowGeometry
from lg_common import ManagedApplication, ManagedWindow
from .kmlalive import KmlAlive

TOOLBAR_HEIGHT = 22

CUSTOM_CONFIG_DIR = '/lg'

class Client:
    """Google Earth client launcher."""
    def __init__(self, config, initial_state=None):
        args, geplus_config, layers_config, kml_content, view_content, \
            tmpdir, instance = config
        self.tmpdir = tmpdir
        self.instance = instance

        # See whether we are running EC or Free.
        use_dir = None
        maybe_dirs = [
            '/opt/google/earth/ec',
            '/opt/google/earth/free',
        ]
        for maybe_dir in maybe_dirs:
            if os.path.isdir(maybe_dir):
                use_dir = maybe_dir
        if use_dir is None:
            raise Exception('Could not find Earth install dir in {}'.format(maybe_dirs))

        # Window placement support for the different versions.
        window_names = {
            '/opt/google/earth/ec': 'Google Earth EC',
            '/opt/google/earth/free': 'Google Earth',
        }
        window_classes = {
            '/opt/google/earth/ec': 'Google Earth EC',
            '/opt/google/earth/free': 'Googleearth-bin',
        }

        # Skip window management if this window is hidden.
        if '--hidegui' in args:
            earth_window = None
        else:
            geometry = ManagedWindow.get_viewport_geometry()
            if geometry is not None:
                geometry.y -= TOOLBAR_HEIGHT
                geometry.height += TOOLBAR_HEIGHT

            earth_window = ManagedWindow(
                geometry=geometry,
                w_class=window_classes[use_dir],
                w_name=window_names[use_dir],
                w_instance=self._get_instance()
            )

        env = copy.copy(os.environ)

        # Override the HOME location. This requires a hack to the Earth
        # libraries if we are not using Enterprise Client. See the lg_earth README.
        env['OLDHOME'] = env['HOME']
        env['HOME'] = self._get_tempdir()

        # Prevent external browser launch.
        env['BROWSER'] = '/dev/null'

        # If the Xorg DISPLAY is not in our environment, assume :0
        if os.getenv('DISPLAY') is None:
            env['DISPLAY'] = ':0'

        # Google Earth has its own copies of some libraries normally found on
        # the system, so we need to tell the loader to look there. This is
        # normally done in the google-earth wrapper script.
        env['LD_LIBRARY_PATH'] += use_dir

        cmd = [use_dir + '/googleearth-bin']

        cmd.extend(args)
        self.earth_proc = ManagedApplication(cmd, window=earth_window,
                                             initial_state=initial_state,
                                             env=env)
        KmlAlive(self.earth_proc)
        # config rendering values
        self.geplus_config = geplus_config
        self.layers_config = layers_config
        self.kml_content = kml_content
        self.view_content = view_content

        self._render_configs()

    def _render_configs(self):
        self._make_tempdir()

        os.mkdir(self._get_tempdir() + '/.googleearth')
        os.mkdir(self._get_tempdir() + '/.googleearth/Cache')

        if not rospy.get_param('~show_google_logo', True):
            source = '/home/lg/etc/localdbrootproto'
            dest = '/home/lg/.googleearth/Cache/localdbrootproto'
            if not os.path.exists(os.path.dirname(os.path.dirname(dest))):
                os.mkdir(os.path.dirname(os.path.dirname(dest)))
            if not os.path.exists(os.path.dirname(dest)):
                os.mkdir(os.path.dirname(dest))
            self._touch_file(source)
            with open(source, 'r') as src:
                with open(dest, 'w') as dst:
                    for line in src.readlines():
                        dst.write(line)

        os.mkdir(self._get_tempdir() + '/.config')
        os.mkdir(self._get_tempdir() + '/.config/Google')

        self._render_config(self.geplus_config,
                            '.config/Google/GoogleEarthEC.conf')
        self._render_config(self.layers_config,
                            '.config/Google/GECommonSettings.conf')
        self._render_file(self.kml_content,
                          '.googleearth/myplaces.kml')
        self._render_file(self.view_content,
                          '.googleearth/cached_default_view.kml')
        if not os.path.exists(self._get_tempdir() + '/.config/Google/GoogleEarthPlus.conf'):
            os.symlink(self._get_tempdir() + '/.config/Google/GoogleEarthEC.conf',
                       self._get_tempdir() + '/.config/Google/GoogleEarthPlus.conf')

        # Check whether a non-standard GECommonSettings file exists
        # and replace if so
        #self._check_for_custom_config('./config/Google/GECommonSettings.conf')

    def _touch_file(self, fname):
        """Touch a file, updating its access time.

        Args:
            fname (str): Absolute path of the file to be touched.
        """
        if os.path.exists(fname):
            os.utime(fname, None)
        else:
            open(fname, 'a').close()

    def _get_instance(self):
        """Get a unique instance name for this client.

        Returns:
            str: This node's ROS name, sanitized for use as a temp path and
                window identifier.
        """
        #return '_earth_instance_' + rospy.get_name().strip('/')
        return self.instance

    def _get_tempdir(self):
        """Get this client's unique temporary path.

        Returns:
            str: Path to this client's temporary directory.
        """
        #return os.path.normpath(systmp() + '/' + self._get_instance())
        return self.tmpdir

    def _make_tempdir(self):
        """Create a temporary directory and register cleanup on shutdown."""
        self._clean_tempdir()
        os.mkdir(self._get_tempdir())
        assert os.path.exists(self._get_tempdir())
        rospy.on_shutdown(self._clean_tempdir)
        rospy.on_shutdown(self._clear_cache)

    def _clean_tempdir(self):
        """Attempt to delete temporary directory."""
        try:
            shutil.rmtree(self._get_tempdir())
        except OSError:
            pass

    def _render_file(self, content, path):
        """Checks if a custom file is provided, if it is use it, if not
        write standard content to that file.

        Args:
            content (str)
            path (str): File path relative to the temporary directory.
        """
        if self._check_for_custom_config(path):
            self._use_custom_config(path)
            return

        with open(self._get_tempdir() + '/' + path, 'w') as f:
            f.write(content)

    def _render_config(self, config, path):
        """Render a configuration file.

        This method first checks for a custom config, if present copy it into
        place. If not it takes a dictionary and writes the values as key value
        pairs. Boolean values are written lowercase.

        Args:
            config (Dict[str, object]): Key value pairs to write to the config
                file.
            path (str): Config file path relative to the temp directory.
        """
        if self._check_for_custom_config(path):
            self._use_custom_config(path)
            return

        with open(self._get_tempdir() + '/' + path, 'w') as f:
            for section, settings in config.items():
                f.write('[' + section + ']\n')
                for k, v in settings.items():
                    r = str(v).lower() if isinstance(v, bool) else str(v)
                    f.write(k + '=' + r + '\n')
                f.write('\n')

    def _check_for_custom_config(self, standard_conf_path):
        """Checks for a custom supplied config file and if present returns
        true.

        Args:
            standard_conf_path (str): Path of the standard config
        """

        ret_val = False
        conf_filename = os.path.basename(standard_conf_path)
        custom_conf_expected_path = CUSTOM_CONFIG_DIR + '/' + self._get_tempdir() + '/' + conf_filename

        if os.path.isfile(custom_conf_expected_path):
            ret_val = True

        return ret_val

    def _use_custom_config(self, standard_conf_path):
        """Moves the custom config file to the standard config path

        Args:
            standard_conf_path (str): Path of the standard config
        """
        conf_filename = os.path.basename(standard_conf_path)
        custom_conf_expected_path = CUSTOM_CONFIG_DIR + '/' + self._get_tempdir() + '/' + conf_filename
        shutil.copy(custom_conf_expected_path,
                    self._get_tempdir() + '/' + standard_conf_path)

    def _clear_cache(self, *args, **kwargs):
        if os.path.exists('/home/lg/.googleearth/Cache'):
            try:
                shutil.rmtree('/home/lg/.googleearth/Cache')
            except OSError:
                pass  # some other instance already deleted this

    def _handle_soft_relaunch(self, msg):
        """
        Clearing up logs is pretty important for soft relaunches
        """
        rospy.logdebug('removing cache for google earth')
        try:
            # deleting out of OLDHOME because that's where the cache is stored
            earth_dir = '%s/.googleearth' % os.environ['OLDHOME']
            shutil.rmtree(earth_dir)
            self._clear_cache()
            os.mkdir(earth_dir)
        except Exception as e:
            rospy.logwarn('found error while removing earth cache: %s, could be normal operation though' % e.message)
        self._render_configs()
        self.earth_proc.handle_soft_relaunch()

    def _handle_staggered_soft_relaunch(self, msg):
        """
        Clearing up logs is pretty important for soft relaunches
        """
        random_sleep_stagger = randint(1, 10)
        rospy.logerr("Sleep Stagger: {} seconds".format(random_sleep_stagger))
        sleep(random_sleep_stagger)
        self._handle_soft_relaunch(msg)
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<commit_msg>Put Earth windows below<commit_after>import copy
import os
import shutil
import threading
import xml.etree.ElementTree as ET
from xml.dom import minidom
from tempfile import gettempdir as systmp
from time import sleep
from random import randint

import rospy
from lg_msg_defs.msg import ApplicationState, WindowGeometry
from lg_common import ManagedApplication, ManagedWindow
from .kmlalive import KmlAlive

TOOLBAR_HEIGHT = 22

CUSTOM_CONFIG_DIR = '/lg'

class Client:
    """Google Earth client launcher."""
    def __init__(self, config, initial_state=None):
        args, geplus_config, layers_config, kml_content, view_content, \
            tmpdir, instance = config
        self.tmpdir = tmpdir
        self.instance = instance

        # See whether we are running EC or Free.
        use_dir = None
        maybe_dirs = [
            '/opt/google/earth/ec',
            '/opt/google/earth/free',
        ]
        for maybe_dir in maybe_dirs:
            if os.path.isdir(maybe_dir):
                use_dir = maybe_dir
        if use_dir is None:
            raise Exception('Could not find Earth install dir in {}'.format(maybe_dirs))

        # Window placement support for the different versions.
        window_names = {
            '/opt/google/earth/ec': 'Google Earth EC',
            '/opt/google/earth/free': 'Google Earth',
        }
        window_classes = {
            '/opt/google/earth/ec': 'Google Earth EC',
            '/opt/google/earth/free': 'Googleearth-bin',
        }

        # Skip window management if this window is hidden.
        if '--hidegui' in args:
            earth_window = None
        else:
            geometry = ManagedWindow.get_viewport_geometry()
            if geometry is not None:
                geometry.y -= TOOLBAR_HEIGHT
                geometry.height += TOOLBAR_HEIGHT

            earth_window = ManagedWindow(
                geometry=geometry,
                layer='below',
                w_class=window_classes[use_dir],
                w_name=window_names[use_dir],
                w_instance=self._get_instance()
            )

        env = copy.copy(os.environ)

        # Override the HOME location. This requires a hack to the Earth
        # libraries if we are not using Enterprise Client. See the lg_earth README.
        env['OLDHOME'] = env['HOME']
        env['HOME'] = self._get_tempdir()

        # Prevent external browser launch.
        env['BROWSER'] = '/dev/null'

        # If the Xorg DISPLAY is not in our environment, assume :0
        if os.getenv('DISPLAY') is None:
            env['DISPLAY'] = ':0'

        # Google Earth has its own copies of some libraries normally found on
        # the system, so we need to tell the loader to look there. This is
        # normally done in the google-earth wrapper script.
        env['LD_LIBRARY_PATH'] += use_dir

        cmd = [use_dir + '/googleearth-bin']

        cmd.extend(args)
        self.earth_proc = ManagedApplication(cmd, window=earth_window,
                                             initial_state=initial_state,
                                             env=env)
        KmlAlive(self.earth_proc)
        # config rendering values
        self.geplus_config = geplus_config
        self.layers_config = layers_config
        self.kml_content = kml_content
        self.view_content = view_content

        self._render_configs()

    def _render_configs(self):
        self._make_tempdir()

        os.mkdir(self._get_tempdir() + '/.googleearth')
        os.mkdir(self._get_tempdir() + '/.googleearth/Cache')

        if not rospy.get_param('~show_google_logo', True):
            source = '/home/lg/etc/localdbrootproto'
            dest = '/home/lg/.googleearth/Cache/localdbrootproto'
            if not os.path.exists(os.path.dirname(os.path.dirname(dest))):
                os.mkdir(os.path.dirname(os.path.dirname(dest)))
            if not os.path.exists(os.path.dirname(dest)):
                os.mkdir(os.path.dirname(dest))
            self._touch_file(source)
            with open(source, 'r') as src:
                with open(dest, 'w') as dst:
                    for line in src.readlines():
                        dst.write(line)

        os.mkdir(self._get_tempdir() + '/.config')
        os.mkdir(self._get_tempdir() + '/.config/Google')

        self._render_config(self.geplus_config,
                            '.config/Google/GoogleEarthEC.conf')
        self._render_config(self.layers_config,
                            '.config/Google/GECommonSettings.conf')
        self._render_file(self.kml_content,
                          '.googleearth/myplaces.kml')
        self._render_file(self.view_content,
                          '.googleearth/cached_default_view.kml')
        if not os.path.exists(self._get_tempdir() + '/.config/Google/GoogleEarthPlus.conf'):
            os.symlink(self._get_tempdir() + '/.config/Google/GoogleEarthEC.conf',
                       self._get_tempdir() + '/.config/Google/GoogleEarthPlus.conf')

        # Check whether a non-standard GECommonSettings file exists
        # and replace if so
        #self._check_for_custom_config('./config/Google/GECommonSettings.conf')

    def _touch_file(self, fname):
        """Touch a file, updating its access time.

        Args:
            fname (str): Absolute path of the file to be touched.
        """
        if os.path.exists(fname):
            os.utime(fname, None)
        else:
            open(fname, 'a').close()

    def _get_instance(self):
        """Get a unique instance name for this client.

        Returns:
            str: This node's ROS name, sanitized for use as a temp path and
                window identifier.
        """
        #return '_earth_instance_' + rospy.get_name().strip('/')
        return self.instance

    def _get_tempdir(self):
        """Get this client's unique temporary path.

        Returns:
            str: Path to this client's temporary directory.
        """
        #return os.path.normpath(systmp() + '/' + self._get_instance())
        return self.tmpdir

    def _make_tempdir(self):
        """Create a temporary directory and register cleanup on shutdown."""
        self._clean_tempdir()
        os.mkdir(self._get_tempdir())
        assert os.path.exists(self._get_tempdir())
        rospy.on_shutdown(self._clean_tempdir)
        rospy.on_shutdown(self._clear_cache)

    def _clean_tempdir(self):
        """Attempt to delete temporary directory."""
        try:
            shutil.rmtree(self._get_tempdir())
        except OSError:
            pass

    def _render_file(self, content, path):
        """Checks if a custom file is provided, if it is use it, if not
        write standard content to that file.

        Args:
            content (str)
            path (str): File path relative to the temporary directory.
        """
        if self._check_for_custom_config(path):
            self._use_custom_config(path)
            return

        with open(self._get_tempdir() + '/' + path, 'w') as f:
            f.write(content)

    def _render_config(self, config, path):
        """Render a configuration file.

        This method first checks for a custom config, if present copy it into
        place. If not it takes a dictionary and writes the values as key value
        pairs. Boolean values are written lowercase.

        Args:
            config (Dict[str, object]): Key value pairs to write to the config
                file.
            path (str): Config file path relative to the temp directory.
        """
        if self._check_for_custom_config(path):
            self._use_custom_config(path)
            return

        with open(self._get_tempdir() + '/' + path, 'w') as f:
            for section, settings in config.items():
                f.write('[' + section + ']\n')
                for k, v in settings.items():
                    r = str(v).lower() if isinstance(v, bool) else str(v)
                    f.write(k + '=' + r + '\n')
                f.write('\n')

    def _check_for_custom_config(self, standard_conf_path):
        """Checks for a custom supplied config file and if present returns
        true.

        Args:
            standard_conf_path (str): Path of the standard config
        """

        ret_val = False
        conf_filename = os.path.basename(standard_conf_path)
        custom_conf_expected_path = CUSTOM_CONFIG_DIR + '/' + self._get_tempdir() + '/' + conf_filename

        if os.path.isfile(custom_conf_expected_path):
            ret_val = True

        return ret_val

    def _use_custom_config(self, standard_conf_path):
        """Moves the custom config file to the standard config path

        Args:
            standard_conf_path (str): Path of the standard config
        """
        conf_filename = os.path.basename(standard_conf_path)
        custom_conf_expected_path = CUSTOM_CONFIG_DIR + '/' + self._get_tempdir() + '/' + conf_filename
        shutil.copy(custom_conf_expected_path,
                    self._get_tempdir() + '/' + standard_conf_path)

    def _clear_cache(self, *args, **kwargs):
        if os.path.exists('/home/lg/.googleearth/Cache'):
            try:
                shutil.rmtree('/home/lg/.googleearth/Cache')
            except OSError:
                pass  # some other instance already deleted this

    def _handle_soft_relaunch(self, msg):
        """
        Clearing up logs is pretty important for soft relaunches
        """
        rospy.logdebug('removing cache for google earth')
        try:
            # deleting out of OLDHOME because that's where the cache is stored
            earth_dir = '%s/.googleearth' % os.environ['OLDHOME']
            shutil.rmtree(earth_dir)
            self._clear_cache()
            os.mkdir(earth_dir)
        except Exception as e:
            rospy.logwarn('found error while removing earth cache: %s, could be normal operation though' % e.message)
        self._render_configs()
        self.earth_proc.handle_soft_relaunch()

    def _handle_staggered_soft_relaunch(self, msg):
        """
        Clearing up logs is pretty important for soft relaunches
        """
        random_sleep_stagger = randint(1, 10)
        rospy.logerr("Sleep Stagger: {} seconds".format(random_sleep_stagger))
        sleep(random_sleep_stagger)
        self._handle_soft_relaunch(msg)
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before>"""
Copyright (c) 2015, Steve Rubin

All rights reserved.

Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

    * Redistributions of source code must retain the above copyright notice,
      this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright notice,
      this list of conditions and the following disclaimer in the documentation
      and/or other materials provided with the distribution.
    * Neither the name of {{ project }} nor the names of its contributors
      may be used to endorse or promote products derived from this software
      without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

import pdb
import os
import sys
import numpy as np

#in the MTX: #rows, #cols, #nnz
#in the MTX: row, col, val
#pdb.set_trace()
if(len(sys.argv) != 2):
	print "ERROR: Please specify the matrix filename"
	#print str(sys.argv[0])
	sys.exit()

def check_for_comments(lines_list):
	number_of_comment_lines = 0

	new_line = matrix_list[number_of_comment_lines]

	while(new_line.find("%") != -1):
		number_of_comment_lines += 1
		new_line = matrix_list[number_of_comment_lines]
	return number_of_comment_lines

matrix_file = open(sys.argv[1], "r")

#read all the lines of a file in a string.
matrix_string = matrix_file.read()
matrix_file.close()

#read all the lines of a file in a list.
matrix_file = open(sys.argv[1], "r")
matrix_list = matrix_file.readlines()
matrix_file.close()
#print kernel_string

print ""

number_of_comment_lines = check_for_comments(matrix_list)
if (number_of_comment_lines != 0 ):
	print "found " + str(number_of_comment_lines) + " lines of comments"

for x in range(0, number_of_comment_lines):
	del matrix_list[0]
	print "deleting line #" + str(x) + " as it is a comment"

#for i in matrix_list:
#	print i

matrix_numbers_list = []
for i in range(0, len(matrix_list)):
	matrix_numbers_list.append(matrix_list[i].split())

#for i in matrix_numbers_list:
#	print i

#print str(len(matrix_numbers_list))
#delete empty first and last elements of the list
#del matrix_numbers_list[0]
#del matrix_numbers_list[len(matrix_numbers_list) - 1]

#if the number of elements in the list is 3 - row, col, val
if(len(matrix_numbers_list[1]) == 3):
	print "the mtx file contains non binary values"
	for i in range(0, len(matrix_numbers_list)):
		matrix_numbers_list[i][0] = int(matrix_numbers_list[i][0])
		matrix_numbers_list[i][1] = int(matrix_numbers_list[i][1])
		matrix_numbers_list[i][2] = float(matrix_numbers_list[i][2])
else:
#if the number of elements in the list is 2 - row, col, 1
	print "the mtx file contains binary values"
	for i in range(0, len(matrix_numbers_list)):
		matrix_numbers_list[i][0] = int(matrix_numbers_list[i][0])
		matrix_numbers_list[i][1] = int(matrix_numbers_list[i][1])
		matrix_numbers_list[i].append(float(1))

#matrix_numbers_list = [int(i) for i in results]

print ""
number_of_rows_cols_nnz = matrix_numbers_list[0]
del matrix_numbers_list[0]
number_of_rows = number_of_rows_cols_nnz[0]
number_of_cols = number_of_rows_cols_nnz[1]
number_of_nnz = int(number_of_rows_cols_nnz[2])
print "#rows - " + str(number_of_rows)
print "#cols - " + str(number_of_cols)
print "#nnz - " + str(number_of_nnz)

##AT THIS POINT we have matrix_numbers_list which is  a list of lists each of them is one element in mtx file
##AT THIS POINT we have number_of_rows_cols_nnz

print ""
list_to_delete = []
number_of_nnz_deleted = 0
for i in range(0, len(matrix_numbers_list)):
	#print i
	if(matrix_numbers_list[i][2] == 0):
		list_to_delete.append(i)

for i in range(0, len(list_to_delete)):
	print "deleting " + str(matrix_numbers_list[list_to_delete[i]]) + " as the value is zero"
	del matrix_numbers_list[list_to_delete[i]]
	number_of_nnz_deleted += 1

print ""
#for i in matrix_numbers_list:
#	print i

list_to_delete = []

##AT THIS POINT we have matrix_numbers_lista with no 0 elements
for i in range(0, len(matrix_numbers_list)):
	#print i
	element_column = matrix_numbers_list[i][1]
	element_row = matrix_numbers_list[i][0]
	element_value = matrix_numbers_list[i][2]
	for j in range ((i+1), len(matrix_numbers_list)):
		if(matrix_numbers_list[j][1] == element_row and matrix_numbers_list[j][0] == element_column
			and matrix_numbers_list[j][2] == element_value):

			# this is part of my other fix -CD
			list_to_delete.append(matrix_numbers_list[j])

#ERROR HERE: with indexes
	# this code generates index errors, so so I rewrote it -CD
"""for i in range(0, len(list_to_delete)):
	print "deleting " + str(matrix_numbers_list[list_to_delete[i]]) + " as the value has a duplicate"
	del matrix_numbers_list[list_to_delete[i]]
	number_of_nnz_deleted += 1
"""
for duplicate in list_to_delete:
	print("removing duplicate entry {0} from matrix".format(duplicate))

	# this makes sure we are getting the *current* index of the duplicate
	indexOfDuplicate = matrix_numbers_list.index(duplicate)
	matrix_numbers_list.pop(indexOfDuplicate) # list.pop() is more "correct"

	number_of_nnz_deleted != 1 # compatibility with the rest of the code

number_of_nnz = number_of_nnz - number_of_nnz_deleted

print ""
#print "#rows - " + str(number_of_rows)
#print "#cols - " + str(number_of_cols)
#print "#nnz - " + str(number_of_nnz)
#print ""

for i in matrix_numbers_list:
	if (i[0] > i[1]):
		print "found an element in the bottom triangle " + str(i) + " moving him to top"
		temp = i[0]
		i[0] = i[1]
		i[1] = temp

##AT THIS POINT we have matrix_numbers_list with no 0 elements, no duplicate elements, no elements in the botton triangle

for i in range(0, len(matrix_numbers_list)):
	matrix_numbers_list[i] = tuple(matrix_numbers_list[i])

#print matrix_numbers_list
data_type = [('row', int), ('col', int), ('val', float)]
#data_type = 'i4, i4, f4'
np_matrix_numbers_list =  np.array(matrix_numbers_list, dtype = data_type)
#print np_matrix_numbers_list

sorted_matrix = np.sort(np_matrix_numbers_list, order=["row", "col"])
print ""
#print sorted_matrix

##AT THIS POINT we have sorted_matrix with no 0 elements, no duplicate elements, sorted by row first col second, no elements in the bottom triangle
## export ti .mtx file
new_mtx_file_name = str(sys.argv[1]) + ".pmtx"
new_mtx_file = open(new_mtx_file_name, "w")
new_mtx_file.write("%%MatrixMarket matrix coordinate real symmetric\n")
new_mtx_file.write(str(number_of_rows_cols_nnz[0]) + " " + str(number_of_rows_cols_nnz[1]) + " " + str(int(number_of_rows_cols_nnz[2])))
new_mtx_file.write("\n")

ptr = []
#ptr.extend([0])
ptr_counter = 0
for i in sorted_matrix:
	new_mtx_file.write(str(i[0]) + " " + str(i[1]) + " " + str(i[2]))
	new_mtx_file.write("\n")
	if (ptr_counter + 1 == i[0]):
		ptr_counter += 1
		itemindex = np.where(sorted_matrix==i)
		ptr.append(itemindex[0])
		#

ptr.append([number_of_nnz])
new_mtx_file.close()

## export ti .valcol file
new_valcol_file_name = str(sys.argv[1]) + ".pvalcol"
new_valcol_file = open(new_valcol_file_name, "w")

new_valcol_file.write(str(number_of_rows) + " " + str(number_of_nnz))
new_valcol_file.write("\n")

#create ptr array for the valcol file
for i in sorted_matrix:
	new_valcol_file.write(str(i[2]) + " " + str(i[1]))
	new_valcol_file.write("\n")

for i in ptr:
	new_valcol_file.write(str(i[0]))
	new_valcol_file.write("\n")

new_valcol_file.close()

print "		!SUCCESS!"
print "		please check synsymmetric matrices: " + new_mtx_file_name + " and " + new_valcol_file_name
print ""

<commit_msg>Steve's generate_symmetric_matrices code has now been fully replaced in function by HercExplorer<commit_after><|endoftext|>