<commit_before>"""
Custom generic managers
"""
import django
from django.core.exceptions import ImproperlyConfigured
from django.db import models
from django.db.models.query import QuerySet
from django.utils.translation import get_language
from django.utils import six
from parler import appsettings
from parler.utils import get_active_language_choices

class TranslatableQuerySet(QuerySet):
    """
    An enhancement of the QuerySet which sets the objects language before they are returned.

    When using this class in combination with *django-polymorphic*, make sure this
    class is first in the chain of inherited classes.
    """

    def __init__(self, *args, **kwargs):
        super(TranslatableQuerySet, self).__init__(*args, **kwargs)
        self._language = None

    def _clone(self, klass=None, setup=False, **kw):
        if django.VERSION < (1, 9):
            kw['klass'] = klass
            kw['setup'] = setup
        c = super(TranslatableQuerySet, self)._clone(**kw)
        c._language = self._language
        return c

    def create(self, **kwargs):
        # Pass language setting to the object, as people start assuming things
        # like .language('xx').create(..) which is a nice API after all.
        if self._language:
            kwargs['_current_language'] = self._language
        return super(TranslatableQuerySet, self).create(**kwargs)

    def _fetch_all(self):
        # Make sure the current language is assigned when Django fetches the data.
        # This low-level method is overwritten as that works better across Django versions.
        # Alternatives include:
        # - overwriting iterator() for Django <= 1.10
        # - hacking _iterable_class, which breaks django-polymorphic
        super(TranslatableQuerySet, self)._fetch_all()
        if self._language is not None and self._result_cache and isinstance(self._result_cache[0], models.Model):
            for obj in self._result_cache:
                obj.set_current_language(self._language)

    def _extract_model_params(self, defaults, **kwargs):
        # default implementation in Django>=1.11 doesn't allow non-field attributes,
        # so process them manually
        translated_defaults = {}
        for field in self.model._parler_meta.get_all_fields():
            try:
                translated_defaults[field] = defaults.pop(field)
            except KeyError:
                pass

        lookup, params = super(TranslatableQuerySet, self)._extract_model_params(defaults, **kwargs)
        params.update(translated_defaults)

        if (1, 7) <= django.VERSION < (1, 8):
            if self._language:
                params['_current_language'] = self._language

        return lookup, params

    def language(self, language_code=None):
        """
        Set the language code to assign to objects retrieved using this QuerySet.
        """
        if language_code is None:
            language_code = appsettings.PARLER_LANGUAGES.get_default_language()

        self._language = language_code
        return self

    def translated(self, *language_codes, **translated_fields):
        """
        Only return translated objects which of the given languages.

        When no language codes are given, only the currently active language is returned.

        .. note::

            Due to Django `ORM limitations <https://docs.djangoproject.com/en/dev/topics/db/queries/#spanning-multi-valued-relationships>`_,
            this method can't be combined with other filters that access the translated fields. As such, query the fields in one filter:

            .. code-block:: python

                qs.translated('en', name="Cheese Omelette")

            This will query the translated model for the ``name`` field.
        """
        relname = self.model._parler_meta.root_rel_name

        if not language_codes:
            language_codes = (get_language(),)

        filters = {}
        for field_name, val in six.iteritems(translated_fields):
            if field_name.startswith('master__'):
                filters[field_name[8:]] = val  # avoid translations__master__ back and forth
            else:
                filters["{0}__{1}".format(relname, field_name)] = val

        if len(language_codes) == 1:
            filters[relname + '__language_code'] = language_codes[0]
            return self.filter(**filters)
        else:
            filters[relname + '__language_code__in'] = language_codes
            return self.filter(**filters).distinct()

    def active_translations(self, language_code=None, **translated_fields):
        """
        Only return objects which are translated, or have a fallback that should be displayed.

        Typically that's the currently active language and fallback language.
        This should be combined with ``.distinct()``.

        When ``hide_untranslated = True``, only the currently active language will be returned.
        """
        # Default:     (language, fallback) when hide_translated == False
        # Alternative: (language,)          when hide_untranslated == True
        language_codes = get_active_language_choices(language_code)
        return self.translated(*language_codes, **translated_fields)

class TranslatableManager(models.Manager):
    """
    The manager class which ensures the enhanced TranslatableQuerySet object is used.
    """
    queryset_class = TranslatableQuerySet

    def get_queryset(self):
        if not issubclass(self.queryset_class, TranslatableQuerySet):
            raise ImproperlyConfigured("{0}.queryset_class does not inherit from TranslatableQuerySet".format(self.__class__.__name__))
        return self.queryset_class(self.model, using=self._db)

    # Leave for Django 1.6/1.7, so backwards compatibility can be fixed.
    # It will be removed in Django 1.8, so remove it here too to avoid false promises.
    if django.VERSION < (1, 8):
        get_query_set = get_queryset

    # NOTE: Fetching the queryset is done by calling self.all() here on purpose.
    # By using .all(), the proper get_query_set()/get_queryset() will be used for each Django version.

    def language(self, language_code=None):
        """
        Set the language code to assign to objects retrieved using this Manager.
        """
        return self.all().language(language_code)

    def translated(self, *language_codes, **translated_fields):
        """
        Only return objects which are translated in the given languages.

        NOTE: due to Django `ORM limitations <https://docs.djangoproject.com/en/dev/topics/db/queries/#spanning-multi-valued-relationships>`_,
        this method can't be combined with other filters that access the translated fields. As such, query the fields in one filter:

        .. code-block:: python

            qs.translated('en', name="Cheese Omelette")

        This will query the translated model for the ``name`` field.
        """
        return self.all().translated(*language_codes, **translated_fields)

    def active_translations(self, language_code=None, **translated_fields):
        """
        Only return objects which are translated, or have a fallback that should be displayed.

        Typically that's the currently active language and fallback language.
        This should be combined with ``.distinct()``.

        When ``hide_untranslated = True``, only the currently active language will be returned.
        """
        return self.all().active_translations(language_code, **translated_fields)

# Export the names in django-hvad style too:
TranslationQueryset = TranslatableQuerySet
TranslationManager = TranslatableManager
<commit_msg>fix get_or_create() when no defaults are given<commit_after>"""
Custom generic managers
"""
import django
from django.core.exceptions import ImproperlyConfigured
from django.db import models
from django.db.models.query import QuerySet
from django.utils.translation import get_language
from django.utils import six
from parler import appsettings
from parler.utils import get_active_language_choices

class TranslatableQuerySet(QuerySet):
    """
    An enhancement of the QuerySet which sets the objects language before they are returned.

    When using this class in combination with *django-polymorphic*, make sure this
    class is first in the chain of inherited classes.
    """

    def __init__(self, *args, **kwargs):
        super(TranslatableQuerySet, self).__init__(*args, **kwargs)
        self._language = None

    def _clone(self, klass=None, setup=False, **kw):
        if django.VERSION < (1, 9):
            kw['klass'] = klass
            kw['setup'] = setup
        c = super(TranslatableQuerySet, self)._clone(**kw)
        c._language = self._language
        return c

    def create(self, **kwargs):
        # Pass language setting to the object, as people start assuming things
        # like .language('xx').create(..) which is a nice API after all.
        if self._language:
            kwargs['_current_language'] = self._language
        return super(TranslatableQuerySet, self).create(**kwargs)

    def _fetch_all(self):
        # Make sure the current language is assigned when Django fetches the data.
        # This low-level method is overwritten as that works better across Django versions.
        # Alternatives include:
        # - overwriting iterator() for Django <= 1.10
        # - hacking _iterable_class, which breaks django-polymorphic
        super(TranslatableQuerySet, self)._fetch_all()
        if self._language is not None and self._result_cache and isinstance(self._result_cache[0], models.Model):
            for obj in self._result_cache:
                obj.set_current_language(self._language)

    def _extract_model_params(self, defaults, **kwargs):
        # default implementation in Django>=1.11 doesn't allow non-field attributes,
        # so process them manually
        translated_defaults = {}
        if defaults:
            for field in self.model._parler_meta.get_all_fields():
                try:
                    translated_defaults[field] = defaults.pop(field)
                except KeyError:
                    pass

        lookup, params = super(TranslatableQuerySet, self)._extract_model_params(defaults, **kwargs)
        params.update(translated_defaults)

        if (1, 7) <= django.VERSION < (1, 8):
            if self._language:
                params['_current_language'] = self._language

        return lookup, params

    def language(self, language_code=None):
        """
        Set the language code to assign to objects retrieved using this QuerySet.
        """
        if language_code is None:
            language_code = appsettings.PARLER_LANGUAGES.get_default_language()

        self._language = language_code
        return self

    def translated(self, *language_codes, **translated_fields):
        """
        Only return translated objects which of the given languages.

        When no language codes are given, only the currently active language is returned.

        .. note::

            Due to Django `ORM limitations <https://docs.djangoproject.com/en/dev/topics/db/queries/#spanning-multi-valued-relationships>`_,
            this method can't be combined with other filters that access the translated fields. As such, query the fields in one filter:

            .. code-block:: python

                qs.translated('en', name="Cheese Omelette")

            This will query the translated model for the ``name`` field.
        """
        relname = self.model._parler_meta.root_rel_name

        if not language_codes:
            language_codes = (get_language(),)

        filters = {}
        for field_name, val in six.iteritems(translated_fields):
            if field_name.startswith('master__'):
                filters[field_name[8:]] = val  # avoid translations__master__ back and forth
            else:
                filters["{0}__{1}".format(relname, field_name)] = val

        if len(language_codes) == 1:
            filters[relname + '__language_code'] = language_codes[0]
            return self.filter(**filters)
        else:
            filters[relname + '__language_code__in'] = language_codes
            return self.filter(**filters).distinct()

    def active_translations(self, language_code=None, **translated_fields):
        """
        Only return objects which are translated, or have a fallback that should be displayed.

        Typically that's the currently active language and fallback language.
        This should be combined with ``.distinct()``.

        When ``hide_untranslated = True``, only the currently active language will be returned.
        """
        # Default:     (language, fallback) when hide_translated == False
        # Alternative: (language,)          when hide_untranslated == True
        language_codes = get_active_language_choices(language_code)
        return self.translated(*language_codes, **translated_fields)

class TranslatableManager(models.Manager):
    """
    The manager class which ensures the enhanced TranslatableQuerySet object is used.
    """
    queryset_class = TranslatableQuerySet

    def get_queryset(self):
        if not issubclass(self.queryset_class, TranslatableQuerySet):
            raise ImproperlyConfigured("{0}.queryset_class does not inherit from TranslatableQuerySet".format(self.__class__.__name__))
        return self.queryset_class(self.model, using=self._db)

    # Leave for Django 1.6/1.7, so backwards compatibility can be fixed.
    # It will be removed in Django 1.8, so remove it here too to avoid false promises.
    if django.VERSION < (1, 8):
        get_query_set = get_queryset

    # NOTE: Fetching the queryset is done by calling self.all() here on purpose.
    # By using .all(), the proper get_query_set()/get_queryset() will be used for each Django version.

    def language(self, language_code=None):
        """
        Set the language code to assign to objects retrieved using this Manager.
        """
        return self.all().language(language_code)

    def translated(self, *language_codes, **translated_fields):
        """
        Only return objects which are translated in the given languages.

        NOTE: due to Django `ORM limitations <https://docs.djangoproject.com/en/dev/topics/db/queries/#spanning-multi-valued-relationships>`_,
        this method can't be combined with other filters that access the translated fields. As such, query the fields in one filter:

        .. code-block:: python

            qs.translated('en', name="Cheese Omelette")

        This will query the translated model for the ``name`` field.
        """
        return self.all().translated(*language_codes, **translated_fields)

    def active_translations(self, language_code=None, **translated_fields):
        """
        Only return objects which are translated, or have a fallback that should be displayed.

        Typically that's the currently active language and fallback language.
        This should be combined with ``.distinct()``.

        When ``hide_untranslated = True``, only the currently active language will be returned.
        """
        return self.all().active_translations(language_code, **translated_fields)

# Export the names in django-hvad style too:
TranslationQueryset = TranslatableQuerySet
TranslationManager = TranslatableManager
<|endoftext|>


<commit_before># coding=utf-8

import datetime
import logging
import traceback
import types

from babelfish import Language

from constants import mode_map

logger = logging.getLogger(__name__)

class SubtitleHistoryItem(object):
    item_title = None
    section_title = None
    rating_key = None
    provider_name = None
    lang_name = None
    lang_data = None
    score = None
    time = None
    mode = "a"

    def __init__(self, item_title, rating_key, section_title=None, subtitle=None, mode="a", time=None):
        self.item_title = item_title
        self.section_title = section_title
        self.rating_key = str(rating_key)
        self.provider_name = subtitle.provider_name
        self.lang_name = str(subtitle.language.name)
        self.lang_data = str(subtitle.language.alpha3), str(subtitle.language.country), str(subtitle.language.script)
        self.score = subtitle.score
        self.time = time or datetime.datetime.now()
        self.mode = mode

    @property
    def title(self):
        return u"%s: %s" % (self.section_title, self.item_title)

    @property
    def language(self):
        if self.lang_data:
            return Language(self.lang_data[0], country=self.lang_data[1], script=self.lang_data[2])

    @property
    def mode_verbose(self):
        return mode_map.get(self.mode, "Unknown")

    def __repr__(self):
        return unicode(self)

    def __unicode__(self):
        return u"%s (Score: %s)" % (unicode(self.item_title), self.score)

    def __str__(self):
        return str(self.rating_key)

    def __hash__(self):
        return hash((self.rating_key, self.score))

    def __eq__(self, other):
        return (self.rating_key, self.score) == (other.rating_key, other.score)

    def __ne__(self, other):
        # Not strictly necessary, but to avoid having both x==y and x!=y
        # True at the same time
        return not (self == other)

class SubtitleHistory(object):
    size = 100
    storage = None

    def __init__(self, storage, size=100):
        self.size = size
        self.storage = storage

    def add(self, item_title, rating_key, section_title=None, subtitle=None, mode="a", time=None):
        items = self.items

        item = SubtitleHistoryItem(item_title, rating_key, section_title=section_title, subtitle=subtitle, mode=mode, time=time)

        # insert item
        items.insert(0, item)

        # clamp item amount
        items = items[:self.size]

        # store items
        self.storage.SaveObject("subtitle_history", items)

    @property
    def items(self):
        try:
            items = self.storage.LoadObject("subtitle_history") or []
        except:
            items = []
            logger.error("Failed to load history storage: %s" % traceback.format_exc())

        if not isinstance(items, types.ListType):
            items = []
        else:
            items = items[:]
        return items

    def destroy(self):
        self.storage = None

<commit_msg>HistoryStorage: don't fail on Language None values<commit_after># coding=utf-8

import datetime
import logging
import traceback
import types

from babelfish import Language

from constants import mode_map

logger = logging.getLogger(__name__)

class SubtitleHistoryItem(object):
    item_title = None
    section_title = None
    rating_key = None
    provider_name = None
    lang_name = None
    lang_data = None
    score = None
    time = None
    mode = "a"

    def __init__(self, item_title, rating_key, section_title=None, subtitle=None, mode="a", time=None):
        self.item_title = item_title
        self.section_title = section_title
        self.rating_key = str(rating_key)
        self.provider_name = subtitle.provider_name
        self.lang_name = str(subtitle.language.name)
        self.lang_data = str(subtitle.language.alpha3), \
                         str(subtitle.language.country) if subtitle.language.country else None, \
                         str(subtitle.language.script) if subtitle.language.script else None
        self.score = subtitle.score
        self.time = time or datetime.datetime.now()
        self.mode = mode

    @property
    def title(self):
        return u"%s: %s" % (self.section_title, self.item_title)

    @property
    def language(self):
        if self.lang_data:
            lang_data = [s if s != "None" else None for s in self.lang_data]
            if lang_data[0]:
                return Language(lang_data[0], country=lang_data[1], script=lang_data[2])

    @property
    def mode_verbose(self):
        return mode_map.get(self.mode, "Unknown")

    def __repr__(self):
        return unicode(self)

    def __unicode__(self):
        return u"%s (Score: %s)" % (unicode(self.item_title), self.score)

    def __str__(self):
        return str(self.rating_key)

    def __hash__(self):
        return hash((self.rating_key, self.score))

    def __eq__(self, other):
        return (self.rating_key, self.score) == (other.rating_key, other.score)

    def __ne__(self, other):
        # Not strictly necessary, but to avoid having both x==y and x!=y
        # True at the same time
        return not (self == other)

class SubtitleHistory(object):
    size = 100
    storage = None

    def __init__(self, storage, size=100):
        self.size = size
        self.storage = storage

    def add(self, item_title, rating_key, section_title=None, subtitle=None, mode="a", time=None):
        items = self.items

        item = SubtitleHistoryItem(item_title, rating_key, section_title=section_title, subtitle=subtitle, mode=mode, time=time)

        # insert item
        items.insert(0, item)

        # clamp item amount
        items = items[:self.size]

        # store items
        self.storage.SaveObject("subtitle_history", items)

    @property
    def items(self):
        try:
            items = self.storage.LoadObject("subtitle_history") or []
        except:
            items = []
            logger.error("Failed to load history storage: %s" % traceback.format_exc())

        if not isinstance(items, types.ListType):
            items = []
        else:
            items = items[:]
        return items

    def destroy(self):
        self.storage = None

<|endoftext|>


<commit_before>from OrthoEvol.Manager.database_management import DatabaseManagement

class DatabaseDispatcher(DatabaseManagement):

    def __init__(self, config_file, proj_mana, upload_refseq_release=False, **kwargs):
        super().__init__(config_file=config_file, proj_mana=proj_mana)
        self.dispatcher, self.configuration = self.get_strategy_dispatcher(db_config_strategy=self.db_config_strategy)
        self.strategies = list(self.dispatcher.keys())
        self.actions = ["archive", "upload", "configure", "delete"]
        # if len(self.dispatcher.keys()) == 1:
        #     self.strategy = list(self.dispatcher.keys())[0]
        #
        #     # Create class methods from dispatcher
        #     self.archive_disp = self.dispatcher[self.strategy]["archive"]
        #     self.configure_disp = self.dispatcher[self.strategy]["configure"]
        #     self.upload_disp = self.dispatcher[self.strategy]["upload"]
        #
        #     # Create class attributes from configuration
        #     self.archive_config = self.configuration[self.strategy]["archive"]
        #     self.configure_config = self.configuration[self.strategy]["configure"]
        #     self.upload_config = self.configuration[self.strategy]["configure"]
        # else:
        #     for disp, c in zip(self.dispatcher, self.configuration):

        if upload_refseq_release == True:
            self.refseq_release_dispatcher(**kwargs)

    def dispatch(self, strategies, dispatcher, configuration):
        for strategy in strategies:
            disp = dispatcher[strategy]
            conf = configuration[strategy]
            if isinstance(disp, list):
                for funk, kw in zip(disp, conf):
                    funk(**kw)
            elif isinstance(disp, dict):
                strats = list(disp.keys())
                self.dispatch(strats, disp, conf)

    def uploading_dispatch(self, strategies):
        dispatcher = {}
        configuration = {}
        for strategy in strategies:
            dispatcher[strategy] = []
            configuration[strategy] = []
            if strategy == "NCBI_refseq_release":
                self.configuration[strategy]['configure_flag'] = False
                self.configuration[strategy]['upload_flag'] = True
                disp, conf = self.NCBI_refseq_release(**self.configuration[strategy])
                dispatcher[strategy].append(disp)
                configuration[strategy].append(conf)

        self.dispatch(strategies, dispatcher, configuration)

    def refseq_release_dispatcher(self, **kwargs):
        self.upload_refseq_release_files(**kwargs)

<commit_msg>Removed commented code in database_dispatcher.<commit_after>from OrthoEvol.Manager.database_management import DatabaseManagement

class DatabaseDispatcher(DatabaseManagement):

    def __init__(self, config_file, proj_mana, upload_refseq_release=False, **kwargs):
        super().__init__(config_file=config_file, proj_mana=proj_mana)
        self.dispatcher, self.configuration = self.get_strategy_dispatcher(db_config_strategy=self.db_config_strategy)
        self.strategies = list(self.dispatcher.keys())
        self.actions = ["archive", "upload", "configure", "delete"]

        if upload_refseq_release == True:
            self.refseq_release_dispatcher(**kwargs)

    def dispatch(self, strategies, dispatcher, configuration):
        for strategy in strategies:
            disp = dispatcher[strategy]
            conf = configuration[strategy]
            if isinstance(disp, list):
                for funk, kw in zip(disp, conf):
                    funk(**kw)
            elif isinstance(disp, dict):
                strats = list(disp.keys())
                self.dispatch(strats, disp, conf)

    def uploading_dispatch(self, strategies):
        dispatcher = {}
        configuration = {}
        for strategy in strategies:
            dispatcher[strategy] = []
            configuration[strategy] = []
            if strategy == "NCBI_refseq_release":
                self.configuration[strategy]['configure_flag'] = False
                self.configuration[strategy]['upload_flag'] = True
                disp, conf = self.NCBI_refseq_release(**self.configuration[strategy])
                dispatcher[strategy].append(disp)
                configuration[strategy].append(conf)

        self.dispatch(strategies, dispatcher, configuration)

    def refseq_release_dispatcher(self, **kwargs):
        self.upload_refseq_release_files(**kwargs)

<|endoftext|>


<commit_before>from urlparse import urlparse

from twitcher.client import TwitcherService

from phoenix.db import mongodb

def includeme(config):
    settings = config.registry.settings

    config.include('twitcher.rpcinterface')
    config.include('twitcher.owsproxy')
    config.include('twitcher.tweens')

def twitcher_service_factory(registry):
    settings = registry.settings
    service = TwitcherService(url=settings.get('twitcher.url'), verify=False)
    return service

def generate_access_token(registry, userid=None, valid_in_hours=1):
    settings = registry.settings
    service = twitcher_service_factory(registry)
    db = mongodb(registry)
    collection = db.users

    environ = {}
    user = collection.find_one({'identifier': userid})
    esgf_token = user.get('esgf_token')
    if esgf_token:
        environ['esgf_access_token'] = esgf_token.get('access_token')
        environ['esgf_slcs_service_url'] = settings.get('esgf.slcs.url')

    # call to service
    token = service.gentoken(valid_in_hours, environ)
    if userid and token:
        collection.update_one(
            {'identifier': userid},
            {'$set': {'twitcher_token': token, }})
    return token

def is_public(registry, name):
    service = twitcher_service_factory(registry)
    return service.is_public(name)
<commit_msg>updated twitcher api<commit_after>from urlparse import urlparse

from twitcher.client import TwitcherService

from phoenix.db import mongodb

def includeme(config):
    settings = config.registry.settings

    config.include('twitcher.rpcinterface')
    config.include('twitcher.owsproxy')
    config.include('twitcher.tweens')

def twitcher_service_factory(registry):
    settings = registry.settings
    service = TwitcherService(url=settings.get('twitcher.url'), verify=False)
    return service

def generate_access_token(registry, userid=None, valid_in_hours=1):
    settings = registry.settings
    service = twitcher_service_factory(registry)
    db = mongodb(registry)
    collection = db.users

    environ = {}
    user = collection.find_one({'identifier': userid})
    esgf_token = user.get('esgf_token')
    if esgf_token:
        environ['esgf_access_token'] = esgf_token.get('access_token')
        environ['esgf_slcs_service_url'] = settings.get('esgf.slcs.url')

    # call to service
    token = service.generate_token(valid_in_hours, environ)
    if userid and token:
        collection.update_one(
            {'identifier': userid},
            {'$set': {'twitcher_token': token, }})
    return token

def is_public(registry, name):
    service = twitcher_service_factory(registry)
    return service.is_public(name)
<|endoftext|>


<commit_before># coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_iso8601,
    unescapeHTML,
)

class PeriscopeBaseIE(InfoExtractor):
    def _call_api(self, method, query, item_id):
        return self._download_json(
            'https://api.periscope.tv/api/v2/%s' % method,
            item_id, query=query)

class PeriscopeIE(PeriscopeBaseIE):
    IE_DESC = 'Periscope'
    IE_NAME = 'periscope'
    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/[^/]+/(?P<id>[^/?#]+)'
    # Alive example URLs can be found here http://onperiscope.com/
    _TESTS = [{
        'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
        'md5': '65b57957972e503fcbbaeed8f4fa04ca',
        'info_dict': {
            'id': '56102209',
            'ext': 'mp4',
            'title': 'Bec Boop - 🚠✈🇬🇧 Fly above #London in Emirates Air Line cable car at night 🇬🇧✈🚠 #BoopScope 🎀💗',
            'timestamp': 1438978559,
            'upload_date': '20150807',
            'uploader': 'Bec Boop',
            'uploader_id': '1465763',
        },
        'skip': 'Expires in 24 hours',
    }, {
        'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
        'only_matching': True,
    }, {
        'url': 'https://www.periscope.tv/bastaakanoggano/1OdKrlkZZjOJX',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<iframe[^>]+src=([\'"])(?P<url>(?:https?:)?//(?:www\.)?periscope\.tv/(?:(?!\1).)+)\1', webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        token = self._match_id(url)

        broadcast_data = self._call_api(
            'getBroadcastPublic', {'broadcast_id': token}, token)
        broadcast = broadcast_data['broadcast']
        status = broadcast['status']

        user = broadcast_data.get('user', {})

        uploader = broadcast.get('user_display_name') or user.get('display_name')
        uploader_id = (broadcast.get('username') or user.get('username') or
                       broadcast.get('user_id') or user.get('id'))

        title = '%s - %s' % (uploader, status) if uploader else status
        state = broadcast.get('state').lower()
        if state == 'running':
            title = self._live_title(title)
        timestamp = parse_iso8601(broadcast.get('created_at'))

        thumbnails = [{
            'url': broadcast[image],
        } for image in ('image_url', 'image_url_small') if broadcast.get(image)]

        stream = self._call_api(
            'getAccessPublic', {'broadcast_id': token}, token)

        formats = []
        for format_id in ('replay', 'rtmp', 'hls', 'https_hls'):
            video_url = stream.get(format_id + '_url')
            if not video_url:
                continue
            f = {
                'url': video_url,
                'ext': 'flv' if format_id == 'rtmp' else 'mp4',
            }
            if format_id != 'rtmp':
                f['protocol'] = 'm3u8_native' if state in ('ended', 'timed_out') else 'm3u8'
            formats.append(f)
        self._sort_formats(formats)

        return {
            'id': broadcast.get('id') or token,
            'title': title,
            'timestamp': timestamp,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'thumbnails': thumbnails,
            'formats': formats,
        }

class PeriscopeUserIE(PeriscopeBaseIE):
    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/(?P<id>[^/]+)/?$'
    IE_DESC = 'Periscope user videos'
    IE_NAME = 'periscope:user'

    _TEST = {
        'url': 'https://www.periscope.tv/LularoeHusbandMike/',
        'info_dict': {
            'id': 'LularoeHusbandMike',
            'title': 'LULAROE HUSBAND MIKE',
            'description': 'md5:6cf4ec8047768098da58e446e82c82f0',
        },
        # Periscope only shows videos in the last 24 hours, so it's possible to
        # get 0 videos
        'playlist_mincount': 0,
    }

    def _real_extract(self, url):
        user_name = self._match_id(url)

        webpage = self._download_webpage(url, user_name)

        data_store = self._parse_json(
            unescapeHTML(self._search_regex(
                r'data-store=(["\'])(?P<data>.+?)\1',
                webpage, 'data store', default='{}', group='data')),
            user_name)

        user = list(data_store['UserCache']['users'].values())[0]['user']
        user_id = user['id']
        session_id = data_store['SessionToken']['public']['broadcastHistory']['token']['session_id']

        broadcasts = self._call_api(
            'getUserBroadcastsPublic',
            {'user_id': user_id, 'session_id': session_id},
            user_name)['broadcasts']

        broadcast_ids = [
            broadcast['id'] for broadcast in broadcasts if broadcast.get('id')]

        title = user.get('display_name') or user.get('username') or user_name
        description = user.get('description')

        entries = [
            self.url_result(
                'https://www.periscope.tv/%s/%s' % (user_name, broadcast_id))
            for broadcast_id in broadcast_ids]

        return self.playlist_result(entries, user_id, title, description)
<commit_msg>[pericope] Add support for pscp.tv URLs<commit_after># coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    parse_iso8601,
    unescapeHTML,
)

class PeriscopeBaseIE(InfoExtractor):
    def _call_api(self, method, query, item_id):
        return self._download_json(
            'https://api.periscope.tv/api/v2/%s' % method,
            item_id, query=query)

class PeriscopeIE(PeriscopeBaseIE):
    IE_DESC = 'Periscope'
    IE_NAME = 'periscope'
    _VALID_URL = r'https?://(?:www\.)?(?:periscope|pscp)\.tv/[^/]+/(?P<id>[^/?#]+)'
    # Alive example URLs can be found here http://onperiscope.com/
    _TESTS = [{
        'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
        'md5': '65b57957972e503fcbbaeed8f4fa04ca',
        'info_dict': {
            'id': '56102209',
            'ext': 'mp4',
            'title': 'Bec Boop - 🚠✈🇬🇧 Fly above #London in Emirates Air Line cable car at night 🇬🇧✈🚠 #BoopScope 🎀💗',
            'timestamp': 1438978559,
            'upload_date': '20150807',
            'uploader': 'Bec Boop',
            'uploader_id': '1465763',
        },
        'skip': 'Expires in 24 hours',
    }, {
        'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
        'only_matching': True,
    }, {
        'url': 'https://www.periscope.tv/bastaakanoggano/1OdKrlkZZjOJX',
        'only_matching': True,
    }, {
        'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
        'only_matching': True,
    }]

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'<iframe[^>]+src=([\'"])(?P<url>(?:https?:)?//(?:www\.)?periscope\.tv/(?:(?!\1).)+)\1', webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        token = self._match_id(url)

        broadcast_data = self._call_api(
            'getBroadcastPublic', {'broadcast_id': token}, token)
        broadcast = broadcast_data['broadcast']
        status = broadcast['status']

        user = broadcast_data.get('user', {})

        uploader = broadcast.get('user_display_name') or user.get('display_name')
        uploader_id = (broadcast.get('username') or user.get('username') or
                       broadcast.get('user_id') or user.get('id'))

        title = '%s - %s' % (uploader, status) if uploader else status
        state = broadcast.get('state').lower()
        if state == 'running':
            title = self._live_title(title)
        timestamp = parse_iso8601(broadcast.get('created_at'))

        thumbnails = [{
            'url': broadcast[image],
        } for image in ('image_url', 'image_url_small') if broadcast.get(image)]

        stream = self._call_api(
            'getAccessPublic', {'broadcast_id': token}, token)

        formats = []
        for format_id in ('replay', 'rtmp', 'hls', 'https_hls'):
            video_url = stream.get(format_id + '_url')
            if not video_url:
                continue
            f = {
                'url': video_url,
                'ext': 'flv' if format_id == 'rtmp' else 'mp4',
            }
            if format_id != 'rtmp':
                f['protocol'] = 'm3u8_native' if state in ('ended', 'timed_out') else 'm3u8'
            formats.append(f)
        self._sort_formats(formats)

        return {
            'id': broadcast.get('id') or token,
            'title': title,
            'timestamp': timestamp,
            'uploader': uploader,
            'uploader_id': uploader_id,
            'thumbnails': thumbnails,
            'formats': formats,
        }

class PeriscopeUserIE(PeriscopeBaseIE):
    _VALID_URL = r'https?://(?:www\.)?(?:periscope|pscp)\.tv/(?P<id>[^/]+)/?$'
    IE_DESC = 'Periscope user videos'
    IE_NAME = 'periscope:user'

    _TEST = {
        'url': 'https://www.periscope.tv/LularoeHusbandMike/',
        'info_dict': {
            'id': 'LularoeHusbandMike',
            'title': 'LULAROE HUSBAND MIKE',
            'description': 'md5:6cf4ec8047768098da58e446e82c82f0',
        },
        # Periscope only shows videos in the last 24 hours, so it's possible to
        # get 0 videos
        'playlist_mincount': 0,
    }

    def _real_extract(self, url):
        user_name = self._match_id(url)

        webpage = self._download_webpage(url, user_name)

        data_store = self._parse_json(
            unescapeHTML(self._search_regex(
                r'data-store=(["\'])(?P<data>.+?)\1',
                webpage, 'data store', default='{}', group='data')),
            user_name)

        user = list(data_store['UserCache']['users'].values())[0]['user']
        user_id = user['id']
        session_id = data_store['SessionToken']['public']['broadcastHistory']['token']['session_id']

        broadcasts = self._call_api(
            'getUserBroadcastsPublic',
            {'user_id': user_id, 'session_id': session_id},
            user_name)['broadcasts']

        broadcast_ids = [
            broadcast['id'] for broadcast in broadcasts if broadcast.get('id')]

        title = user.get('display_name') or user.get('username') or user_name
        description = user.get('description')

        entries = [
            self.url_result(
                'https://www.periscope.tv/%s/%s' % (user_name, broadcast_id))
            for broadcast_id in broadcast_ids]

        return self.playlist_result(entries, user_id, title, description)
<|endoftext|>


<commit_before>import trough.client
import sys
import argparse
import os
import cmd
import logging
import readline

HISTORY_FILE = os.path.expanduser('~/.trough_history')

class BetterArgumentDefaultsHelpFormatter(
                argparse.ArgumentDefaultsHelpFormatter,
                argparse.RawDescriptionHelpFormatter):
    '''
    HelpFormatter with these properties:

    - formats option help like argparse.ArgumentDefaultsHelpFormatter except
      that it omits the default value for arguments with action='store_const'
    - like argparse.RawDescriptionHelpFormatter, does not reformat description
      string
    '''
    def _get_help_string(self, action):
        if isinstance(action, argparse._StoreConstAction):
            return action.help
        else:
            return argparse.ArgumentDefaultsHelpFormatter._get_help_string(self, action)

class TroughRepl(cmd.Cmd):
    logger = logging.getLogger('trough.client.TroughRepl')

    def __init__(
            self, trough_client, segment_id, writable=False,
            schema_id='default'):
        super().__init__()
        self.cli = trough_client
        self.segment_id = segment_id
        self.writable = writable
        self.schema_id = schema_id

        self.prompt = 'trough:%s(%s)> ' % (
                segment_id, 'rw' if writable else 'ro')

    def do_select(self, line):
        result = self.cli.read(self.segment_id, 'select ' + line)
        print(result)
    do_SELECT = do_select

    def emptyline(self):
        pass

    def default(self, line):
        if line == 'EOF':
            print()
            return True
        elif self.writable:
            self.cli.write(self.segment_id, line, schema_id=self.schema_id)
        else:
            self.logger.error('read-only!')

    def do_quit(self, args):
        if not args:
            return True
    do_exit = do_quit
    do_bye = do_quit

def trough_client(argv=None):
    argv = argv or sys.argv
    arg_parser = argparse.ArgumentParser(
            prog=os.path.basename(argv[0]),
            formatter_class=BetterArgumentDefaultsHelpFormatter)
    arg_parser.add_argument(
            '-u', '--rethinkdb-trough-db-url',
            default='rethinkdb://localhost/trough_configuration')
    arg_parser.add_argument('-w', '--writable', action='store_true')
    arg_parser.add_argument(
            '-s', '--schema', default='default',
            help='schema id for new segment')
    arg_parser.add_argument('segment')
    args = arg_parser.parse_args(args=argv[1:])

    logging.basicConfig(
            stream=sys.stdout, level=logging.DEBUG, format=(
                '%(asctime)s %(levelname)s %(name)s.%(funcName)s'
                '(%(filename)s:%(lineno)d) %(message)s'))

    cli = trough.client.TroughClient(args.rethinkdb_trough_db_url)
    shell = TroughRepl(cli, args.segment, args.writable, args.schema)

    if os.path.exists(HISTORY_FILE):
        readline.read_history_file(HISTORY_FILE)

    try:
        shell.cmdloop()
    finally:
        readline.write_history_file(HISTORY_FILE)

<commit_msg>add some options for CLI: verbose, pretty print.<commit_after>import trough.client
import sys
import argparse
import os
import cmd
import logging
import readline
from prettytable import PrettyTable
import datetime

HISTORY_FILE = os.path.expanduser('~/.trough_history')

class BetterArgumentDefaultsHelpFormatter(
                argparse.ArgumentDefaultsHelpFormatter,
                argparse.RawDescriptionHelpFormatter):
    '''
    HelpFormatter with these properties:

    - formats option help like argparse.ArgumentDefaultsHelpFormatter except
      that it omits the default value for arguments with action='store_const'
    - like argparse.RawDescriptionHelpFormatter, does not reformat description
      string
    '''
    def _get_help_string(self, action):
        if isinstance(action, argparse._StoreConstAction):
            return action.help
        else:
            return argparse.ArgumentDefaultsHelpFormatter._get_help_string(self, action)

class TroughRepl(cmd.Cmd):
    intro = 'Welcome to the trough shell. Type help or ? to list commands.\n'
    logger = logging.getLogger('trough.client.TroughRepl')

    def __init__(
            self, trough_client, segment_id, writable=False,
            schema_id='default'):
        super().__init__()
        self.cli = trough_client
        self.segment_id = segment_id
        self.writable = writable
        self.schema_id = schema_id
        self.pretty_print = True

        self.prompt = 'trough:%s(%s)> ' % (
                segment_id, 'rw' if writable else 'ro')

    def do_pretty(self, ignore):
        '''Toggle pretty-printed results'''
        self.pretty_print = not self.pretty_print
        print('pretty print %s' % ("on" if self.pretty_print else "off"))

    def do_select(self, line):
        '''Send a query to the currently-connected trough segment.

        Syntax: select...

        Example: Send query "select * from host_statistics;" to server
        trough> query select * from host_statistics;
        '''
        start = datetime.datetime.now()
        result = self.cli.read(self.segment_id, 'select ' + line)
        end = datetime.datetime.now()
        if self.pretty_print:
            header = result[0].keys()
            pt = PrettyTable(header)
            for item in header:
                pt.align[item] = "l"
            pt.padding_width = 1
            for row in result:
                pt.add_row([row.get(column) for column in header])
            print(pt)
        else:
            print(result)
        print("%s results in %s" % (len(result), end - start))

    do_SELECT = do_select

    def emptyline(self):
        pass

    def default(self, line):
        if line == 'EOF':
            print()
            return True
        elif self.writable:
            self.cli.write(self.segment_id, line, schema_id=self.schema_id)
        else:
            self.logger.error('read-only!')

    def do_quit(self, args):
        if not args:
            print('bye!')
            return True
    do_EOF = do_quit
    do_exit = do_quit
    do_bye = do_quit

def trough_client(argv=None):
    argv = argv or sys.argv
    arg_parser = argparse.ArgumentParser(
            prog=os.path.basename(argv[0]),
            formatter_class=BetterArgumentDefaultsHelpFormatter)
    arg_parser.add_argument(
            '-u', '--rethinkdb-trough-db-url',
            default='rethinkdb://localhost/trough_configuration')
    arg_parser.add_argument('-w', '--writable', action='store_true')
    arg_parser.add_argument('-v', '--verbose', action='store_true')
    arg_parser.add_argument(
            '-s', '--schema', default='default',
            help='schema id for new segment')
    arg_parser.add_argument('segment')
    args = arg_parser.parse_args(args=argv[1:])

    logging.basicConfig(
            stream=sys.stdout, level=logging.DEBUG if args.verbose else logging.WARN, format=(
                '%(asctime)s %(levelname)s %(name)s.%(funcName)s'
                '(%(filename)s:%(lineno)d) %(message)s'))

    cli = trough.client.TroughClient(args.rethinkdb_trough_db_url)
    shell = TroughRepl(cli, args.segment, args.writable, args.schema)

    if os.path.exists(HISTORY_FILE):
        readline.read_history_file(HISTORY_FILE)

    try:
        shell.cmdloop()
    finally:
        readline.write_history_file(HISTORY_FILE)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function
import argparse
import datetime
import os
from random import randint

import pandas as pd
from langdetect import *
from lxml import etree

parser = argparse.ArgumentParser()
parser.add_argument("-ownName", dest='ownName', type=str,
                    help="name of the owner of the chat logs, written as in the logs", required=True)
parser.add_argument('-f', '-filePath', dest='filePath', help='Facebook chat log file (HTML file)',
                    default='raw/messages')
parser.add_argument("-max", "-maxExportedMessages", dest='maxExportedMessages', type=int, default=1000000,
                    help="maximum number of messages to export")
args = parser.parse_args()

maxExportedMessages = args.maxExportedMessages
ownName = args.ownName
filePath = args.filePath

data = []
warnedNameChanges = []
nbInvalidSender = 0

# make sure we don't crash if chat logs contain exotic characters
etree.set_default_parser(etree.XMLParser(encoding='utf-8', ns_clean=True, recover=True))

for filename in os.listdir(filePath):

    if not filename.endswith('.html'):
        continue

    archive = etree.parse(filePath + "/" + filename)

    conversationId = filename.replace('.html', '')
    groupConversation = False
    timestamp = ''
    senderName = ''
    conversationWithName = None
    text = ''

    for element in archive.iter():
        tag = element.tag
        className = element.get('class')
        content = element.text

        if tag == 'p':
            text = content

            if conversationWithName is not "" and senderName is not "":

                # handles when the interlocutor's name changed at some point
                if (senderName != conversationWithName) and (senderName != ownName) and (
                    senderName not in warnedNameChanges) and (not groupConversation):
                    if senderName not in warnedNameChanges:
                        print('\t', 'Assuming', senderName, 'is', conversationWithName)
                        warnedNameChanges.append(senderName)

                    senderName = conversationWithName

                data += [[timestamp, conversationId, conversationWithName, senderName, text]]

            else:
                nbInvalidSender = nbInvalidSender + 1

        elif tag == 'span':
            if className == 'user':
                senderName = content
            elif className == 'meta':
                # TODO recognize the date format and use the appropriate regexp. infer_datetime_format is suuuper slow.
                timestamp = pd.to_datetime(content, infer_datetime_format=True).strftime("%s")

        elif tag == 'div' and className == 'thread':
            nbParticipants = str(element.xpath("text()")).count(', ') + 1
            if nbParticipants > 1:
                groupConversation = True

        elif tag == 'h3':
            if conversationWithName is not None:
                print('Something is wrong. File format changed? (multiple conversation hearder in a single file)')
                exit(0)
            else:
                content = content.replace('Conversation with ', '')
                conversationWithName = content

            print(conversationId, conversationWithName, "(group?", groupConversation, ")")

        if len(data) >= maxExportedMessages:
            break

print(len(data), 'messages parsed.')

if nbInvalidSender > 0:
    print(nbInvalidSender, 'messages discarded because of bad ID.')

if len(data) < 1:
    print('Nothing to save.')
    exit(0)

print('Converting to DataFrame...')
df = pd.DataFrame(data)
df.columns = ['timestamp', 'conversationId', 'conversationWithName', 'senderName', 'text']
df['platform'] = 'messenger'

print('Detecting languages...')
df['language'] = 'unknown'
for name, group in df.groupby(df.conversationWithName):
    sample = ''
    df2 = df[df.conversationWithName == name].dropna()

    if len(df2) > 10:
        for x in range(0, min(len(df2), 100)):
            sample = sample + df2.iloc[randint(0, len(df2) - 1)]['text']

        print('\t', name, detect(sample), "(", len(df2), "msgs)")
        df.loc[df.conversationWithName == name, 'language'] = detect(sample)

print('Computing dates...')
df['datetime'] = df[df["timestamp"] != ""]['timestamp'].apply(
    lambda x: datetime.datetime.fromtimestamp(float(x)).toordinal())

print(df.head())

print('Saving to pickle file...')
df.to_pickle('data/messenger.pkl')

print('Done.')
<commit_msg>Fixed date parsing when using Python 3. Closes #6<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function
import argparse
import datetime
import os
from random import randint

import pandas as pd
import time
from langdetect import *
from lxml import etree

parser = argparse.ArgumentParser()
parser.add_argument("-ownName", dest='ownName', type=str,
                    help="name of the owner of the chat logs, written as in the logs", required=True)
parser.add_argument('-f', '-filePath', dest='filePath', help='Facebook chat log file (HTML file)',
                    default='raw/messages')
parser.add_argument("-max", "-maxExportedMessages", dest='maxExportedMessages', type=int, default=1000000,
                    help="maximum number of messages to export")
args = parser.parse_args()

maxExportedMessages = args.maxExportedMessages
ownName = args.ownName
filePath = args.filePath

data = []
warnedNameChanges = []
nbInvalidSender = 0

# make sure we don't crash if chat logs contain exotic characters
etree.set_default_parser(etree.XMLParser(encoding='utf-8', ns_clean=True, recover=True))

for filename in os.listdir(filePath):

    if not filename.endswith('.html'):
        continue

    archive = etree.parse(filePath + "/" + filename)

    conversationId = filename.replace('.html', '')
    groupConversation = False
    timestamp = ''
    senderName = ''
    conversationWithName = None
    text = ''

    for element in archive.iter():
        tag = element.tag
        className = element.get('class')
        content = element.text

        if tag == 'p':
            text = content

            if conversationWithName is not "" and senderName is not "":

                # handles when the interlocutor's name changed at some point
                if (senderName != conversationWithName) and (senderName != ownName) and (
                    senderName not in warnedNameChanges) and (not groupConversation):
                    if senderName not in warnedNameChanges:
                        print('\t', 'Assuming', senderName, 'is', conversationWithName)
                        warnedNameChanges.append(senderName)

                    senderName = conversationWithName

                data += [[timestamp, conversationId, conversationWithName, senderName, text]]

            else:
                nbInvalidSender = nbInvalidSender + 1

        elif tag == 'span':
            if className == 'user':
                senderName = content
            elif className == 'meta':
                # TODO recognize the date format and use the appropriate regexp. infer_datetime_format is suuuper slow.
                timestamp = time.mktime(pd.to_datetime(content, infer_datetime_format=True).timetuple())

        elif tag == 'div' and className == 'thread':
            nbParticipants = str(element.xpath("text()")).count(', ') + 1
            if nbParticipants > 1:
                groupConversation = True

        elif tag == 'h3':
            if conversationWithName is not None:
                print('Something is wrong. File format changed? (multiple conversation hearder in a single file)')
                exit(0)
            else:
                content = content.replace('Conversation with ', '')
                conversationWithName = content

            print(conversationId, conversationWithName, "(group?", groupConversation, ")")

        if len(data) >= maxExportedMessages:
            break

print(len(data), 'messages parsed.')

if nbInvalidSender > 0:
    print(nbInvalidSender, 'messages discarded because of bad ID.')

if len(data) < 1:
    print('Nothing to save.')
    exit(0)

print('Converting to DataFrame...')
df = pd.DataFrame(data)
df.columns = ['timestamp', 'conversationId', 'conversationWithName', 'senderName', 'text']
df['platform'] = 'messenger'

print('Detecting languages...')
df['language'] = 'unknown'
for name, group in df.groupby(df.conversationWithName):
    sample = ''
    df2 = df[df.conversationWithName == name].dropna()

    if len(df2) > 10:
        for x in range(0, min(len(df2), 100)):
            sample = sample + df2.iloc[randint(0, len(df2) - 1)]['text']

        print('\t', name, detect(sample), "(", len(df2), "msgs)")
        df.loc[df.conversationWithName == name, 'language'] = detect(sample)

print('Computing dates...')
ordinate = lambda x: datetime.datetime.fromtimestamp(float(x)).toordinal()
df['datetime'] = df['timestamp'].apply(ordinate)

print(df.head())

print('Saving to pickle file...')
df.to_pickle('data/messenger.pkl')

print('Done.')
<|endoftext|>


<commit_before># Copyright (C) 2015 Atsushi Togo
# All rights reserved.
#
# This file is part of phonopy.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in
#   the documentation and/or other materials provided with the
#   distribution.
#
# * Neither the name of the phonopy project nor the names of its
#   contributors may be used to endorse or promote products derived
#   from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import numpy as np
from phonopy.harmonic.dynmat_to_fc import get_commensurate_points
from phonopy.structure.atoms import PhonopyAtoms
from phonopy.structure.cells import get_supercell

class Unfolding(object):
    """

    Implementation of an unfolding method by
    P. B. Allen et al., Phys. Rev. B 87, 085322 (2013)

    T(r_i) in this implementation is defined as

        T(r_i) f(x) = f(x - r_i).

    The sign is opposite from that written in the Allen's paper.
    Bloch wave is defined in the same way for phase convention

        Psi_k(x + r) = exp(ikr) Psi_k(x).

    By these, sign of phase in Eq.(3) (Eq.(7) as well) is opposite.

    """

    def __init__(self,
                 phonon,
                 supercell_matrix,
                 ideal_positions,
                 atom_mapping,
                 qpoints):
        """

        Parameters
        ----------
        phonon : Phonopy
            Phonopy object made with supercell as the primitive cell.
        supercell_matrix : array_like
            Matrix that represents the primitive translation enforced within
            the supercell. This works like an inverse primitive matrix.
            shape=(3, 3), dtype='intc'
        ideal_positions : array_like
            Atomic positions of perfect supercell.
            shape=(3, 3), dtype='intc'
        atom_mapping : list
            Atomic index mapping from ideal_positions to supercell atoms in
            phonon. None is used for Vacancies.
        qpoints : array_like
            q-points in reciprocal primitive cell coordinates
            shape=(num_qpoints, 3), dtype='double'

        """

        self._phonon = phonon
        self._supercell_matrix = np.array(supercell_matrix, dtype='intc')
        self._ideal_positions = np.array(ideal_positions, dtype='double')
        self._qpoints_p = qpoints  # in PBZ
        self._qpoints_s = self._get_qpoints_in_SBZ()  # in SBZ
        self._symprec = self._phonon.symmetry.get_symmetry_tolerance()

        self._frequencies = None
        self._eigvecs = None  # This may have anormal array shape.
        self._unfolding_weights = None
        self._q_count = None  # As counter for iterator

        # Commensurate q-vectors in PBZ
        self._comm_points = get_commensurate_points(self._supercell_matrix)

        self._trans_s = None  # in SC (see docstring in _set_translations)
        self._trans_p = None  # in PC (see docstring in _set_translations)
        self._N = None
        self._set_translations()

        self._atom_mapping = None
        self._index_map_inv = None
        self._set_index_map(atom_mapping)

    def __iter__(self):
        return self

    def run(self):
        self.prepare()
        for x in self:
            pass

    def __next__(self):
        if self._q_count == len(self._qpoints_s):
            self._eigvecs = None
            raise StopIteration

        self._solve_phonon()
        self._unfolding_weights[self._q_count] = self._get_unfolding_weights()
        self._q_count += 1
        return self._unfolding_weights[self._q_count - 1]

    def next(self):
        return self.__next__()

    def prepare(self):
        self._q_count = 0
        self._unfolding_weights = np.zeros(
            (len(self._qpoints_s),
             len(self._phonon.supercell) * 3), dtype='double')
        self._frequencies = np.zeros_like(self._unfolding_weights)

    @property
    def commensurate_points(self):
        return self._comm_points

    def get_commensurate_points(self):
        return self.commensurate_points

    @property
    def unfolding_weights(self):
        return self._unfolding_weights

    def get_unfolding_weights(self):
        return self.unfolding_weights

    @property
    def frequencies(self):
        return self._frequencies

    def get_frequencies(self):
        return self.frequencies

    def _get_qpoints_in_SBZ(self):
        qpoints = np.dot(self._qpoints_p, self._supercell_matrix)
        qpoints -= np.rint(qpoints)
        return qpoints

    def _set_translations(self):
        """Set primitive translations in supercell

        _trans_s
            Translations with respect to supercell basis vectors
        _trans_p
            Translations with respect to primitive cell basis vectors
        _N
            Number of the translations = det(supercel_matrix)

        """

        pcell = PhonopyAtoms(numbers=[1],
                             scaled_positions=[[0, 0, 0]],
                             cell=np.diag([1, 1, 1]))
        smat = self._supercell_matrix
        self._trans_s = get_supercell(pcell, smat).scaled_positions
        self._trans_p = np.dot(self._trans_s, self._supercell_matrix.T)
        self._N = len(self._trans_s)

    def _set_index_map(self, atom_mapping):
        """T(r_i) in Eq.(3) is given as permutation of atom indices.

        _index_set : ndarray
            For each translation (shift), atomic indices of the positions
            (_ideal_positions - shift) are searched and stored. The indices
            are used to select eigenvectors, by which T(r_i)|KJ> is
            represented.
            shape=(num_trans, num_sites), dtype='intc'

        """

        lattice = self._phonon.supercell.get_cell()
        natom = len(self._ideal_positions)
        index_map_inv = np.zeros((self._N, natom), dtype='intc')
        for i, shift in enumerate(self._trans_s):
            for j, p in enumerate(self._ideal_positions - shift):  # minus r_i
                diff = self._ideal_positions - p
                diff -= np.rint(diff)
                dist = np.sqrt((np.dot(diff, lattice) ** 2).sum(axis=1))
                # k is index in _ideal_positions.
                k = np.where(dist < self._symprec)[0][0]
                index_map_inv[i, j] = k
        self._index_map_inv = index_map_inv

        self._atom_mapping = np.zeros(len(atom_mapping), dtype='int')
        for i, idx in enumerate(atom_mapping):
            if idx is None:
                self._atom_mapping[i] = -1
            else:
                self._atom_mapping[i] = idx

    def _solve_phonon(self):
        self._phonon.run_qpoints(self._qpoints_s[self._q_count],
                                 with_eigenvectors=True)
        qpt = self._phonon.get_qpoints_dict()
        self._frequencies[self._q_count] = qpt['frequencies'][0]
        eigvecs = qpt['eigenvectors'][0]

        if -1 in self._atom_mapping:
            shape = list(eigvecs.shape)
            shape[0] += 3
            self._eigvecs = np.zeros(tuple(shape), dtype=eigvecs.dtype)
            self._eigvecs[:-3, :] = eigvecs
        else:
            self._eigvecs = eigvecs

    def _get_unfolding_weights(self):
        """Calculate Eq. (7)

        k = K + G + g

        K -> _qpoints_s[_q_index] (in SBZ)
        k -> _qpoints_p[_q_index] (in PBZ)
        G -> _comm_points (in PBZ)
        g -> a reciprocal lattice point in PBZ (unused explicitly)
        j -> Primitive translations in supercell (_trans_p)
        J -> Band indices of supercell phonon modes (axis=1 or eigvecs)

        The phase factor corresponding to K is not included in eigvecs
        with our choice of dynamical matrix.

        """

        eigvecs = self._eigvecs
        dtype = "c%d" % (np.dtype('double').itemsize * 2)
        q_p = self._qpoints_p[self._q_count]  # k
        q_s = self._qpoints_s[self._q_count]  # K
        diff = q_p - np.dot(q_s, np.linalg.inv(self._supercell_matrix))

        # Search G points corresponding to k = G + K
        for G in self._comm_points:
            d = diff - G
            d -= np.rint(d)
            if (np.abs(d) < 1e-5).all():
                break

        e = np.zeros((len(self._atom_mapping) * 3, eigvecs.shape[1]),
                     dtype=dtype)
        phases = np.exp(2j * np.pi * np.dot(self._trans_p, G))

        for phase, indices in zip(
                phases, self._atom_mapping[self._index_map_inv]):
            eig_indices = (
                np.c_[indices * 3, indices * 3 + 1, indices * 3 + 2]).ravel()
            e += eigvecs[eig_indices, :] * phase
        e /= self._N
        weights = (e.conj() * e).sum(axis=0)

        # indices = self._atom_mapping
        # eig_indices_r = (
        #     np.c_[indices * 3, indices * 3 + 1, indices * 3 + 2]).ravel()
        # for phase, indices in zip(
        #         phases, self._atom_mapping[self._index_map_inv]):
        #     eig_indices_l = (
        #         np.c_[indices * 3, indices * 3 + 1, indices * 3 + 2]).ravel()
        #     e += eigvecs[eig_indices_l, :] * eigvecs[eig_indices_r, :].conj() * phase
        # e /= self._N
        # weights = e.sum(axis=0)

        if (weights.imag > 1e-5).any():
            print("Phonopy warning: Encountered imaginary values.")

        return weights.real
<commit_msg>Update of docstrings in Unfolding<commit_after># Copyright (C) 2015 Atsushi Togo
# All rights reserved.
#
# This file is part of phonopy.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in
#   the documentation and/or other materials provided with the
#   distribution.
#
# * Neither the name of the phonopy project nor the names of its
#   contributors may be used to endorse or promote products derived
#   from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import numpy as np
from phonopy.harmonic.dynmat_to_fc import get_commensurate_points
from phonopy.structure.atoms import PhonopyAtoms
from phonopy.structure.cells import get_supercell

class Unfolding(object):
    """Calculation of a phonon unfolding method

    Implementation of an unfolding method by
    P. B. Allen et al., Phys. Rev. B 87, 085322 (2013)

    T(r_i) in this implementation is defined as

        T(r_i) f(x) = f(x - r_i).

    The sign is opposite from that written in the Allen's paper.
    Bloch wave is defined in the same way for phase convention

        Psi_k(x + r) = exp(ikr) Psi_k(x).

    By these, sign of phase in Eq.(3) (Eq.(7) as well) is opposite.

    Vacancies are treated as atomic sites where no atoms exist.
    Interstitials are treated as atomic sites with the interstitial
    atoms in a specific virtual primitive cell, but the respective
    atomic sites in the other virtual
    primitive cells are set as vacancies.

    Attributes
    ----------
    unfolding_weights
    frequencies
    commensurate_points

    """

    def __init__(self,
                 phonon,
                 supercell_matrix,
                 ideal_positions,
                 atom_mapping,
                 qpoints):
        """

        Parameters
        ----------
        phonon : Phonopy
            Phonopy object made with supercell as the primitive cell.
        supercell_matrix : array_like
            Matrix that represents the primitive translation enforced within
            the supercell. This works like an inverse primitive matrix.
            shape=(3, 3), dtype='intc'
        ideal_positions : array_like
            Positions of atomic sites in supercell, which corresponds to those
            in a set of virtual primitive cells in the supercell.
            shape=(3, 3), dtype='intc'
        atom_mapping : list
            Atomic index mapping from ideal_positions to supercell atoms in
            parameter ``phonon``. The elements are intergers for atoms and
            None for vacancies.
        qpoints : array_like
            q-points in reciprocal primitive cell coordinates
            shape=(num_qpoints, 3), dtype='double'

        """

        self._phonon = phonon
        self._supercell_matrix = np.array(supercell_matrix, dtype='intc')
        self._ideal_positions = np.array(ideal_positions, dtype='double')
        self._qpoints_p = qpoints  # in PBZ
        self._qpoints_s = self._get_qpoints_in_SBZ()  # in SBZ
        self._symprec = self._phonon.symmetry.get_symmetry_tolerance()

        self._frequencies = None
        self._eigvecs = None  # This may have anormal array shape.
        self._unfolding_weights = None
        self._q_count = None  # As counter for iterator

        # Commensurate q-vectors in PBZ
        self._comm_points = get_commensurate_points(self._supercell_matrix)

        self._trans_s = None  # in SC (see docstring in _set_translations)
        self._trans_p = None  # in PC (see docstring in _set_translations)
        self._N = None
        self._set_translations()

        self._atom_mapping = None
        self._index_map_inv = None
        self._set_index_map(atom_mapping)

    def __iter__(self):
        return self

    def run(self):
        self.prepare()
        for x in self:
            pass

    def __next__(self):
        if self._q_count == len(self._qpoints_s):
            self._eigvecs = None
            raise StopIteration

        self._solve_phonon()
        self._unfolding_weights[self._q_count] = self._get_unfolding_weights()
        self._q_count += 1
        return self._unfolding_weights[self._q_count - 1]

    def next(self):
        return self.__next__()

    def prepare(self):
        self._q_count = 0
        self._unfolding_weights = np.zeros(
            (len(self._qpoints_s),
             len(self._phonon.supercell) * 3), dtype='double')
        self._frequencies = np.zeros_like(self._unfolding_weights)

    @property
    def commensurate_points(self):
        return self._comm_points

    def get_commensurate_points(self):
        return self.commensurate_points

    @property
    def unfolding_weights(self):
        return self._unfolding_weights

    def get_unfolding_weights(self):
        return self.unfolding_weights

    @property
    def frequencies(self):
        return self._frequencies

    def get_frequencies(self):
        return self.frequencies

    def _get_qpoints_in_SBZ(self):
        qpoints = np.dot(self._qpoints_p, self._supercell_matrix)
        qpoints -= np.rint(qpoints)
        return qpoints

    def _set_translations(self):
        """Set primitive translations in supercell

        _trans_s
            Translations with respect to supercell basis vectors
        _trans_p
            Translations with respect to primitive cell basis vectors
        _N
            Number of the translations = det(supercel_matrix)

        """

        pcell = PhonopyAtoms(numbers=[1],
                             scaled_positions=[[0, 0, 0]],
                             cell=np.diag([1, 1, 1]))
        smat = self._supercell_matrix
        self._trans_s = get_supercell(pcell, smat).scaled_positions
        self._trans_p = np.dot(self._trans_s, self._supercell_matrix.T)
        self._N = len(self._trans_s)

    def _set_index_map(self, atom_mapping):
        """T(r_i) in Eq.(3) is given as permutation of atom indices.

        _index_set : ndarray
            For each translation (shift), atomic indices of the positions
            (_ideal_positions - shift) are searched and stored. The indices
            are used to select eigenvectors, by which T(r_i)|KJ> is
            represented.
            shape=(num_trans, num_sites), dtype='intc'

        """

        lattice = self._phonon.supercell.get_cell()
        natom = len(self._ideal_positions)
        index_map_inv = np.zeros((self._N, natom), dtype='intc')
        for i, shift in enumerate(self._trans_s):
            for j, p in enumerate(self._ideal_positions - shift):  # minus r_i
                diff = self._ideal_positions - p
                diff -= np.rint(diff)
                dist = np.sqrt((np.dot(diff, lattice) ** 2).sum(axis=1))
                # k is index in _ideal_positions.
                k = np.where(dist < self._symprec)[0][0]
                index_map_inv[i, j] = k
        self._index_map_inv = index_map_inv

        self._atom_mapping = np.zeros(len(atom_mapping), dtype='int')
        for i, idx in enumerate(atom_mapping):
            if idx is None:
                self._atom_mapping[i] = -1
            else:
                self._atom_mapping[i] = idx

    def _solve_phonon(self):
        self._phonon.run_qpoints(self._qpoints_s[self._q_count],
                                 with_eigenvectors=True)
        qpt = self._phonon.get_qpoints_dict()
        self._frequencies[self._q_count] = qpt['frequencies'][0]
        eigvecs = qpt['eigenvectors'][0]

        if -1 in self._atom_mapping:
            shape = list(eigvecs.shape)
            shape[0] += 3
            self._eigvecs = np.zeros(tuple(shape), dtype=eigvecs.dtype)
            self._eigvecs[:-3, :] = eigvecs
        else:
            self._eigvecs = eigvecs

    def _get_unfolding_weights(self):
        """Calculate Eq. (7)

        k = K + G + g

        K -> _qpoints_s[_q_index] (in SBZ)
        k -> _qpoints_p[_q_index] (in PBZ)
        G -> _comm_points (in PBZ)
        g -> a reciprocal lattice point in PBZ (unused explicitly)
        j -> Primitive translations in supercell (_trans_p)
        J -> Band indices of supercell phonon modes (axis=1 or eigvecs)

        The phase factor corresponding to K is not included in eigvecs
        with our choice of dynamical matrix.

        """

        eigvecs = self._eigvecs
        dtype = "c%d" % (np.dtype('double').itemsize * 2)
        q_p = self._qpoints_p[self._q_count]  # k
        q_s = self._qpoints_s[self._q_count]  # K
        diff = q_p - np.dot(q_s, np.linalg.inv(self._supercell_matrix))

        # Search G points corresponding to k = G + K
        for G in self._comm_points:
            d = diff - G
            d -= np.rint(d)
            if (np.abs(d) < 1e-5).all():
                break

        e = np.zeros((len(self._atom_mapping) * 3, eigvecs.shape[1]),
                     dtype=dtype)
        phases = np.exp(2j * np.pi * np.dot(self._trans_p, G))

        for phase, indices in zip(
                phases, self._atom_mapping[self._index_map_inv]):
            eig_indices = (
                np.c_[indices * 3, indices * 3 + 1, indices * 3 + 2]).ravel()
            e += eigvecs[eig_indices, :] * phase
        e /= self._N
        weights = (e.conj() * e).sum(axis=0)

        # indices = self._atom_mapping
        # eig_indices_r = (
        #     np.c_[indices * 3, indices * 3 + 1, indices * 3 + 2]).ravel()
        # for phase, indices in zip(
        #         phases, self._atom_mapping[self._index_map_inv]):
        #     eig_indices_l = (
        #         np.c_[indices * 3, indices * 3 + 1, indices * 3 + 2]).ravel()
        #     e += eigvecs[eig_indices_l, :] * eigvecs[eig_indices_r, :].conj() * phase
        # e /= self._N
        # weights = e.sum(axis=0)

        if (weights.imag > 1e-5).any():
            print("Phonopy warning: Encountered imaginary values.")

        return weights.real
<|endoftext|>


<commit_before>from System.Diagnostics import *
from System.IO import *

from Deadline.Events import *
from Deadline.Scripting import *

###############################################################
## This is the function called by Deadline to get an instance
## of the VRay Pre Pass Verify event listener.
###############################################################
def GetDeadlineEventListener():
    return VRayPrePassEventListener()

def CleanupDeadlineEventListener( eventListener ):
    eventListener.Cleanup()

###############################################################
## The VRay event listener class.
###############################################################
class VRayPrePassEventListener( DeadlineEventListener ):
    '''
    Verify V-Ray animation pre-pass *.vrmap files have been successfully saved back to the network file server.
    If the *.vrmap file is missing, then requeue the task(s) automatically.
    '''
    def __init__( self ):
        self.OnJobFinishedCallback += self.OnJobFinished

    def Cleanup( self ):
        del self.OnJobFinishedCallback

    ## This is called when the job finishes rendering.
    def OnJobFinished( self, job ):

        # Check job is a 3dsMax job
        if job.JobPlugin != "3dsmax":
            return

        # Check job is a V-Ray job
        vrayFilterExists = self.GetPluginInfoEntryWithDefault( "vray_filter_on", "" )
        if vrayFilterExists == "":
            return

        # Check GI is enabled
        vray_gi_on = self.GetPluginInfoEntryWithDefault( "vray_GI_on", "" )
        if vray_gi_on != "true":
            return

        # Check job is an animation pre-pass job
        vray_adv_irradmap_mode = self.GetPluginInfoEntryWithDefault( "vray_adv_irradmap_mode", "" )
        if vray_adv_irradmap_mode != "6":
            return

        self.LogInfo( "Event Plugin: V-Ray PrePass Verify Started" )

        vray_adv_irradmap_autoSaveFileName = self.GetPluginInfoEntryWithDefault( "vray_adv_irradmap_autoSaveFileName", "" )
        filePath = Path.GetDirectoryName( vray_adv_irradmap_autoSaveFileName )
        fileName = Path.GetFileNameWithoutExtension( vray_adv_irradmap_autoSaveFileName )
        vrmapFile = str( fileName ) + "####.vrmap"
        vrmapFile = Path.Combine( filePath, vrmapFile )
        padding = "####"
        taskIdsToRequeue = []

        tasks = RepositoryUtils.GetJobTasks( job, True )

        for task in tasks:
            for frame in task.TaskFrameList:
                frameNumber = StringUtils.ToZeroPaddedString( frame, 4 )
                currFile = vrmapFile.replace( padding, frameNumber )

                if not File.Exists( currFile ):

                    self.LogWarning( "Missing: %s" % currFile )

                    if not task in taskIdsToRequeue:
                        taskIdsToRequeue.append( task.TaskId )

        if len(taskIdsToRequeue) > 0:

            taskIdsToRequeue = list(set(taskIdsToRequeue))
            tasks = RepositoryUtils.GetJobTasks( job, True )

            for task in tasks.Tasks:
                for i in taskIdsToRequeue:
                    if i == task.TaskId:
                        self.LogInfo( "Requeuing Task: %s" % task.TaskId )
                        RepositoryUtils.RequeueTasks( job, [task,] )

        self.LogInfo( "Event Plugin: V-Ray PrePass Verify Finished" )
<commit_msg>Removed duplicate function.<commit_after>from System.Diagnostics import *
from System.IO import *

from Deadline.Events import *
from Deadline.Scripting import *

###############################################################
## This is the function called by Deadline to get an instance
## of the VRay Pre Pass Verify event listener.
###############################################################
def GetDeadlineEventListener():
    return VRayPrePassEventListener()

def CleanupDeadlineEventListener( eventListener ):
    eventListener.Cleanup()

###############################################################
## The VRay event listener class.
###############################################################
class VRayPrePassEventListener( DeadlineEventListener ):
    '''
    Verify V-Ray animation pre-pass *.vrmap files have been successfully saved back to the network file server.
    If the *.vrmap file is missing, then requeue the task(s) automatically.
    '''
    def __init__( self ):
        self.OnJobFinishedCallback += self.OnJobFinished

    def Cleanup( self ):
        del self.OnJobFinishedCallback

    ## This is called when the job finishes rendering.
    def OnJobFinished( self, job ):

        # Check job is a 3dsMax job
        if job.JobPlugin != "3dsmax":
            return

        # Check job is a V-Ray job
        vrayFilterExists = self.GetPluginInfoEntryWithDefault( "vray_filter_on", "" )
        if vrayFilterExists == "":
            return

        # Check GI is enabled
        vray_gi_on = self.GetPluginInfoEntryWithDefault( "vray_GI_on", "" )
        if vray_gi_on != "true":
            return

        # Check job is an animation pre-pass job
        vray_adv_irradmap_mode = self.GetPluginInfoEntryWithDefault( "vray_adv_irradmap_mode", "" )
        if vray_adv_irradmap_mode != "6":
            return

        self.LogInfo( "Event Plugin: V-Ray PrePass Verify Started" )

        vray_adv_irradmap_autoSaveFileName = self.GetPluginInfoEntryWithDefault( "vray_adv_irradmap_autoSaveFileName", "" )
        filePath = Path.GetDirectoryName( vray_adv_irradmap_autoSaveFileName )
        fileName = Path.GetFileNameWithoutExtension( vray_adv_irradmap_autoSaveFileName )
        vrmapFile = str( fileName ) + "####.vrmap"
        vrmapFile = Path.Combine( filePath, vrmapFile )
        padding = "####"
        taskIdsToRequeue = []

        tasks = RepositoryUtils.GetJobTasks( job, True )

        for task in tasks:
            for frame in task.TaskFrameList:
                frameNumber = StringUtils.ToZeroPaddedString( frame, 4 )
                currFile = vrmapFile.replace( padding, frameNumber )

                if not File.Exists( currFile ):

                    self.LogWarning( "Missing: %s" % currFile )

                    if not task in taskIdsToRequeue:
                        taskIdsToRequeue.append( task.TaskId )

        if len(taskIdsToRequeue) > 0:

            taskIdsToRequeue = list(set(taskIdsToRequeue))

            for task in tasks.Tasks:
                for i in taskIdsToRequeue:
                    if i == task.TaskId:
                        self.LogInfo( "Requeuing Task: %s" % task.TaskId )
                        RepositoryUtils.RequeueTasks( job, [task,] )

        self.LogInfo( "Event Plugin: V-Ray PrePass Verify Finished" )
<|endoftext|>


<commit_before>'''
Created on Nov 14, 2014
Copied from GitHubResearchDataMiner

@author: bgt
'''

import ConfigParser
import os
import sys
import time
import CsvDao
import GitHubDao
from GitHubFacadeForProcessDistibutionDao import GitHubFacadeForProcessDistibutionDao
from github.GithubException import GithubException
import ProcessDistributionDao
from BgtConfiguration import BgtConfiguration
from BgtConfiguration import BadCommandLineException

def analyseRepositories(hits):
    countDooku = 0

    for repo in hits:
        countDooku += 1
        try:
            print "Analysing repository #"+str(countDooku)+", "+repo.full_name+" "+str(repo.id)
            if config.searching == "facebook":
                analysis = connection.usesFacebookGraph(repo)
            else:
                analysis = connection.usesSsl(repo)
            csvDao.addRow(analysis)
        except GithubException:
            print "...except that repository has blocked access"
    csvDao.close()

def delegateRepositories(config, hits):
    minions = ProcessDistributionDao.ProcessDistributionDao(config, True)
    for repo in hits:
        minions.pushToDelegationFile(str(repo.full_name)+','+str(repo.id))

'''
Given the duration of runtime as seconds, this method will print out the duration
as hours, minutes and seconds.
'''
def announceRunTimeFromSeconds(seconds):
    remains = int(seconds)
    actualSeconds = remains-60*int(remains/60) #remove full minutes and bigger
    remains -= actualSeconds
    actualMinutes = int((remains-3600*int(remains/3600))/60) #remove full hours and covert seconds to minutes
    remains -= actualMinutes*60
    actualHours   = int(remains/21600) #convert seconds to hours
    outputString = "Run time:"
    if actualHours>0:
        outputString += " "+str(actualHours)+" hours"
    if actualMinutes>0:
        outputString += " "+str(actualMinutes)+" minutes"
    if actualSeconds>0:
        outputString += " "+str(actualSeconds)+" seconds ("+str(seconds)+" total seconds)"
    print outputString

def printHowToUse():
    print "Usage: python NonSomeFinder.py [-ssl|-facebook] [-issues] -search searchword outputfile.csv"
    print "Usage: python NonSomeFinder.py [-ssl|-facebook] [-issues] -search searchword [-delegate delegationfile1[,delegationfile2[...]]"
    print "Usage: python NonSomeFinder.py -facebook [-since #id] outputfile.csv"
    print "Usage: python NonSomeFinder.py -facebook [-since #id] [-delegate delegationfile1[,delegationfile2[...]]"
    print "Usage: python NonSomeFinder.py -ssl|-facebook -takeover delegationfile outputfile.csv"

if __name__ == '__main__':
    config = BgtConfiguration()
    try:
        config.readConfigfile(os.path.dirname(__file__)+'/config.cfg')
        config.parseCommandLine(sys.argv)
    except BadCommandLineException as e:
        print e.message
        printHowToUse()
        sys.exit()

    #Ready to start working!
    if config.search != "":
        print "Looking for "+config.search
    connection = GitHubDao.GitHubDao(config)
    if config.outputfile != "":
        csvDao     = CsvDao.CsvDao(config.outputfile)
    startTime = int(time.time())
    if config.delegation == "takeover":   #We will either analyse repositories from a delegation file...
        hits = GitHubFacadeForProcessDistibutionDao(config.delegatepath, connection)
    elif config.search == "":             #...or look through all repositories...
        hits = connection.findAllRepositories(config.sinceid)
    elif config.lookinto == "reponame":   #...or seek by the names of the repositories...
        hits = connection.findRepositoryNamesWithSearchPhrase(config.search)
    else:                          #...or seek for repositories with keyword in issues.
        hits = connection.findRepositoryIssuesWithSearchPhrase(config.search)

    if config.delegation == "delegate":   #Delegate forth the analysing job
        delegateRepositories(config, hits)
    else:                          #The search result pipe is given as a parameter
        analyseRepositories(hits)

    endTime = int(time.time())
    announceRunTimeFromSeconds(endTime-startTime)
<commit_msg>bugfix<commit_after>'''
Created on Nov 14, 2014
Copied from GitHubResearchDataMiner

@author: bgt
'''

import ConfigParser
import os
import sys
import time
import CsvDao
import GitHubDao
from GitHubFacadeForProcessDistibutionDao import GitHubFacadeForProcessDistibutionDao
from github.GithubException import GithubException
import ProcessDistributionDao
from BgtConfiguration import BgtConfiguration
from BgtConfiguration import BadCommandLineException

def analyseRepositories(hits):
    countDooku = 0

    for repo in hits:
        countDooku += 1
        try:
            print "Analysing repository #"+str(countDooku)+", "+repo.full_name+" "+str(repo.id)
            if config.searching == "facebook":
                analysis = connection.usesFacebookGraph(repo)
            else:
                analysis = connection.usesSsl(repo)
            csvDao.addRow(analysis)
        except GithubException:
            print "...except that repository has blocked access"
    csvDao.close()

def delegateRepositories(config, hits):
    minions = ProcessDistributionDao.ProcessDistributionDao(config, True)
    for repo in hits:
        minions.pushToDelegationFile(str(repo.full_name)+','+str(repo.id))

'''
Given the duration of runtime as seconds, this method will print out the duration
as hours, minutes and seconds.
'''
def announceRunTimeFromSeconds(seconds):
    remains = int(seconds)
    actualSeconds = remains-60*int(remains/60) #remove full minutes and bigger
    remains -= actualSeconds
    actualMinutes = int((remains-3600*int(remains/3600))/60) #remove full hours and covert seconds to minutes
    remains -= actualMinutes*60
    actualHours   = int(remains/21600) #convert seconds to hours
    outputString = "Run time:"
    if actualHours>0:
        outputString += " "+str(actualHours)+" hours"
    if actualMinutes>0:
        outputString += " "+str(actualMinutes)+" minutes"
    if actualSeconds>0:
        outputString += " "+str(actualSeconds)+" seconds ("+str(seconds)+" total seconds)"
    print outputString

def printHowToUse():
    print "Usage: python NonSomeFinder.py [-ssl|-facebook] [-issues] -search searchword outputfile.csv"
    print "Usage: python NonSomeFinder.py [-ssl|-facebook] [-issues] -search searchword [-delegate delegationfile1[,delegationfile2[...]]"
    print "Usage: python NonSomeFinder.py -facebook [-since #id] outputfile.csv"
    print "Usage: python NonSomeFinder.py -facebook [-since #id] [-delegate delegationfile1[,delegationfile2[...]]"
    print "Usage: python NonSomeFinder.py -ssl|-facebook -takeover delegationfile outputfile.csv"

if __name__ == '__main__':
    config = BgtConfiguration()
    try:
        config.readConfigfile(os.path.dirname(__file__)+'/config.cfg')
        config.parseCommandLine(sys.argv)
    except BadCommandLineException as e:
        print e.message
        printHowToUse()
        sys.exit()

    #Ready to start working!
    if config.search != "":
        print "Looking for "+config.search
    connection = GitHubDao.GitHubDao(config)
    if config.outputfile != "":
        csvDao     = CsvDao.CsvDao(config.outputfile)
    startTime = int(time.time())
    if config.delegation == "takeover":   #We will either analyse repositories from a delegation file...
        hits = GitHubFacadeForProcessDistibutionDao(config, connection)
    elif config.search == "":             #...or look through all repositories...
        hits = connection.findAllRepositories(config.sinceid)
    elif config.lookinto == "reponame":   #...or seek by the names of the repositories...
        hits = connection.findRepositoryNamesWithSearchPhrase(config.search)
    else:                          #...or seek for repositories with keyword in issues.
        hits = connection.findRepositoryIssuesWithSearchPhrase(config.search)

    if config.delegation == "delegate":   #Delegate forth the analysing job
        delegateRepositories(config, hits)
    else:                          #The search result pipe is given as a parameter
        analyseRepositories(hits)

    endTime = int(time.time())
    announceRunTimeFromSeconds(endTime-startTime)
<|endoftext|>