<commit_before>import caching.base as caching
import jingo
import jinja2
from tower import ugettext_lazy as _

import amo
from addons.models import Addon
from api.views import addon_filter
from bandwagon.models import Collection, MonthlyPick as MP
from versions.compare import version_int

# The global registry for promo modules.  Managed through PromoModuleMeta.
registry = {}

class PromoModuleMeta(type):
    """Adds new PromoModules to the module registry."""

    def __new__(mcs, name, bases, dict_):
        cls = type.__new__(mcs, name, bases, dict_)
        if 'abstract' not in dict_:
            registry[cls.slug] = cls
        return cls

class PromoModule(object):
    """
    Base class for promo modules in the discovery pane.

    Subclasses should assign a slug and define render().  The slug is only used
    internally, so it doesn't have to really be a slug.
    """
    __metaclass__ = PromoModuleMeta
    abstract = True
    slug = None

    def __init__(self, request, platform, version):
        self.request = request
        self.platform = platform
        self.version = version
        self.compat_mode = 'strict'
        if version_int(self.version) >= version_int('10.0'):
            self.compat_mode = 'ignore'

    def render(self):
        raise NotImplementedError

class TemplatePromo(PromoModule):
    abstract = True
    template = None

    def context(self):
        return {}

    def render(self, **kw):
        c = dict(self.context())
        c.update(kw)
        r = jingo.render_to_string(self.request, self.template, c)
        return jinja2.Markup(r)

class MonthlyPick(TemplatePromo):
    slug = 'Monthly Pick'
    template = 'discovery/modules/monthly.html'

    def context(self):
        try:
            pick = MP.objects.filter(locale=self.request.LANG)[0]
        except IndexError:
            try:
                pick = MP.objects.filter(locale='')[0]
            except IndexError:
                pick = None
        return {'pick': pick, 'module_context': 'discovery'}

class GoMobile(TemplatePromo):
    slug = 'Go Mobile'
    template = 'discovery/modules/go-mobile.html'

class CollectionPromo(PromoModule):
    abstract = True
    template = 'discovery/modules/collection.html'
    title = None
    subtitle = None
    cls = 'promo'
    limit = 3
    linkify_title = False

    def __init__(self, *args, **kw):
        super(CollectionPromo, self).__init__(*args, **kw)
        self.collection = None
        if hasattr(self, 'pk'):
            try:
                self.collection = Collection.objects.get(pk=self.pk)
            except Collection.DoesNotExist:
                pass
        elif (hasattr(self, 'collection_author') and
              hasattr(self, 'collection_slug')):
            try:
                self.collection = Collection.objects.get(
                    author__username=self.collection_author,
                    slug=self.collection_slug)
            except Collection.DoesNotExist:
                pass

    def get_descriptions(self):
        return {}

    def get_addons(self):
        addons = self.collection.addons.filter(status=amo.STATUS_PUBLIC)
        kw = dict(addon_type='ALL', limit=self.limit, app=self.request.APP,
                  platform=self.platform, version=self.version,
                  compat_mode=self.compat_mode)
        f = lambda: addon_filter(addons, **kw)
        return caching.cached_with(addons, f, repr(kw))

    def render(self, module_context='discovery'):
        if module_context == 'home':
            self.platform = 'ALL'
            self.version = None
        c = dict(promo=self, module_context=module_context,
                 descriptions=self.get_descriptions())
        if self.collection:
            c.update(addons=self.get_addons())
        return jinja2.Markup(
            jingo.render_to_string(self.request, self.template, c))

class ShoppingCollection(CollectionPromo):
    slug = 'Shopping Collection'
    collection_author, collection_slug = 'mozilla', 'onlineshopping'
    cls = 'promo promo-purple'
    title = _(u'Shopping Made Easy')
    subtitle = _(u'Save on your favorite items '
                  'from the comfort of your browser.')

class WebdevCollection(CollectionPromo):
    slug = 'Webdev Collection'
    pk = 10
    cls = 'webdev'
    title = _(u'Build the perfect website')

class TesterCollection(CollectionPromo):
    slug = 'Firefox Tester Tools'
    pk = 82266
    cls = 'tester'
    title = _(u'Help test Firefox with these tools')

class StarterPack(CollectionPromo):
    slug = 'Starter Pack'
    pk = 153649
    id = 'starter'
    cls = 'promo'
    title = _(u'First time with Add-ons?')
    subtitle = _(u'Not to worry, here are three to get started.')

    def get_descriptions(self):
        return {
            2257: _(u'Translate content on the web from and into over 40 '
                    'languages.'),
            1833: _(u"Easily connect to your social networks, and share or "
                    "comment on the page you're visiting."),
            11377: _(u'A quick view to compare prices when you shop online '
                     'or search for flights.')
        }

class Fx4Collection(CollectionPromo):
    slug = 'Fx4 Collection'
    pk = 153651
    id = 'fx4-collection'
    cls = 'promo'
    title = _(u'Firefox 4 Collection')
    subtitle = _(u'Here are some great add-ons for Firefox 4.')
    linkify_title = True

class StPatricksPersonas(CollectionPromo):
    slug = 'St. Pat Themes'
    pk = 666627
    id = 'st-patricks'
    cls = 'promo'
    title = _(u'St. Patrick’s Day Themes')
    subtitle = _(u'Decorate your browser to celebrate '
                 'St. Patrick’s Day.')

class FxSummerCollection(CollectionPromo):
    slug = 'Fx Summer Collection'
    pk = 2128026
    id = 'fx4-collection'
    cls = 'promo'
    title = _(u'Firefox Summer Collection')
    subtitle = _(u'Here are some great add-ons for Firefox.')

class ThunderbirdCollection(CollectionPromo):
    slug = 'Thunderbird Collection'
    pk = 2128303
    id = 'tb-collection'
    cls = 'promo'
    title = _(u'Thunderbird Collection')
    subtitle = _(u'Here are some great add-ons for Thunderbird.')

class TravelCollection(CollectionPromo):
    slug = 'Travelers Pack'
    pk = 4
    id = 'travel'
    cls = 'promo'
    title = _(u'Sit Back and Relax')
    subtitle = _(u'Add-ons that help you on your travels!')

    def get_descriptions(self):
        return {
            5791: _(u"Displays a country flag depicting the location of the "
                    "current website's server and more."),
            1117: _(u'FoxClocks let you keep an eye on the time around the '
                    'world.'),
            11377: _(u'Automatically get the lowest price when you shop '
                     'online or search for flights.')
        }

class SchoolCollection(CollectionPromo):
    slug = 'School'
    pk = 2133887
    id = 'school'
    cls = 'promo'
    title = _(u'A+ add-ons for School')
    subtitle = _(u'Add-ons for teachers, parents, and students heading back '
                 'to school.')

    def get_descriptions(self):
        return {
            3456: _(u'Would you like to know which websites you can trust?'),
            2410: _(u'Xmarks is the #1 bookmarking add-on.'),
            2444: _(u'Web page and text translator, dictionary, and more!')
        }

# The add-ons that go with the promo modal. Not an actual PromoModule
class PromoVideoCollection():
    items = (349111, 349155, 349157, 52659, 5579, 252539, 11377, 2257)

    def get_items(self):
        items = Addon.objects.in_bulk(self.items)
        return [items[i] for i in self.items if i in items]

class NewYearCollection(CollectionPromo):
    slug = 'New Year'
    collection_author, collection_slug = 'mozilla', 'newyear_2012'
    id = 'new-year'
    title = _(u'Add-ons to help you on your way in 2012')

class ValentinesDay(CollectionPromo):
    slug = 'Valentines Day'
    collection_author, collection_slug = 'amoteam', 'va'
    id = 'valentines'
    title = _(u'Love is in the Air')
    subtitle = _(u'Add some romance to your Firefox.')

class MobileThemes(CollectionPromo):
    slug = 'Mobile Themes'
    cls = 'promo promo-grey'
    collection_author, collection_slug = 'mozilla', 'mobilethemes'
    title = _(u'Put a Theme on It!')
    subtitle = _(u'Visit addons.mozilla.org from Firefox for Android and '
                  'dress up your mobile browser to match your style, mood, '
                  'or the season.')

class Fitness(CollectionPromo):
    slug = 'Fitness'
    cls = 'promo promo-yellow'
    collection_author, collection_slug = 'mozilla', 'fitness'
    title = _(u'Get up and move!')
    subtitle = _(u'Install these fitness add-ons to keep you active and '
                  'healthy.')

class UpAndComing(CollectionPromo):
    slug = 'Up & Coming'
    cls = 'promo promo-blue'
    collection_author, collection_slug = 'mozilla', 'up_coming'
    title = _(u'New & Now')
    subtitle = _(u'Get the latest, must-have add-ons of the moment.')

class Olympics(TemplatePromo):
    slug = 'Olympics'
    template = 'discovery/modules/olympics.html'

class ContestWinners(TemplatePromo):
    slug = 'Contest Winners'
    template = 'discovery/modules/contest-winners.html'

    def render(self, module_context='discovery'):
        # Hide on discovery pane.
        if module_context == 'home':
            return super(ContestWinners, self).render()

class Holiday(TemplatePromo):
    slug = 'Holiday'
    template = 'discovery/modules/holiday.html'

    def render(self, module_context='discovery'):
        # Hide on discovery pane.
        if module_context == 'home':
            return super(Holiday, self).render()

class Privacy(CollectionPromo):
    slug = 'Privacy Collection'
    cls = 'promo promo-purple'
    collection_author, collection_slug = 'mozilla', 'privacy'
    title = _(u'Worry-free browsing')
    subtitle = _(u'Protect your privacy online with the add-ons in this '
                  'collection.')

class Featured(CollectionPromo):
    slug = 'Featured Add-ons Collection'
    cls = 'promo promo-yellow'
    collection_author, collection_slug = 'mozilla', 'featured-add-ons'
    title = _(u'Featured Add-ons')
    subtitle = _(u'Great add-ons for work, fun, privacy, productivity… '
                  'just about anything!')

class AmpYourFirefoxWinner(CollectionPromo):
    slug = 'Amp Your Firefox Best Overall Add-on'
    template = 'discovery/modules/amp-your-firefox-winner.html'
<commit_msg>Update identifiers for ValentinesDay (bug 970500)<commit_after>import caching.base as caching
import jingo
import jinja2
from tower import ugettext_lazy as _

import amo
from addons.models import Addon
from api.views import addon_filter
from bandwagon.models import Collection, MonthlyPick as MP
from versions.compare import version_int

# The global registry for promo modules.  Managed through PromoModuleMeta.
registry = {}

class PromoModuleMeta(type):
    """Adds new PromoModules to the module registry."""

    def __new__(mcs, name, bases, dict_):
        cls = type.__new__(mcs, name, bases, dict_)
        if 'abstract' not in dict_:
            registry[cls.slug] = cls
        return cls

class PromoModule(object):
    """
    Base class for promo modules in the discovery pane.

    Subclasses should assign a slug and define render().  The slug is only used
    internally, so it doesn't have to really be a slug.
    """
    __metaclass__ = PromoModuleMeta
    abstract = True
    slug = None

    def __init__(self, request, platform, version):
        self.request = request
        self.platform = platform
        self.version = version
        self.compat_mode = 'strict'
        if version_int(self.version) >= version_int('10.0'):
            self.compat_mode = 'ignore'

    def render(self):
        raise NotImplementedError

class TemplatePromo(PromoModule):
    abstract = True
    template = None

    def context(self):
        return {}

    def render(self, **kw):
        c = dict(self.context())
        c.update(kw)
        r = jingo.render_to_string(self.request, self.template, c)
        return jinja2.Markup(r)

class MonthlyPick(TemplatePromo):
    slug = 'Monthly Pick'
    template = 'discovery/modules/monthly.html'

    def context(self):
        try:
            pick = MP.objects.filter(locale=self.request.LANG)[0]
        except IndexError:
            try:
                pick = MP.objects.filter(locale='')[0]
            except IndexError:
                pick = None
        return {'pick': pick, 'module_context': 'discovery'}

class GoMobile(TemplatePromo):
    slug = 'Go Mobile'
    template = 'discovery/modules/go-mobile.html'

class CollectionPromo(PromoModule):
    abstract = True
    template = 'discovery/modules/collection.html'
    title = None
    subtitle = None
    cls = 'promo'
    limit = 3
    linkify_title = False

    def __init__(self, *args, **kw):
        super(CollectionPromo, self).__init__(*args, **kw)
        self.collection = None
        if hasattr(self, 'pk'):
            try:
                self.collection = Collection.objects.get(pk=self.pk)
            except Collection.DoesNotExist:
                pass
        elif (hasattr(self, 'collection_author') and
              hasattr(self, 'collection_slug')):
            try:
                self.collection = Collection.objects.get(
                    author__username=self.collection_author,
                    slug=self.collection_slug)
            except Collection.DoesNotExist:
                pass

    def get_descriptions(self):
        return {}

    def get_addons(self):
        addons = self.collection.addons.filter(status=amo.STATUS_PUBLIC)
        kw = dict(addon_type='ALL', limit=self.limit, app=self.request.APP,
                  platform=self.platform, version=self.version,
                  compat_mode=self.compat_mode)
        f = lambda: addon_filter(addons, **kw)
        return caching.cached_with(addons, f, repr(kw))

    def render(self, module_context='discovery'):
        if module_context == 'home':
            self.platform = 'ALL'
            self.version = None
        c = dict(promo=self, module_context=module_context,
                 descriptions=self.get_descriptions())
        if self.collection:
            c.update(addons=self.get_addons())
        return jinja2.Markup(
            jingo.render_to_string(self.request, self.template, c))

class ShoppingCollection(CollectionPromo):
    slug = 'Shopping Collection'
    collection_author, collection_slug = 'mozilla', 'onlineshopping'
    cls = 'promo promo-purple'
    title = _(u'Shopping Made Easy')
    subtitle = _(u'Save on your favorite items '
                  'from the comfort of your browser.')

class WebdevCollection(CollectionPromo):
    slug = 'Webdev Collection'
    pk = 10
    cls = 'webdev'
    title = _(u'Build the perfect website')

class TesterCollection(CollectionPromo):
    slug = 'Firefox Tester Tools'
    pk = 82266
    cls = 'tester'
    title = _(u'Help test Firefox with these tools')

class StarterPack(CollectionPromo):
    slug = 'Starter Pack'
    pk = 153649
    id = 'starter'
    cls = 'promo'
    title = _(u'First time with Add-ons?')
    subtitle = _(u'Not to worry, here are three to get started.')

    def get_descriptions(self):
        return {
            2257: _(u'Translate content on the web from and into over 40 '
                    'languages.'),
            1833: _(u"Easily connect to your social networks, and share or "
                    "comment on the page you're visiting."),
            11377: _(u'A quick view to compare prices when you shop online '
                     'or search for flights.')
        }

class Fx4Collection(CollectionPromo):
    slug = 'Fx4 Collection'
    pk = 153651
    id = 'fx4-collection'
    cls = 'promo'
    title = _(u'Firefox 4 Collection')
    subtitle = _(u'Here are some great add-ons for Firefox 4.')
    linkify_title = True

class StPatricksPersonas(CollectionPromo):
    slug = 'St. Pat Themes'
    pk = 666627
    id = 'st-patricks'
    cls = 'promo'
    title = _(u'St. Patrick’s Day Themes')
    subtitle = _(u'Decorate your browser to celebrate '
                 'St. Patrick’s Day.')

class FxSummerCollection(CollectionPromo):
    slug = 'Fx Summer Collection'
    pk = 2128026
    id = 'fx4-collection'
    cls = 'promo'
    title = _(u'Firefox Summer Collection')
    subtitle = _(u'Here are some great add-ons for Firefox.')

class ThunderbirdCollection(CollectionPromo):
    slug = 'Thunderbird Collection'
    pk = 2128303
    id = 'tb-collection'
    cls = 'promo'
    title = _(u'Thunderbird Collection')
    subtitle = _(u'Here are some great add-ons for Thunderbird.')

class TravelCollection(CollectionPromo):
    slug = 'Travelers Pack'
    pk = 4
    id = 'travel'
    cls = 'promo'
    title = _(u'Sit Back and Relax')
    subtitle = _(u'Add-ons that help you on your travels!')

    def get_descriptions(self):
        return {
            5791: _(u"Displays a country flag depicting the location of the "
                    "current website's server and more."),
            1117: _(u'FoxClocks let you keep an eye on the time around the '
                    'world.'),
            11377: _(u'Automatically get the lowest price when you shop '
                     'online or search for flights.')
        }

class SchoolCollection(CollectionPromo):
    slug = 'School'
    pk = 2133887
    id = 'school'
    cls = 'promo'
    title = _(u'A+ add-ons for School')
    subtitle = _(u'Add-ons for teachers, parents, and students heading back '
                 'to school.')

    def get_descriptions(self):
        return {
            3456: _(u'Would you like to know which websites you can trust?'),
            2410: _(u'Xmarks is the #1 bookmarking add-on.'),
            2444: _(u'Web page and text translator, dictionary, and more!')
        }

# The add-ons that go with the promo modal. Not an actual PromoModule
class PromoVideoCollection():
    items = (349111, 349155, 349157, 52659, 5579, 252539, 11377, 2257)

    def get_items(self):
        items = Addon.objects.in_bulk(self.items)
        return [items[i] for i in self.items if i in items]

class NewYearCollection(CollectionPromo):
    slug = 'New Year'
    collection_author, collection_slug = 'mozilla', 'newyear_2012'
    id = 'new-year'
    title = _(u'Add-ons to help you on your way in 2012')

class ValentinesDay(CollectionPromo):
    slug = 'Valentines Day'
    collection_author, collection_slug = 'mozilla', 'bemine'
    id = 'valentines'
    title = _(u'Love is in the Air')
    subtitle = _(u'Add some romance to your Firefox.')

class MobileThemes(CollectionPromo):
    slug = 'Mobile Themes'
    cls = 'promo promo-grey'
    collection_author, collection_slug = 'mozilla', 'mobilethemes'
    title = _(u'Put a Theme on It!')
    subtitle = _(u'Visit addons.mozilla.org from Firefox for Android and '
                  'dress up your mobile browser to match your style, mood, '
                  'or the season.')

class Fitness(CollectionPromo):
    slug = 'Fitness'
    cls = 'promo promo-yellow'
    collection_author, collection_slug = 'mozilla', 'fitness'
    title = _(u'Get up and move!')
    subtitle = _(u'Install these fitness add-ons to keep you active and '
                  'healthy.')

class UpAndComing(CollectionPromo):
    slug = 'Up & Coming'
    cls = 'promo promo-blue'
    collection_author, collection_slug = 'mozilla', 'up_coming'
    title = _(u'New & Now')
    subtitle = _(u'Get the latest, must-have add-ons of the moment.')

class Olympics(TemplatePromo):
    slug = 'Olympics'
    template = 'discovery/modules/olympics.html'

class ContestWinners(TemplatePromo):
    slug = 'Contest Winners'
    template = 'discovery/modules/contest-winners.html'

    def render(self, module_context='discovery'):
        # Hide on discovery pane.
        if module_context == 'home':
            return super(ContestWinners, self).render()

class Holiday(TemplatePromo):
    slug = 'Holiday'
    template = 'discovery/modules/holiday.html'

    def render(self, module_context='discovery'):
        # Hide on discovery pane.
        if module_context == 'home':
            return super(Holiday, self).render()

class Privacy(CollectionPromo):
    slug = 'Privacy Collection'
    cls = 'promo promo-purple'
    collection_author, collection_slug = 'mozilla', 'privacy'
    title = _(u'Worry-free browsing')
    subtitle = _(u'Protect your privacy online with the add-ons in this '
                  'collection.')

class Featured(CollectionPromo):
    slug = 'Featured Add-ons Collection'
    cls = 'promo promo-yellow'
    collection_author, collection_slug = 'mozilla', 'featured-add-ons'
    title = _(u'Featured Add-ons')
    subtitle = _(u'Great add-ons for work, fun, privacy, productivity… '
                  'just about anything!')

class AmpYourFirefoxWinner(CollectionPromo):
    slug = 'Amp Your Firefox Best Overall Add-on'
    template = 'discovery/modules/amp-your-firefox-winner.html'
<|endoftext|>


<commit_before>from __future__ import unicode_literals, division, absolute_import
from builtins import *  # pylint: disable=unused-import, redefined-builtin
from future.utils import PY3

import csv
import logging
import re
from collections import MutableSet
from datetime import datetime

from requests.exceptions import RequestException

from sqlalchemy import Column, Unicode, String
from sqlalchemy.orm import relation
from sqlalchemy.schema import ForeignKey

from flexget import plugin, db_schema
from flexget.entry import Entry
from flexget.event import event
from flexget.plugin import PluginError
from flexget.manager import Session
from flexget.utils.database import json_synonym
from flexget.utils.requests import Session as RequestSession, TimedLimiter
from flexget.utils.soup import get_soup

log = logging.getLogger('imdb_list')
IMMUTABLE_LISTS = ['ratings', 'checkins']

Base = db_schema.versioned_base('imdb_list', 0)

MOVIE_TYPES = ['feature film', 'documentary']
SERIES_TYPES = ['tv series']

class IMDBListUser(Base):
    __tablename__ = "imdb_list_user"

    user_id = Column(String, primary_key=True)
    user_name = Column(Unicode)
    _cookies = Column('cookies', Unicode)
    cookies = json_synonym('_cookies')

    lists = relation('IMDBListList', backref='imdb_user', cascade='all, delete, delete-orphan')

    def __init__(self, user_name, user_id, cookies):
        self.user_name = user_name
        self.user_id = user_id
        self.cookies = cookies

class IMDBListList(Base):
    __tablename__ = "imdb_list_lists"

    list_id = Column(Unicode, primary_key=True)
    list_name = Column(Unicode)
    user_id = Column(String, ForeignKey('imdb_list_user.user_id'))

    def __init__(self, list_id, list_name, user_id):
        self.list_id = list_id
        self.list_name = list_name
        self.user_id = user_id

if PY3:
    csv_reader = csv.reader
else:
    def csv_reader(iterable, dialect='excel', *args, **kwargs):
        """
        Compatibilty function to make python 2 act like python 3
        Always takes and returns text (no bytes).
        """
        iterable = (l.encode('utf-8') for l in iterable)
        for row in csv.reader(iterable):
            yield [cell.decode('utf-8') for cell in row]

class ImdbEntrySet(MutableSet):
    schema = {
        'type': 'object',
        'properties': {
            'login': {'type': 'string'},
            'password': {'type': 'string'},
            'list': {'type': 'string'},
            'force_language': {'type': 'string', 'default': 'en-us'}
        },
        'additionalProperties': False,
        'required': ['login', 'password', 'list']
    }

    def __init__(self, config):
        self.config = config
        self._session = RequestSession()
        self._session.add_domain_limiter(TimedLimiter('imdb.com', '5 seconds'))
        self._session.headers.update({'Accept-Language': config.get('force_language', 'en-us')})
        self.user_id = None
        self.list_id = None
        self.cookies = None
        self._items = None
        self._authenticated = False

    @property
    def session(self):
        if not self._authenticated:
            self.authenticate()
        return self._session

    def authenticate(self):
        """Authenticates a session with IMDB, and grabs any IDs needed for getting/modifying list."""
        cached_credentials = False
        with Session() as session:
            user = session.query(IMDBListUser).filter(IMDBListUser.user_name == self.config.get('login')).one_or_none()
            if user and user.cookies and user.user_id:
                log.debug('login credentials found in cache, testing')
                self.cookies = user.cookies
                self.user_id = user.user_id
                r = self._session.head('http://www.imdb.com/profile', allow_redirects=False, cookies=self.cookies)
                if not r.headers.get('location') or 'login' in r.headers['location']:
                    log.debug('cache credentials expired')
                else:
                    cached_credentials = True
            if not cached_credentials:
                log.debug('user credentials not found in cache or outdated, fetching from IMDB')
                url_credentials = (
                'https://www.imdb.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.imdb.com%2Fap-signin-'
                'handler&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&'
                'openid.assoc_handle=imdb_mobile_us&openid.mode=checkid_setup&openid.claimed_id=http%3A%'
                '2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.ope'
                'nid.net%2Fauth%2F2.0'
                )
                try:
                    r = self._session.get(url_credentials)
                except RequestException as e:
                    raise PluginError(e.args[0])
                soup = get_soup(r.content)
                inputs = soup.select('form#ap_signin_form input')
                data = dict((i['name'], i.get('value')) for i in inputs if i.get('name'))
                data['email'] = self.config['login']
                data['password'] = self.config['password']
                action = soup.find('form', id='ap_signin_form').get('action')
                log.debug('email=%s, password=%s', data['email'], data['password'])
                self._session.headers.update({'Referer': url_credentials})
                d = self._session.post(action, data=data)
                self._session.headers.update({'Referer': 'http://www.imdb.com/'})
                # Get user id by extracting from redirect url
                r = self._session.head('http://www.imdb.com/profile', allow_redirects=False)
                if not r.headers.get('location') or 'login' in r.headers['location']:
                    raise plugin.PluginError('Login to IMDB failed. Check your credentials.')
                self.user_id = re.search('ur\d+(?!\d)', r.headers['location']).group()
                self.cookies = dict(d.cookies)
                # Get list ID
            if user:
                for list in user.lists:
                    if self.config['list'] == list.list_name:
                        log.debug('found list ID %s matching list name %s in cache', list.list_id, list.list_name)
                        self.list_id = list.list_id
            if not self.list_id:
                log.debug('could not find list ID in cache, fetching from IMDB')
                if self.config['list'] == 'watchlist':
                    data = {'consts[]': 'tt0133093', 'tracking_tag': 'watchlistRibbon'}
                    wl_data = self._session.post('http://www.imdb.com/list/_ajax/watchlist_has', data=data,
                                                 cookies=self.cookies).json()
                    try:
                        self.list_id = wl_data['list_id']
                    except KeyError:
                        raise PluginError('No list ID could be received. Please initialize list by '
                                          'manually adding an item to it and try again')
                elif self.config['list'] in IMMUTABLE_LISTS or self.config['list'].startswith('ls'):
                    self.list_id = self.config['list']
                else:
                    data = {'tconst': 'tt0133093'}
                    list_data = self._session.post('http://www.imdb.com/list/_ajax/wlb_dropdown', data=data,
                                                   cookies=self.cookies).json()
                    for li in list_data['items']:
                        if li['wlb_text'] == self.config['list']:
                            self.list_id = li['data_list_id']
                            break
                    else:
                        raise plugin.PluginError('Could not find list %s' % self.config['list'])

            user = IMDBListUser(self.config['login'], self.user_id, self.cookies)
            list = IMDBListList(self.list_id, self.config['list'], self.user_id)
            user.lists.append(list)
            session.merge(user)

        self._authenticated = True

    def invalidate_cache(self):
        self._items = None

    @property
    def items(self):
        if self._items is None:
            log.debug('fetching items from IMDB')
            try:
                r = self.session.get('http://www.imdb.com/list/export?list_id=%s&author_id=%s' %
                                     (self.list_id, self.user_id), cookies=self.cookies)

            except RequestException as e:
                raise PluginError(e.args[0])
            lines = r.iter_lines(decode_unicode=True)
            # Throw away first line with headers
            next(lines)
            self._items = []
            for row in csv_reader(lines):
                log.debug('parsing line from csv: %s', ', '.join(row))
                if not len(row) == 16:
                    log.debug('no movie row detected, skipping. %s', ', '.join(row))
                    continue
                entry = Entry({
                    'title': '%s (%s)' % (row[5], row[11]) if row[11] != '????' else '%s' % row[5],
                    'url': row[15],
                    'imdb_id': row[1],
                    'imdb_url': row[15],
                    'imdb_list_position': int(row[0]),
                    'imdb_list_created': datetime.strptime(row[2], '%a %b %d %H:%M:%S %Y') if row[2] else None,
                    'imdb_list_modified': datetime.strptime(row[3], '%a %b %d %H:%M:%S %Y') if row[3] else None,
                    'imdb_list_description': row[4],
                    'imdb_name': row[5],
                    'imdb_year': int(row[11]) if row[11] != '????' else None,
                    'imdb_score': float(row[9]) if row[9] else None,
                    'imdb_user_score': float(row[8]) if row[8] else None,
                    'imdb_votes': int(row[13]) if row[13] else None,
                    'imdb_genres': [genre.strip() for genre in row[12].split(',')]
                })
                item_type = row[6].lower()
                name = row[5]
                year = int(row[11]) if row[11] != '????' else None
                if item_type in MOVIE_TYPES:
                    entry['movie_name'] = name
                    entry['movie_year'] = year
                elif item_type in SERIES_TYPES:
                    entry['series_name'] = name
                    entry['series_year'] = year
                else:
                    log.verbose('Unknown IMDB type entry received: %s. Skipping', item_type)
                    continue
                self._items.append(entry)
        return self._items

    @property
    def immutable(self):
        if self.config['list'] in IMMUTABLE_LISTS:
            return '%s list is not modifiable' % self.config['list']

    def _from_iterable(cls, it):
        # TODO: is this the right answer? the returned object won't have our custom __contains__ logic
        return set(it)

    def _find_movie(self, entry):
        log.debug('trying to match %s to existing list items', entry['imdb_id'])
        for e in self.items:
            if e['imdb_id'] == entry['imdb_id']:
                return e
        log.debug('could not match %s to existing list items', entry['imdb_id'])

    def __contains__(self, entry):
        if not entry.get('imdb_id'):
            log.debug('entry %s does not have imdb_id, skipping', entry)
            return False
        return self._find_movie(entry) is not None

    def __iter__(self):
        return iter(self.items)

    def discard(self, entry):
        if self.config['list'] in IMMUTABLE_LISTS:
            raise plugin.PluginError('%s lists are not modifiable' % ' and '.join(IMMUTABLE_LISTS))
        if 'imdb_id' not in entry:
            log.warning('Cannot remove %s from imdb_list because it does not have an imdb_id', entry['title'])
            return
        # Get the list item id
        item_ids = None
        if self.config['list'] == 'watchlist':
            data = {'consts[]': entry['imdb_id'], 'tracking_tag': 'watchlistRibbon'}
            status = self.session.post('http://www.imdb.com/list/_ajax/watchlist_has', data=data,
                                       cookies=self.cookies).json()
            item_ids = status.get('has', {}).get(entry['imdb_id'])
        else:
            data = {'tconst': entry['imdb_id']}
            status = self.session.post('http://www.imdb.com/list/_ajax/wlb_dropdown', data=data,
                                       cookies=self.cookies).json()
            for a_list in status['items']:
                if a_list['data_list_id'] == self.list_id:
                    item_ids = a_list['data_list_item_ids']
                    break
        if not item_ids:
            log.warning('%s is not in list %s, cannot be removed', entry['imdb_id'], self.list_id)
            return
        data = {
            'action': 'delete',
            'list_id': self.list_id,
            'ref_tag': 'title'
        }
        for item_id in item_ids:
            log.debug('found movie %s with ID %s in list %s, removing', entry['title'], entry['imdb_id'], self.list_id)
            self.session.post('http://www.imdb.com/list/_ajax/edit', data=dict(data, list_item_id=item_id),
                              cookies=self.cookies)
            # We don't need to invalidate our cache if we remove the item
            self._items = [i for i in self._items if i['imdb_id'] != entry['imdb_id']] if self._items else None

    def _add(self, entry):
        """Submit a new movie to imdb. (does not update cache)"""
        if self.config['list'] in IMMUTABLE_LISTS:
            raise plugin.PluginError('%s lists are not modifiable' % ' and '.join(IMMUTABLE_LISTS))
        if 'imdb_id' not in entry:
            log.warning('Cannot add %s to imdb_list because it does not have an imdb_id', entry['title'])
            return
        # Manually calling authenticate to fetch list_id and cookies
        self.authenticate()
        data = {
            'const': entry['imdb_id'],
            'list_id': self.list_id,
            'ref_tag': 'title'
        }
        log.debug('adding title %s with ID %s to imdb %s', entry['title'], entry['imdb_id'], self.list_id)
        self.session.post('http://www.imdb.com/list/_ajax/edit', data=data, cookies=self.cookies)

    def add(self, entry):
        self._add(entry)
        # Invalidate the cache so that we get the canonical entry from the imdb list
        self.invalidate_cache()

    def __ior__(self, entries):
        for entry in entries:
            self._add(entry)
        self.invalidate_cache()
        return self

    def __len__(self):
        return len(self.items)

    @property
    def online(self):
        """ Set the online status of the plugin, online plugin should be treated differently in certain situations,
        like test mode"""
        return True

    def get(self, entry):
        return self._find_movie(entry)

class ImdbList(object):
    schema = ImdbEntrySet.schema

    @staticmethod
    def get_list(config):
        return ImdbEntrySet(config)

    def on_task_input(self, task, config):
        return list(self.get_list(config))

@event('plugin.register')
def register_plugin():
    plugin.register(ImdbList, 'imdb_list', api_ver=2, groups=['list'])
<commit_msg>[fix] IMDB List - Added more supported types. Fixes #1343<commit_after>from __future__ import unicode_literals, division, absolute_import
from builtins import *  # pylint: disable=unused-import, redefined-builtin
from future.utils import PY3

import csv
import logging
import re
from collections import MutableSet
from datetime import datetime

from requests.exceptions import RequestException

from sqlalchemy import Column, Unicode, String
from sqlalchemy.orm import relation
from sqlalchemy.schema import ForeignKey

from flexget import plugin, db_schema
from flexget.entry import Entry
from flexget.event import event
from flexget.plugin import PluginError
from flexget.manager import Session
from flexget.utils.database import json_synonym
from flexget.utils.requests import Session as RequestSession, TimedLimiter
from flexget.utils.soup import get_soup

log = logging.getLogger('imdb_list')
IMMUTABLE_LISTS = ['ratings', 'checkins']

Base = db_schema.versioned_base('imdb_list', 0)

MOVIE_TYPES = ['feature film', 'documentary', 'tv movie', 'video']
SERIES_TYPES = ['tv series', 'mini-series']

class IMDBListUser(Base):
    __tablename__ = "imdb_list_user"

    user_id = Column(String, primary_key=True)
    user_name = Column(Unicode)
    _cookies = Column('cookies', Unicode)
    cookies = json_synonym('_cookies')

    lists = relation('IMDBListList', backref='imdb_user', cascade='all, delete, delete-orphan')

    def __init__(self, user_name, user_id, cookies):
        self.user_name = user_name
        self.user_id = user_id
        self.cookies = cookies

class IMDBListList(Base):
    __tablename__ = "imdb_list_lists"

    list_id = Column(Unicode, primary_key=True)
    list_name = Column(Unicode)
    user_id = Column(String, ForeignKey('imdb_list_user.user_id'))

    def __init__(self, list_id, list_name, user_id):
        self.list_id = list_id
        self.list_name = list_name
        self.user_id = user_id

if PY3:
    csv_reader = csv.reader
else:
    def csv_reader(iterable, dialect='excel', *args, **kwargs):
        """
        Compatibilty function to make python 2 act like python 3
        Always takes and returns text (no bytes).
        """
        iterable = (l.encode('utf-8') for l in iterable)
        for row in csv.reader(iterable):
            yield [cell.decode('utf-8') for cell in row]

class ImdbEntrySet(MutableSet):
    schema = {
        'type': 'object',
        'properties': {
            'login': {'type': 'string'},
            'password': {'type': 'string'},
            'list': {'type': 'string'},
            'force_language': {'type': 'string', 'default': 'en-us'}
        },
        'additionalProperties': False,
        'required': ['login', 'password', 'list']
    }

    def __init__(self, config):
        self.config = config
        self._session = RequestSession()
        self._session.add_domain_limiter(TimedLimiter('imdb.com', '5 seconds'))
        self._session.headers.update({'Accept-Language': config.get('force_language', 'en-us')})
        self.user_id = None
        self.list_id = None
        self.cookies = None
        self._items = None
        self._authenticated = False

    @property
    def session(self):
        if not self._authenticated:
            self.authenticate()
        return self._session

    def authenticate(self):
        """Authenticates a session with IMDB, and grabs any IDs needed for getting/modifying list."""
        cached_credentials = False
        with Session() as session:
            user = session.query(IMDBListUser).filter(IMDBListUser.user_name == self.config.get('login')).one_or_none()
            if user and user.cookies and user.user_id:
                log.debug('login credentials found in cache, testing')
                self.cookies = user.cookies
                self.user_id = user.user_id
                r = self._session.head('http://www.imdb.com/profile', allow_redirects=False, cookies=self.cookies)
                if not r.headers.get('location') or 'login' in r.headers['location']:
                    log.debug('cache credentials expired')
                else:
                    cached_credentials = True
            if not cached_credentials:
                log.debug('user credentials not found in cache or outdated, fetching from IMDB')
                url_credentials = (
                'https://www.imdb.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.imdb.com%2Fap-signin-'
                'handler&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&'
                'openid.assoc_handle=imdb_mobile_us&openid.mode=checkid_setup&openid.claimed_id=http%3A%'
                '2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.ope'
                'nid.net%2Fauth%2F2.0'
                )
                try:
                    r = self._session.get(url_credentials)
                except RequestException as e:
                    raise PluginError(e.args[0])
                soup = get_soup(r.content)
                inputs = soup.select('form#ap_signin_form input')
                data = dict((i['name'], i.get('value')) for i in inputs if i.get('name'))
                data['email'] = self.config['login']
                data['password'] = self.config['password']
                action = soup.find('form', id='ap_signin_form').get('action')
                log.debug('email=%s, password=%s', data['email'], data['password'])
                self._session.headers.update({'Referer': url_credentials})
                d = self._session.post(action, data=data)
                self._session.headers.update({'Referer': 'http://www.imdb.com/'})
                # Get user id by extracting from redirect url
                r = self._session.head('http://www.imdb.com/profile', allow_redirects=False)
                if not r.headers.get('location') or 'login' in r.headers['location']:
                    raise plugin.PluginError('Login to IMDB failed. Check your credentials.')
                self.user_id = re.search('ur\d+(?!\d)', r.headers['location']).group()
                self.cookies = dict(d.cookies)
                # Get list ID
            if user:
                for list in user.lists:
                    if self.config['list'] == list.list_name:
                        log.debug('found list ID %s matching list name %s in cache', list.list_id, list.list_name)
                        self.list_id = list.list_id
            if not self.list_id:
                log.debug('could not find list ID in cache, fetching from IMDB')
                if self.config['list'] == 'watchlist':
                    data = {'consts[]': 'tt0133093', 'tracking_tag': 'watchlistRibbon'}
                    wl_data = self._session.post('http://www.imdb.com/list/_ajax/watchlist_has', data=data,
                                                 cookies=self.cookies).json()
                    try:
                        self.list_id = wl_data['list_id']
                    except KeyError:
                        raise PluginError('No list ID could be received. Please initialize list by '
                                          'manually adding an item to it and try again')
                elif self.config['list'] in IMMUTABLE_LISTS or self.config['list'].startswith('ls'):
                    self.list_id = self.config['list']
                else:
                    data = {'tconst': 'tt0133093'}
                    list_data = self._session.post('http://www.imdb.com/list/_ajax/wlb_dropdown', data=data,
                                                   cookies=self.cookies).json()
                    for li in list_data['items']:
                        if li['wlb_text'] == self.config['list']:
                            self.list_id = li['data_list_id']
                            break
                    else:
                        raise plugin.PluginError('Could not find list %s' % self.config['list'])

            user = IMDBListUser(self.config['login'], self.user_id, self.cookies)
            list = IMDBListList(self.list_id, self.config['list'], self.user_id)
            user.lists.append(list)
            session.merge(user)

        self._authenticated = True

    def invalidate_cache(self):
        self._items = None

    @property
    def items(self):
        if self._items is None:
            log.debug('fetching items from IMDB')
            try:
                r = self.session.get('http://www.imdb.com/list/export?list_id=%s&author_id=%s' %
                                     (self.list_id, self.user_id), cookies=self.cookies)

            except RequestException as e:
                raise PluginError(e.args[0])
            lines = r.iter_lines(decode_unicode=True)
            # Throw away first line with headers
            next(lines)
            self._items = []
            for row in csv_reader(lines):
                log.debug('parsing line from csv: %s', ', '.join(row))
                if not len(row) == 16:
                    log.debug('no movie row detected, skipping. %s', ', '.join(row))
                    continue
                entry = Entry({
                    'title': '%s (%s)' % (row[5], row[11]) if row[11] != '????' else '%s' % row[5],
                    'url': row[15],
                    'imdb_id': row[1],
                    'imdb_url': row[15],
                    'imdb_list_position': int(row[0]),
                    'imdb_list_created': datetime.strptime(row[2], '%a %b %d %H:%M:%S %Y') if row[2] else None,
                    'imdb_list_modified': datetime.strptime(row[3], '%a %b %d %H:%M:%S %Y') if row[3] else None,
                    'imdb_list_description': row[4],
                    'imdb_name': row[5],
                    'imdb_year': int(row[11]) if row[11] != '????' else None,
                    'imdb_score': float(row[9]) if row[9] else None,
                    'imdb_user_score': float(row[8]) if row[8] else None,
                    'imdb_votes': int(row[13]) if row[13] else None,
                    'imdb_genres': [genre.strip() for genre in row[12].split(',')]
                })
                item_type = row[6].lower()
                name = row[5]
                year = int(row[11]) if row[11] != '????' else None
                if item_type in MOVIE_TYPES:
                    entry['movie_name'] = name
                    entry['movie_year'] = year
                elif item_type in SERIES_TYPES:
                    entry['series_name'] = name
                    entry['series_year'] = year
                else:
                    log.verbose('Unknown IMDB type entry received: %s. Skipping', item_type)
                    continue
                self._items.append(entry)
        return self._items

    @property
    def immutable(self):
        if self.config['list'] in IMMUTABLE_LISTS:
            return '%s list is not modifiable' % self.config['list']

    def _from_iterable(cls, it):
        # TODO: is this the right answer? the returned object won't have our custom __contains__ logic
        return set(it)

    def _find_movie(self, entry):
        log.debug('trying to match %s to existing list items', entry['imdb_id'])
        for e in self.items:
            if e['imdb_id'] == entry['imdb_id']:
                return e
        log.debug('could not match %s to existing list items', entry['imdb_id'])

    def __contains__(self, entry):
        if not entry.get('imdb_id'):
            log.debug('entry %s does not have imdb_id, skipping', entry)
            return False
        return self._find_movie(entry) is not None

    def __iter__(self):
        return iter(self.items)

    def discard(self, entry):
        if self.config['list'] in IMMUTABLE_LISTS:
            raise plugin.PluginError('%s lists are not modifiable' % ' and '.join(IMMUTABLE_LISTS))
        if 'imdb_id' not in entry:
            log.warning('Cannot remove %s from imdb_list because it does not have an imdb_id', entry['title'])
            return
        # Get the list item id
        item_ids = None
        if self.config['list'] == 'watchlist':
            data = {'consts[]': entry['imdb_id'], 'tracking_tag': 'watchlistRibbon'}
            status = self.session.post('http://www.imdb.com/list/_ajax/watchlist_has', data=data,
                                       cookies=self.cookies).json()
            item_ids = status.get('has', {}).get(entry['imdb_id'])
        else:
            data = {'tconst': entry['imdb_id']}
            status = self.session.post('http://www.imdb.com/list/_ajax/wlb_dropdown', data=data,
                                       cookies=self.cookies).json()
            for a_list in status['items']:
                if a_list['data_list_id'] == self.list_id:
                    item_ids = a_list['data_list_item_ids']
                    break
        if not item_ids:
            log.warning('%s is not in list %s, cannot be removed', entry['imdb_id'], self.list_id)
            return
        data = {
            'action': 'delete',
            'list_id': self.list_id,
            'ref_tag': 'title'
        }
        for item_id in item_ids:
            log.debug('found movie %s with ID %s in list %s, removing', entry['title'], entry['imdb_id'], self.list_id)
            self.session.post('http://www.imdb.com/list/_ajax/edit', data=dict(data, list_item_id=item_id),
                              cookies=self.cookies)
            # We don't need to invalidate our cache if we remove the item
            self._items = [i for i in self._items if i['imdb_id'] != entry['imdb_id']] if self._items else None

    def _add(self, entry):
        """Submit a new movie to imdb. (does not update cache)"""
        if self.config['list'] in IMMUTABLE_LISTS:
            raise plugin.PluginError('%s lists are not modifiable' % ' and '.join(IMMUTABLE_LISTS))
        if 'imdb_id' not in entry:
            log.warning('Cannot add %s to imdb_list because it does not have an imdb_id', entry['title'])
            return
        # Manually calling authenticate to fetch list_id and cookies
        self.authenticate()
        data = {
            'const': entry['imdb_id'],
            'list_id': self.list_id,
            'ref_tag': 'title'
        }
        log.debug('adding title %s with ID %s to imdb %s', entry['title'], entry['imdb_id'], self.list_id)
        self.session.post('http://www.imdb.com/list/_ajax/edit', data=data, cookies=self.cookies)

    def add(self, entry):
        self._add(entry)
        # Invalidate the cache so that we get the canonical entry from the imdb list
        self.invalidate_cache()

    def __ior__(self, entries):
        for entry in entries:
            self._add(entry)
        self.invalidate_cache()
        return self

    def __len__(self):
        return len(self.items)

    @property
    def online(self):
        """ Set the online status of the plugin, online plugin should be treated differently in certain situations,
        like test mode"""
        return True

    def get(self, entry):
        return self._find_movie(entry)

class ImdbList(object):
    schema = ImdbEntrySet.schema

    @staticmethod
    def get_list(config):
        return ImdbEntrySet(config)

    def on_task_input(self, task, config):
        return list(self.get_list(config))

@event('plugin.register')
def register_plugin():
    plugin.register(ImdbList, 'imdb_list', api_ver=2, groups=['list'])
<|endoftext|>


<commit_before>"""read Sparameters from a CSV file and returns sax model.

TODO: write dat to csv converter
(from lumerical interconnect to SAX format and the other way around)
"""
import pathlib
from typing import Union

import jax
import jax.numpy as jnp
import numpy as np
import pandas as pd
from sax.typing_ import Float, Model

import gdsfactory as gf
from gdsfactory.config import sparameters_path
from gdsfactory.simulation.get_sparameters_path import get_sparameters_path_lumerical

wl_cband = np.linspace(1.500, 1.600, 128)

PathType = Union[str, pathlib.Path]

def model_from_csv(
    filepath: Union[PathType, pd.DataFrame],
    xkey: str = "wavelengths",
    xunits: float = 1,
    prefix: str = "s",
) -> Model:
    """Returns a SAX Model from Sparameters from a CSV file.

    The SAX Model is a function that returns a SAX SDict interpolated over wavelength.

    Args:
        filepath: CSV Sparameters path or pandas DataFrame.
        wl: wavelength to interpolate (um).
        xkey: key for wavelengths in file.
        xunits: x units in um from the loaded file (um). 1 means 1um.
        prefix: for the sparameters column names in file.
    """
    df = pd.read_csv(filepath) if not isinstance(filepath, pd.DataFrame) else filepath
    assert isinstance(df, pd.DataFrame)
    df = df.reset_index()  # maybe there is useful info in the index...
    dic = dict(zip(df.columns, jnp.asarray(df.values.T)))
    keys = list(dic.keys())

    if xkey not in keys:
        raise ValueError(f"{xkey!r} not in {keys}")

    nsparameters = (len(keys) - 1) // 2
    nports = int(nsparameters**0.5)

    x = dic[xkey] * xunits

    @jax.jit
    def model(wl: Float = jnp.asarray(wl_cband)):
        S = {}
        zero = jnp.zeros_like(x)
        for i in range(1, nports + 1):
            for j in range(1, nports + 1):
                m = jnp.interp(wl, x, dic.get(f"{prefix}{i}{j}m", zero))
                a = jnp.interp(wl, x, dic.get(f"{prefix}{i}{j}a", zero))
                S[f"o{i}", f"o{j}"] = m * jnp.exp(1j * a)

        return S

    return model

def _demo_mmi_lumerical_csv() -> None:
    import matplotlib.pyplot as plt
    from plot_model import plot_model

    filepath = get_sparameters_path_lumerical(gf.c.mmi1x2)
    mmi = model_from_csv(filepath=filepath, xkey="wavelengths")
    plot_model(mmi)
    plt.show()

def sdict_from_component_lumerical(component, **kwargs) -> Model:
    """Returns SDict based on lumerical simulation."""
    filepath = get_sparameters_path_lumerical(component=component, **kwargs)
    return model_from_csv(filepath=filepath)

mmi1x2 = gf.partial(sdict_from_component_lumerical, component=gf.components.mmi1x2)
mmi2x2 = gf.partial(sdict_from_component_lumerical, component=gf.components.mmi2x2)

grating_coupler_elliptical = model_from_csv(
    filepath=sparameters_path / "grating_coupler_ellipti_9d85a0c6_18c08cac.csv"
)

model_factory = dict(
    mmi1x2=mmi1x2, mmi2x2=mmi2x2, grating_coupler_elliptical=grating_coupler_elliptical
)

if __name__ == "__main__":

    import matplotlib.pyplot as plt
    from plot_model import plot_model

    # s = from_csv(filepath=filepath, wavelength=wl_cband, xunits=1e-3)
    # s21 = s[("o1", "o3")]
    # plt.plot(wl_cband, np.abs(s21) ** 2)
    # plt.show()
    # filepath = get_sparameters_path_lumerical(gf.c.mmi1x2)
    # mmi = model_from_csv(filepath=filepath, xkey="wavelengths")
    # plot_model(mmi)
    # plt.show()

    plot_model(grating_coupler_elliptical)
    plt.show()

    # model = partial(
    #     from_csv, filepath=filepath, xunits=1, xkey="wavelengths", prefix="s"
    # )
    # sax.plot.plot_model(model)
    # plt.show()
<commit_msg>'Refactored by Sourcery'<commit_after>"""read Sparameters from a CSV file and returns sax model.

TODO: write dat to csv converter
(from lumerical interconnect to SAX format and the other way around)
"""
import pathlib
from typing import Union

import jax
import jax.numpy as jnp
import numpy as np
import pandas as pd
from sax.typing_ import Float, Model

import gdsfactory as gf
from gdsfactory.config import sparameters_path
from gdsfactory.simulation.get_sparameters_path import get_sparameters_path_lumerical

wl_cband = np.linspace(1.500, 1.600, 128)

PathType = Union[str, pathlib.Path]

def model_from_csv(
    filepath: Union[PathType, pd.DataFrame],
    xkey: str = "wavelengths",
    xunits: float = 1,
    prefix: str = "s",
) -> Model:
    """Returns a SAX Model from Sparameters from a CSV file.

    The SAX Model is a function that returns a SAX SDict interpolated over wavelength.

    Args:
        filepath: CSV Sparameters path or pandas DataFrame.
        wl: wavelength to interpolate (um).
        xkey: key for wavelengths in file.
        xunits: x units in um from the loaded file (um). 1 means 1um.
        prefix: for the sparameters column names in file.
    """
    df = filepath if isinstance(filepath, pd.DataFrame) else pd.read_csv(filepath)
    assert isinstance(df, pd.DataFrame)
    df = df.reset_index()  # maybe there is useful info in the index...
    dic = dict(zip(df.columns, jnp.asarray(df.values.T)))
    keys = list(dic.keys())

    if xkey not in keys:
        raise ValueError(f"{xkey!r} not in {keys}")

    nsparameters = (len(keys) - 1) // 2
    nports = int(nsparameters**0.5)

    x = dic[xkey] * xunits

    @jax.jit
    def model(wl: Float = jnp.asarray(wl_cband)):
        S = {}
        zero = jnp.zeros_like(x)
        for i in range(1, nports + 1):
            for j in range(1, nports + 1):
                m = jnp.interp(wl, x, dic.get(f"{prefix}{i}{j}m", zero))
                a = jnp.interp(wl, x, dic.get(f"{prefix}{i}{j}a", zero))
                S[f"o{i}", f"o{j}"] = m * jnp.exp(1j * a)

        return S

    return model

def _demo_mmi_lumerical_csv() -> None:
    import matplotlib.pyplot as plt
    from plot_model import plot_model

    filepath = get_sparameters_path_lumerical(gf.c.mmi1x2)
    mmi = model_from_csv(filepath=filepath, xkey="wavelengths")
    plot_model(mmi)
    plt.show()

def sdict_from_component_lumerical(component, **kwargs) -> Model:
    """Returns SDict based on lumerical simulation."""
    filepath = get_sparameters_path_lumerical(component=component, **kwargs)
    return model_from_csv(filepath=filepath)

mmi1x2 = gf.partial(sdict_from_component_lumerical, component=gf.components.mmi1x2)
mmi2x2 = gf.partial(sdict_from_component_lumerical, component=gf.components.mmi2x2)

grating_coupler_elliptical = model_from_csv(
    filepath=sparameters_path / "grating_coupler_ellipti_9d85a0c6_18c08cac.csv"
)

model_factory = dict(
    mmi1x2=mmi1x2, mmi2x2=mmi2x2, grating_coupler_elliptical=grating_coupler_elliptical
)

if __name__ == "__main__":

    import matplotlib.pyplot as plt
    from plot_model import plot_model

    # s = from_csv(filepath=filepath, wavelength=wl_cband, xunits=1e-3)
    # s21 = s[("o1", "o3")]
    # plt.plot(wl_cband, np.abs(s21) ** 2)
    # plt.show()
    # filepath = get_sparameters_path_lumerical(gf.c.mmi1x2)
    # mmi = model_from_csv(filepath=filepath, xkey="wavelengths")
    # plot_model(mmi)
    # plt.show()

    plot_model(grating_coupler_elliptical)
    plt.show()

    # model = partial(
    #     from_csv, filepath=filepath, xunits=1, xkey="wavelengths", prefix="s"
    # )
    # sax.plot.plot_model(model)
    # plt.show()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Python libs
import sys, time

# numpy and scipy
import numpy as np
from scipy.ndimage import filters

# OpenCV
import cv2, cv

# Ros libraries
import roslib
import rospy

# Ros Messages
from sensor_msgs.msg import Image, CompressedImage
from cv_bridge import CvBridge, CvBridgeError

class image_converter:

    def __init__(self):
        '''Initialize ros publisher, ros subscriber'''
        # topic where we publish

        self.bridge = CvBridge()

        out_image = rospy.get_param(rospy.search_param('out'))
        image = rospy.get_param(rospy.search_param('image'))
        # subscribed Topic
        self.image_pub = rospy.Publisher(out_image, Image)
        self.subscriber = rospy.Subscriber(image, CompressedImage, self.callback, queue_size = 1000)

    def callback(self, ros_data):
        '''Callback function of subscribed topic.
        Here images get converted and features detected'''

        #### direct conversion to CV2 ####
        np_arr = np.fromstring(ros_data.data, np.uint8)
        image_np = cv2.imdecode(np_arr, cv2.CV_LOAD_IMAGE_COLOR)
        image_cv = cv.fromarray(image_np)

        # Publish new image
        try:
            self.image_pub.publish(self.bridge.cv_to_imgmsg(image_cv, "passthrough"))
        except CvBridgeError, e:
            print e

def main(args):
    '''Initializes and cleanup ros node'''
    rospy.init_node('image_converter', anonymous=True)
    ic = image_converter()

    try:
        rospy.spin()
    except KeyboardInterrupt:
        print "Shutting down ROS Image feature detector module"
    cv2.destroyAllWindows()

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>updating ov_converter<commit_after>#!/usr/bin/env python
# Python libs
import sys, time

# numpy and scipy
import numpy as np
from scipy.ndimage import filters

# OpenCV
import cv2, cv

# Ros libraries
import roslib
import rospy

# Ros Messages
from sensor_msgs.msg import Image, CompressedImage
from cv_bridge import CvBridge, CvBridgeError

class image_converter:

    def __init__(self):
        '''Initialize ros publisher, ros subscriber'''
        # topic where we publish

        self.bridge = CvBridge()

        out_image = rospy.get_param(rospy.search_param('out'))
        image = rospy.get_param(rospy.search_param('image'))
        # subscribed Topic
        self.image_pub = rospy.Publisher(out_image, Image)
        self.subscriber = rospy.Subscriber(image, CompressedImage, self.callback, queue_size = 1000)

    def callback(self, ros_data):
        '''Callback function of subscribed topic.
        Here images get converted and features detected'''

        #### direct conversion to CV2 ####
        np_arr = np.fromstring(ros_data.data, np.uint8)
        image_np = cv2.imdecode(np_arr, cv2.CV_LOAD_IMAGE_COLOR)
        image_cv = cv.fromarray(image_np)

        # Publish new image
        try:
            self.image_pub.publish(self.bridge.cv_to_imgmsg(image_cv, "bgr8"))
        except CvBridgeError, e:
            print e

def main(args):
    '''Initializes and cleanup ros node'''
    rospy.init_node('image_converter', anonymous=True)
    ic = image_converter()

    try:
        rospy.spin()
    except KeyboardInterrupt:
        print "Shutting down ROS Image feature detector module"
    cv2.destroyAllWindows()

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
import logging
import datetime
import random

from django.utils.translation import ugettext_lazy as _
from django.forms import models
from django import forms

from util import get_replication_topology_instance
from drivers.factory import DriverFactory
from physical.models import Plan, Environment, Engine, Pool
from logical.forms.fields import AdvancedModelChoiceField
from logical.models import Database
from logical.validators import database_name_evironment_constraint

LOG = logging.getLogger(__name__)

class DatabaseForm(models.ModelForm):

    WEEKDAYS = [
        (0, 'Sunday'),
        (1, 'Monday'),
        (2, 'Tuesday'),
        (3, 'Wednesday'),
        (4, 'Thursday'),
        (5, 'Friday'),
        (6, 'Saturday')
    ]

    BACKUP_HOUR_CHOICES = [
        (hour, datetime.time(hour, 0).strftime(format='%H:%M'))
        for hour in range(24)
    ]
    MAINTENANCE_WINDOW_CHOICES = [
        (hour,
         datetime.time(hour, 0).strftime('%H:%M - ' + str(
            (datetime.datetime.combine(
                datetime.date.today(),
                datetime.time(hour)
            ) + datetime.timedelta(hours=1)
            ).strftime('%H:%M')))) for hour in range(24)
    ]
    environment = forms.ModelChoiceField(queryset=Environment.objects)
    engine = forms.ModelChoiceField(queryset=Engine.objects)
    pool = forms.ModelChoiceField(queryset=Pool.objects, required=False)
    plan = AdvancedModelChoiceField(
        queryset=Plan.objects.filter(is_active='True'),
        required=False, widget=forms.RadioSelect,
        empty_label=None
    )
    backup_hour = forms.ChoiceField(
                    choices=BACKUP_HOUR_CHOICES,
                    help_text='This field must not be the same as the'
                    ' maintenance window.')
    maintenance_window = forms.ChoiceField(
        choices=MAINTENANCE_WINDOW_CHOICES,
        help_text='This field must not be the same as the'
        ' backup hour.')
    maintenance_day = forms.ChoiceField(
        choices=WEEKDAYS)

    class Meta:
        model = Database
        fields = [
            'name', 'description', 'project', 'environment', 'team', 'pool',
            'engine', 'subscribe_to_email_events', 'backup_hour',
            'maintenance_window', 'maintenance_day', 'is_in_quarantine',
            'plan'
        ]

    def __init__(self, *args, **kwargs):
        super(DatabaseForm, self).__init__(*args, **kwargs)
        self.fields['is_in_quarantine'].widget = forms.HiddenInput()
        backup_hour, maintenance_hour, maintenance_day = (
            self.randomize_backup_and_maintenance_hour()
        )
        self.fields['backup_hour'].initial = backup_hour
        self.fields['maintenance_window'].initial = maintenance_hour
        self.fields['maintenance_day'].initial = maintenance_day

    @staticmethod
    def randomize_backup_and_maintenance_hour():
        backup_hour = random.randint(0, 6)
        maintenance_choices = range(6)
        if backup_hour < 6:
            maintenance_choices.remove(backup_hour)
        maintenance_hour = random.choice(maintenance_choices)
        maintenance_day = random.randint(1, 7)

        return backup_hour, maintenance_hour, maintenance_day

    def _validate_description(self, cleaned_data):
        if 'description' in cleaned_data:
            if not cleaned_data.get('description', None):
                self._errors["description"] = self.error_class(
                    [_("Description: This field is required.")])

    def _validate_project(self, cleaned_data):
        if 'project' in cleaned_data:
            if not cleaned_data.get('project', None):
                self._errors["project"] = self.error_class(
                    [_("Project: This field is required.")])

    def _validate_team(self, cleaned_data):
        if 'team' in cleaned_data:
            if not cleaned_data['team']:
                LOG.warning("No team specified in database form")
                self._errors["team"] = self.error_class(
                    [_("Team: This field is required.")])

    def _validate_team_resources(self, cleaned_data):
        team = cleaned_data['team']

        if team:
            dbs = team.databases_in_use_for(cleaned_data['environment'])
            database_alocation_limit = team.database_alocation_limit
            LOG.debug("dbs: %s | type: %s" % (dbs, type(dbs)))

            if (database_alocation_limit != 0
                    and len(dbs) >= database_alocation_limit):
                LOG.warning(
                    ("The database alocation limit of {} has been exceeded "
                     "for the selected team {} => {}").format(
                        database_alocation_limit, team, list(dbs))
                )
                self._errors["team"] = self.error_class(
                    [_(("The database alocation limit of {} has been "
                        "exceeded for the selected team: {}")).format(
                            database_alocation_limit, list(dbs))]
                )

    def _validate_name(self, cleaned_data):
        if len(cleaned_data['name']) > 40:
            self._errors["name"] = self.error_class(
                [_("Database name too long")])

        plan = cleaned_data['plan']

        class_path = plan.replication_topology.class_path
        driver_name = get_replication_topology_instance(class_path).driver_name
        driver = DriverFactory.get_driver_class(driver_name)

        if cleaned_data['name'] in driver.RESERVED_DATABASES_NAME:
            raise forms.ValidationError(
                _("%s is a reserved database name" % cleaned_data['name']))

    def _validate_backup_hour(self, cleaned_data):
        backup_hour = cleaned_data['backup_hour']
        maintenance_window = cleaned_data['maintenance_window']

        if backup_hour == maintenance_window:
            raise forms.ValidationError(
                "Backup hour must not be equal to maintenance window."
            )

    def _validate_pool(self, cleaned_data):
        env = cleaned_data['environment']
        pool = cleaned_data['pool']
        if env.pools.exists() and pool is None:
            self._errors["pool"] = self.error_class(
                [_("Team: This field is required.")])

    def clean(self):
        cleaned_data = super(DatabaseForm, self).clean()

        if not self.is_valid():
            raise forms.ValidationError(self.errors)

        if 'plan' in cleaned_data:
            plan = cleaned_data.get('plan', None)
            if not plan:
                self._errors["plan"] = self.error_class(
                    [_("Plan: This field is required.")])

        self._validate_name(cleaned_data)
        self._validate_project(cleaned_data)
        self._validate_description(cleaned_data)
        self._validate_team(cleaned_data)
        self._validate_backup_hour(cleaned_data)
        self._validate_pool(cleaned_data)

        if 'environment' in cleaned_data:
            environment = cleaned_data.get('environment', None)
            database_name = cleaned_data.get('name', None)
            if not environment or environment not in plan.environments.all():
                raise forms.ValidationError(
                    _("Invalid plan for selected environment."))

            if Database.objects.filter(
                    name=database_name, environment__name=environment
            ):
                self._errors["name"] = self.error_class(
                    [_(("this name already exists in the selected "
                        "environment"))])
                del cleaned_data["name"]

        self._validate_team_resources(cleaned_data)
        if database_name_evironment_constraint(
                database_name, environment.name):
            raise forms.ValidationError(
                _('%s already exists in production!') % database_name
            )

        if self._errors:
            return cleaned_data

        return cleaned_data

class LogDatabaseForm(forms.Form):
    database_id = forms.CharField(widget=forms.HiddenInput())

    def __init__(self, *args, **kwargs):

        super(LogDatabaseForm, self).__init__(*args, **kwargs)
        if 'initial' in kwargs:
            instance = Database.objects.get(
                id=kwargs['initial']['database_id'])

            if instance:
                LOG.debug("instance database form found! %s" % instance)

class DatabaseDetailsForm(forms.ModelForm):

    class Meta:
        model = Database
        fields = [
            'team', 'project', 'is_protected', 'subscribe_to_email_events',
            'description'
        ]

    def __init__(self, *args, **kwargs):
        super(DatabaseDetailsForm, self).__init__(*args, **kwargs)

        self.fields['team'].required = True
        # self.fields['project'].required = True
        self.fields['description'].required = True
<commit_msg>Fix randonint to do between 0 and 6<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
import logging
import datetime
import random

from django.utils.translation import ugettext_lazy as _
from django.forms import models
from django import forms

from util import get_replication_topology_instance
from drivers.factory import DriverFactory
from physical.models import Plan, Environment, Engine, Pool
from logical.forms.fields import AdvancedModelChoiceField
from logical.models import Database
from logical.validators import database_name_evironment_constraint

LOG = logging.getLogger(__name__)

class DatabaseForm(models.ModelForm):

    WEEKDAYS = [
        (0, 'Sunday'),
        (1, 'Monday'),
        (2, 'Tuesday'),
        (3, 'Wednesday'),
        (4, 'Thursday'),
        (5, 'Friday'),
        (6, 'Saturday')
    ]

    BACKUP_HOUR_CHOICES = [
        (hour, datetime.time(hour, 0).strftime(format='%H:%M'))
        for hour in range(24)
    ]
    MAINTENANCE_WINDOW_CHOICES = [
        (hour,
         datetime.time(hour, 0).strftime('%H:%M - ' + str(
            (datetime.datetime.combine(
                datetime.date.today(),
                datetime.time(hour)
            ) + datetime.timedelta(hours=1)
            ).strftime('%H:%M')))) for hour in range(24)
    ]
    environment = forms.ModelChoiceField(queryset=Environment.objects)
    engine = forms.ModelChoiceField(queryset=Engine.objects)
    pool = forms.ModelChoiceField(queryset=Pool.objects, required=False)
    plan = AdvancedModelChoiceField(
        queryset=Plan.objects.filter(is_active='True'),
        required=False, widget=forms.RadioSelect,
        empty_label=None
    )
    backup_hour = forms.ChoiceField(
                    choices=BACKUP_HOUR_CHOICES,
                    help_text='This field must not be the same as the'
                    ' maintenance window.')
    maintenance_window = forms.ChoiceField(
        choices=MAINTENANCE_WINDOW_CHOICES,
        help_text='This field must not be the same as the'
        ' backup hour.')
    maintenance_day = forms.ChoiceField(
        choices=WEEKDAYS)

    class Meta:
        model = Database
        fields = [
            'name', 'description', 'project', 'environment', 'team', 'pool',
            'engine', 'subscribe_to_email_events', 'backup_hour',
            'maintenance_window', 'maintenance_day', 'is_in_quarantine',
            'plan'
        ]

    def __init__(self, *args, **kwargs):
        super(DatabaseForm, self).__init__(*args, **kwargs)
        self.fields['is_in_quarantine'].widget = forms.HiddenInput()
        backup_hour, maintenance_hour, maintenance_day = (
            self.randomize_backup_and_maintenance_hour()
        )
        self.fields['backup_hour'].initial = backup_hour
        self.fields['maintenance_window'].initial = maintenance_hour
        self.fields['maintenance_day'].initial = maintenance_day

    @staticmethod
    def randomize_backup_and_maintenance_hour():
        backup_hour = random.randint(0, 6)
        maintenance_choices = range(6)
        if backup_hour < 6:
            maintenance_choices.remove(backup_hour)
        maintenance_hour = random.choice(maintenance_choices)
        maintenance_day = random.randint(0, 6)

        return backup_hour, maintenance_hour, maintenance_day

    def _validate_description(self, cleaned_data):
        if 'description' in cleaned_data:
            if not cleaned_data.get('description', None):
                self._errors["description"] = self.error_class(
                    [_("Description: This field is required.")])

    def _validate_project(self, cleaned_data):
        if 'project' in cleaned_data:
            if not cleaned_data.get('project', None):
                self._errors["project"] = self.error_class(
                    [_("Project: This field is required.")])

    def _validate_team(self, cleaned_data):
        if 'team' in cleaned_data:
            if not cleaned_data['team']:
                LOG.warning("No team specified in database form")
                self._errors["team"] = self.error_class(
                    [_("Team: This field is required.")])

    def _validate_team_resources(self, cleaned_data):
        team = cleaned_data['team']

        if team:
            dbs = team.databases_in_use_for(cleaned_data['environment'])
            database_alocation_limit = team.database_alocation_limit
            LOG.debug("dbs: %s | type: %s" % (dbs, type(dbs)))

            if (database_alocation_limit != 0
                    and len(dbs) >= database_alocation_limit):
                LOG.warning(
                    ("The database alocation limit of {} has been exceeded "
                     "for the selected team {} => {}").format(
                        database_alocation_limit, team, list(dbs))
                )
                self._errors["team"] = self.error_class(
                    [_(("The database alocation limit of {} has been "
                        "exceeded for the selected team: {}")).format(
                            database_alocation_limit, list(dbs))]
                )

    def _validate_name(self, cleaned_data):
        if len(cleaned_data['name']) > 40:
            self._errors["name"] = self.error_class(
                [_("Database name too long")])

        plan = cleaned_data['plan']

        class_path = plan.replication_topology.class_path
        driver_name = get_replication_topology_instance(class_path).driver_name
        driver = DriverFactory.get_driver_class(driver_name)

        if cleaned_data['name'] in driver.RESERVED_DATABASES_NAME:
            raise forms.ValidationError(
                _("%s is a reserved database name" % cleaned_data['name']))

    def _validate_backup_hour(self, cleaned_data):
        backup_hour = cleaned_data['backup_hour']
        maintenance_window = cleaned_data['maintenance_window']

        if backup_hour == maintenance_window:
            raise forms.ValidationError(
                "Backup hour must not be equal to maintenance window."
            )

    def _validate_pool(self, cleaned_data):
        env = cleaned_data['environment']
        pool = cleaned_data['pool']
        if env.pools.exists() and pool is None:
            self._errors["pool"] = self.error_class(
                [_("Team: This field is required.")])

    def clean(self):
        cleaned_data = super(DatabaseForm, self).clean()

        if not self.is_valid():
            raise forms.ValidationError(self.errors)

        if 'plan' in cleaned_data:
            plan = cleaned_data.get('plan', None)
            if not plan:
                self._errors["plan"] = self.error_class(
                    [_("Plan: This field is required.")])

        self._validate_name(cleaned_data)
        self._validate_project(cleaned_data)
        self._validate_description(cleaned_data)
        self._validate_team(cleaned_data)
        self._validate_backup_hour(cleaned_data)
        self._validate_pool(cleaned_data)

        if 'environment' in cleaned_data:
            environment = cleaned_data.get('environment', None)
            database_name = cleaned_data.get('name', None)
            if not environment or environment not in plan.environments.all():
                raise forms.ValidationError(
                    _("Invalid plan for selected environment."))

            if Database.objects.filter(
                    name=database_name, environment__name=environment
            ):
                self._errors["name"] = self.error_class(
                    [_(("this name already exists in the selected "
                        "environment"))])
                del cleaned_data["name"]

        self._validate_team_resources(cleaned_data)
        if database_name_evironment_constraint(
                database_name, environment.name):
            raise forms.ValidationError(
                _('%s already exists in production!') % database_name
            )

        if self._errors:
            return cleaned_data

        return cleaned_data

class LogDatabaseForm(forms.Form):
    database_id = forms.CharField(widget=forms.HiddenInput())

    def __init__(self, *args, **kwargs):

        super(LogDatabaseForm, self).__init__(*args, **kwargs)
        if 'initial' in kwargs:
            instance = Database.objects.get(
                id=kwargs['initial']['database_id'])

            if instance:
                LOG.debug("instance database form found! %s" % instance)

class DatabaseDetailsForm(forms.ModelForm):

    class Meta:
        model = Database
        fields = [
            'team', 'project', 'is_protected', 'subscribe_to_email_events',
            'description'
        ]

    def __init__(self, *args, **kwargs):
        super(DatabaseDetailsForm, self).__init__(*args, **kwargs)

        self.fields['team'].required = True
        # self.fields['project'].required = True
        self.fields['description'].required = True
<|endoftext|>


<commit_before>"""
Custom datasources related to ``corosync``
"""
import os
from insights.core.context import HostContext
from insights.core.dr import SkipComponent
from insights.core.plugins import datasource
from insights.components.rhel_version import IsRhel7, IsRhel8, IsRhel9

@datasource(HostContext, [IsRhel7, IsRhel8, IsRhel9])
def corosync_cmapctl_cmds(broker):
    """
    corosync-cmapctl use different arguments on RHEL7 and RHEL8.

    Returns:
        list: A list of related corosync-cmapctl commands based the RHEL version.
    """
    corosync_cmd = '/usr/sbin/corosync-cmapctl'
    if os.path.exists(corosync_cmd):
        print(broker, broker.get(IsRhel7))
        # RHEL 7
        if broker.get(IsRhel7):
            return [
                corosync_cmd,
                ' '.join([corosync_cmd, '-d runtime.schedmiss.timestamp']),
                ' '.join([corosync_cmd, '-d runtime.schedmiss.delay'])]
        # Others
        elif broker.get(IsRhel8) or broker.get(IsRhel9):
            return [
                corosync_cmd,
                ' '.join([corosync_cmd, '-m stats']),
                ' '.join([corosync_cmd, '-C schedmiss'])]

    raise SkipComponent()
<commit_msg>fix: bz-2130242, remove the print statement (#3535)<commit_after>"""
Custom datasources related to ``corosync``
"""
import os
from insights.core.context import HostContext
from insights.core.dr import SkipComponent
from insights.core.plugins import datasource
from insights.components.rhel_version import IsRhel7, IsRhel8, IsRhel9

@datasource(HostContext, [IsRhel7, IsRhel8, IsRhel9])
def corosync_cmapctl_cmds(broker):
    """
    corosync-cmapctl use different arguments on RHEL7 and RHEL8.

    Returns:
        list: A list of related corosync-cmapctl commands based the RHEL version.
    """
    corosync_cmd = '/usr/sbin/corosync-cmapctl'
    if os.path.exists(corosync_cmd):
        # RHEL 7
        if broker.get(IsRhel7):
            return [
                corosync_cmd,
                ' '.join([corosync_cmd, '-d runtime.schedmiss.timestamp']),
                ' '.join([corosync_cmd, '-d runtime.schedmiss.delay'])]
        # Others
        elif broker.get(IsRhel8) or broker.get(IsRhel9):
            return [
                corosync_cmd,
                ' '.join([corosync_cmd, '-m stats']),
                ' '.join([corosync_cmd, '-C schedmiss'])]

    raise SkipComponent()
<|endoftext|>


<commit_before>import time
import pickle
import os

class Project:
	'''this class is gonna be implemented later'''
	def __init__(self, name):
		self.name = name
	def add_tasks(self, tasks):
		self.tasks = tasks.get_list()
	def get_tasks(self):
		return self.tasks

		#TODO : Add reminders(time alarm, notifactions .....etc)
class Task:
	def __init__(self, caption, desc):
		self.caption = caption
		self.desc = desc

	def __str__(self):
		return "\"{}\"  :\n\t{}".format(self.caption, self.desc)

#TODO: add multi todolists
class ToDoList:
	'''this class is for the to do list tasks,
	 it contains a list (lst) for the new added tasks,
	 and another list for the checked tasks (oldlst),
	 '''
	def __init__(self, tasks_caption):
		self.tasks_caption = tasks_caption
		self.lst = []
		self.oldlst = []
		#self.display_func = lambda lst: "".join([str(i) + ' : ' + lst[i] + '\n' + ('-' * 50) + '\n' for i in range(len(lst))])
	def add(self, item):
		self.lst.append(item)

	#TODO: Add database support
	def save(self, f):
		#change this to somthing else.
		pickle.dump(self, f)

	def get_list(self):
		return self.lst

	def check(self, choice):
		self.oldlst.append( self.lst[choice]+ '      ' +time.ctime() )
		del self.lst[choice]

	def get_archive(self):
		return self.oldlst

	def display_current_list(self):
		current_list_str = 'The current list: \n\n'  + display_list(self.get_list()) + '\n\n'
		return current_list_str

	def display_checked_list(self):
		checked_list_str = 'The checked list: \') \n' + display_list(self.get_archive()) + '\n\n'
		return checked_list_str

def display_list(lst):
        return "".join([str(i) + ' : ' + lst[i] + '\n' + ('-' * 50) + '\n' \
                        for i in range(len(lst))])

def load(f):
	'''this function takes a file object and return a ToDoList object'''
	c = pickle.load(f)
	return c

def display_main_menu():
	main_menu_str = "-"*15 + "Hello in pyTDL. Wish for you a good luck! :)" + "-"*15 +'''\n
	The main menu :  \n
	\t1:Add item to the current list.\n
	\t2:Show the current list.\n
	\t3:Show checked list.\n
	\t4:Save list.\n
	\t0:Quit.\n
	'''
	return main_menu_str

######### Main program #########

#TODO: make the code cleaner
#TODO: Add tests
def main():
	'''this function is just for testing the functionality of ToDoList class'''
	if os.path.exists("lst.tdl"):
		with open("lst.tdl", 'rb') as f:
			tasks = pickle.load(f)
	else:
		tasks = ToDoList('Test tasks')
	while True:
		os.system('clear || cls')
		print(display_main_menu())
		choice = input('Enter your choice :')
		if choice == '1':
			title = input('Enter the task\'s title : ')
			desc = input('Enter the tasks\'s description: ')
			item = Task(title, desc)
			tasks.add(str(item))
		elif choice == '2':
			os.system('clear || cls')
			print(tasks.display_current_list())
			current_list_choice = input('c : Check\t q: Quit : ').lower()
			if current_list_choice == 'c':
				checked = int(input("enter the task number: "))
				tasks.check(checked)
			elif current_list_choice == 'q':
				continue
		elif choice == '3':
			os.system('clear || cls')
			print(tasks.display_checked_list())
			checked_list_choice = input('Quit ? (y/n) : ')
			if checked_list_choice == 'y':
				continue
		elif choice == '4':
			os.system('clear || cls')
			with open("lst.tdl", 'wb') as f:
				tasks.save(f)
			continue
		elif choice == '0':
			os.system('clear || cls')
			break

if __name__ == '__main__':
	main()
<commit_msg>change 'checked' to 'completed'<commit_after>import time
import pickle
import os

class Project:
	'''this class is gonna be implemented later'''
	def __init__(self, name):
		self.name = name
	def add_tasks(self, tasks):
		self.tasks = tasks.get_list()
	def get_tasks(self):
		return self.tasks

		#TODO : Add reminders(time alarm, notifactions .....etc)
class Task:
	def __init__(self, caption, desc):
		self.caption = caption
		self.desc = desc

	def __str__(self):
		return "\"{}\"  :\n\t{}".format(self.caption, self.desc)

#TODO: add multi todolists
class ToDoList:
	'''this class is for the to do list tasks,
	 it contains a list (lst) for the new added tasks,
	 and another list for the completed tasks (oldlst),
	 '''
	def __init__(self, tasks_caption):
		self.tasks_caption = tasks_caption
		self.lst = []
		self.oldlst = []
		#self.display_func = lambda lst: "".join([str(i) + ' : ' + lst[i] + '\n' + ('-' * 50) + '\n' for i in range(len(lst))])
	def add(self, item):
		self.lst.append(item)

	#TODO: Add database support
	def save(self, f):
		#change this to somthing else.
		pickle.dump(self, f)

	def get_list(self):
		return self.lst

	def check(self, choice):
		self.oldlst.append( self.lst[choice]+ '      ' +time.ctime() )
		del self.lst[choice]

	def get_archive(self):
		return self.oldlst

	def display_current_list(self):
		current_list_str = 'The current list: \n\n'  + display_list(self.get_list()) + '\n\n'
		return current_list_str

	def display_completed_list(self):
		completed_list_str = 'The completed list: \') \n' + display_list(self.get_archive()) + '\n\n'
		return completed_list_str

def display_list(lst):
        return "".join([str(i) + ' : ' + lst[i] + '\n' + ('-' * 50) + '\n' \
                        for i in range(len(lst))])

def load(f):
	'''this function takes a file object and return a ToDoList object'''
	c = pickle.load(f)
	return c

def display_main_menu():
	main_menu_str = "-"*15 + "Hello in pyTDL. Wish for you a good luck! :)" + "-"*15 +'''\n
	The main menu :  \n
	\t1:Add item to the current list.\n
	\t2:Show the current list.\n
	\t3:Show completed list.\n
	\t4:Save list.\n
	\t0:Quit.\n
	'''
	return main_menu_str

######### Main program #########

#TODO: make the code cleaner
#TODO: Add tests
def main():
	'''this function is just for testing the functionality of ToDoList class'''
	if os.path.exists("lst.tdl"):
		with open("lst.tdl", 'rb') as f:
			tasks = pickle.load(f)
	else:
		tasks = ToDoList('Test tasks')
	while True:
		os.system('clear || cls')
		print(display_main_menu())
		choice = input('Enter your choice :')
		if choice == '1':
			title = input('Enter the task\'s title : ')
			desc = input('Enter the tasks\'s description: ')
			item = Task(title, desc)
			tasks.add(str(item))
		elif choice == '2':
			os.system('clear || cls')
			print(tasks.display_current_list())
			current_list_choice = input('c : Check\t q: Quit : ').lower()
			if current_list_choice == 'c':
				completed = int(input("enter the task number: "))
				tasks.check(completed)
			elif current_list_choice == 'q':
				continue
		elif choice == '3':
			os.system('clear || cls')
			print(tasks.display_completed_list())
			completed_list_choice = input('Quit ? (y/n) : ')
			if completed_list_choice == 'y':
				continue
		elif choice == '4':
			os.system('clear || cls')
			with open("lst.tdl", 'wb') as f:
				tasks.save(f)
			continue
		elif choice == '0':
			os.system('clear || cls')
			break

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>import tempfile
import insights.client.schedule as sched

def test_set_daily():
    target = tempfile.mktemp()
    with tempfile.NamedTemporaryFile() as source:
        schedule = sched.get_scheduler(source.name, target)
        assert not schedule.active
        assert schedule.set_daily()
        assert schedule.active
        schedule.remove_scheduling()
        assert not schedule.active

def test_failed_removal():
    """
    Just verifying that trying to remove scheduling does not raise an exception
    """
    fname = tempfile.mktemp()
    schedule = sched.get_scheduler(fname)
    schedule.remove_scheduling()
<commit_msg>make the temporary scheduler files (#938)<commit_after>import tempfile
import insights.client.schedule as sched

def test_set_daily():
    target = tempfile.mktemp()
    with tempfile.NamedTemporaryFile() as source:
        schedule = sched.get_scheduler(source.name, target)
        assert not schedule.active
        assert schedule.set_daily()
        assert schedule.active
        schedule.remove_scheduling()
        assert not schedule.active

def test_failed_removal():
    """
    Just verifying that trying to remove scheduling does not raise an exception
    """
    target = tempfile.mktemp()
    with tempfile.NamedTemporaryFile() as source:
        schedule = sched.get_scheduler(source.name, target)
        schedule.remove_scheduling()
<|endoftext|>


<commit_before>from __future__ import print_function

import os
import sys, logging
import json
import re
import mechanize
import boto3

mechlog = logging.getLogger("mechanize")
mechlog.addHandler(logging.StreamHandler(sys.stdout))

if os.getenv('DEBUG') != None:
    logging.basicConfig(level=logging.DEBUG)
    mechlog.setLevel(logging.DEBUG)

confirm_url = re.compile("https://certificates.amazon.com/approvals\?[A-Za-z0-9=&-]+")
approval_text = re.compile("You have approved")

domain_re = re.compile(".*Domain name.*?\s+(.*?)\s.*", re.DOTALL)
accountid_re = re.compile(".*AWS account number.*?\s+(.*?)\s.*", re.DOTALL)
region_re = re.compile(".*AWS Region.*?\s+(.*?)\s.*", re.DOTALL)
certid_re = re.compile(".*Certificate identifier.*?\s+(.*?)\s.*", re.DOTALL)

acm = boto3.client('acm')

def panic(msg):
    raise Exception(msg)

def validate(event, context):
    msg = json.loads(event['Records'][0]['Sns']['Message'])
    match = confirm_url.search(msg['content'])
    if match:
        url = match.group(0)
        logging.info("CONFIRMATION URL: %s" % url)

        br = mechanize.Browser()
        br.set_handle_robots(False)

        # Fetch approval page
        logging.debug("OPENING CONFIRMATION URL")
        response = br.open(url)
        logging.debug("OPENED CONFIRMATION URL")
        content = response.get_data()

        # Extract confirmation page details
        domain, account_id, region, cert_id = [regex.match(content).group(1)
            if regex.match(content) else panic("Couldn't parse confirmation page!")
            for regex in (domain_re, accountid_re, region_re, certid_re)]

        # Remove dashes from account_id
        account_id = account_id.translate(None, '-')

        # Always log what we're confirming
        print("Domain: '%s'" % domain)
        print("Account ID: '%s'" % account_id)
        print("Region: '%s'" % region)
        print("Certificate ID: '%s'" % cert_id)

        # Check if the cert is pending validation
        cert = acm.describe_certificate(CertificateArn="arn:aws:acm:%s:%s:certificate/%s"
            % (region, account_id, cert_id))
        logging.debug(cert)

        if cert['Certificate']['DomainName'] != domain:
            panic("Couldn't match confirmation to certificate domain!")

        if cert['Certificate']['Status'] != 'PENDING_VALIDATION':
            panic("Confirmation certificate is not pending validation!")

        # It's the first and only form on the page
        # Could we match on action="/approvals"?
        br.select_form(nr=0)
        logging.info("SUBMITTING CONFIRMATION FORM")
        response = br.submit(name='commit')
        logging.info("SUBMITTED CONFIRMATION FORM")
        content = response.get_data()

        match = approval_text.search(content)
        if match:
            print("Certificate for %s approved!" % domain)
        else:
            logging.error(content)
            panic("No confirmation of certificate approval!")

    else:
        logging.error(msg['content'])
        panic("Email did not match certificate approval URL!")
<commit_msg>Log validation URL just in case<commit_after>from __future__ import print_function

import os
import sys, logging
import json
import re
import mechanize
import boto3

mechlog = logging.getLogger("mechanize")
mechlog.addHandler(logging.StreamHandler(sys.stdout))

if os.getenv('DEBUG') != None:
    logging.basicConfig(level=logging.DEBUG)
    mechlog.setLevel(logging.DEBUG)

confirm_url = re.compile("https://certificates.amazon.com/approvals\?[A-Za-z0-9=&-]+")
approval_text = re.compile("You have approved")

domain_re = re.compile(".*Domain name.*?\s+(.*?)\s.*", re.DOTALL)
accountid_re = re.compile(".*AWS account number.*?\s+(.*?)\s.*", re.DOTALL)
region_re = re.compile(".*AWS Region.*?\s+(.*?)\s.*", re.DOTALL)
certid_re = re.compile(".*Certificate identifier.*?\s+(.*?)\s.*", re.DOTALL)

acm = boto3.client('acm')

def panic(msg):
    raise Exception(msg)

def validate(event, context):
    msg = json.loads(event['Records'][0]['Sns']['Message'])
    match = confirm_url.search(msg['content'])
    if match:
        url = match.group(0)
        logging.info("CONFIRMATION URL: %s" % url)

        br = mechanize.Browser()
        br.set_handle_robots(False)

        # Fetch approval page
        logging.debug("OPENING CONFIRMATION URL")
        response = br.open(url)
        logging.debug("OPENED CONFIRMATION URL")
        content = response.get_data()

        # Extract confirmation page details
        domain, account_id, region, cert_id = [regex.match(content).group(1)
            if regex.match(content) else panic("Couldn't parse confirmation page!")
            for regex in (domain_re, accountid_re, region_re, certid_re)]

        # Remove dashes from account_id
        account_id = account_id.translate(None, '-')

        # Always log what we're confirming
        print("Validation URL: '%s'" % url)
        print("Domain: '%s'" % domain)
        print("Account ID: '%s'" % account_id)
        print("Region: '%s'" % region)
        print("Certificate ID: '%s'" % cert_id)

        # Check if the cert is pending validation
        cert = acm.describe_certificate(CertificateArn="arn:aws:acm:%s:%s:certificate/%s"
            % (region, account_id, cert_id))
        logging.debug(cert)

        if cert['Certificate']['DomainName'] != domain:
            panic("Couldn't match confirmation to certificate domain!")

        if cert['Certificate']['Status'] != 'PENDING_VALIDATION':
            panic("Confirmation certificate is not pending validation!")

        # It's the first and only form on the page
        # Could we match on action="/approvals"?
        br.select_form(nr=0)
        logging.info("SUBMITTING CONFIRMATION FORM")
        response = br.submit(name='commit')
        logging.info("SUBMITTED CONFIRMATION FORM")
        content = response.get_data()

        match = approval_text.search(content)
        if match:
            print("Certificate for %s approved!" % domain)
        else:
            logging.error(content)
            panic("No confirmation of certificate approval!")

    else:
        logging.error(msg['content'])
        panic("Email did not match certificate approval URL!")
<|endoftext|>