<commit_before>from django.core.urlresolvers import reverse
from django.utils.safestring import mark_safe
from django.utils.text import capfirst
from django.utils.translation import ugettext_lazy as _
from admin_tools.utils import AppListElementMixin

class MenuItem(object):
    """
    This is the base class for custom menu items.
    A menu item can have the following properties:

    ``title``
        String that contains the menu item title, make sure you use the
        django gettext functions if your application is multilingual.
        Default value: 'Untitled menu item'.

    ``url``
        String that contains the menu item URL.
        Default value: '#'.

    ``css_classes``
        A list of css classes to be added to the menu item ``li`` class
        attribute. Default value: [].

    ``accesskey``
        The menu item accesskey. Default value: None.

    ``description``
        An optional string that will be used as the ``title`` attribute of
        the menu-item ``a`` tag. Default value: None.

    ``enabled``
        Boolean that determines whether the menu item is enabled or not.
        Disabled items are displayed but are not clickable.
        Default value: True.

    ``template``
        The template to use to render the menu item.
        Default value: 'admin_tools/menu/item.html'.

    ``children``
        A list of children menu items. All children items must be instances of
        the ``MenuItem`` class.
    """

    title = 'Untitled menu item'
    url = '#'
    css_classes = None
    accesskey = None
    description = None
    enabled = True
    template = 'admin_tools/menu/item.html'
    children = None

    def __init__(self, title=None, url=None, **kwargs):

        if title is not None:
            self.title = title

        if url is not None:
            self.url = url

        for key in kwargs:
            if hasattr(self.__class__, key):
                setattr(self, key, kwargs[key])
        self.children = self.children or []
        self.css_classes = self.css_classes or []

    def init_with_context(self, context):
        """
        Like for menus, menu items have a ``init_with_context`` method that is
        called with a ``django.template.RequestContext`` instance as unique
        argument.
        This gives you enough flexibility to build complex items, for example,
        let's build a "history" menu item, that will list the last ten visited
        pages::

            from admin_tools.menu.items import MenuItem

            class HistoryMenuItem(MenuItem):
                title = 'History'

                def init_with_context(self, context):
                    request = context['request']
                    # we use sessions to store the visited pages stack
                    history = request.session.get('history', [])
                    for item in history:
                        self.children.append(MenuItem(
                            title=item['title'],
                            url=item['url']
                        ))
                    # add the current page to the history
                    history.insert(0, {
                        'title': context['title'],
                        'url': request.META['PATH_INFO']
                    })
                    if len(history) > 10:
                        history = history[:10]
                    request.session['history'] = history

        Here's a screenshot of our history item:

        .. image:: images/history_menu_item.png
        """
        pass

    def is_selected(self, request):
        """
        Helper method that returns ``True`` if the menu item is active.
        A menu item is considered as active if it's URL or one of its
        descendants URL is equals to the current URL.
        """
        current_url = request.get_full_path()
        return self.url == current_url or \
            len([c for c in self.children if c.is_selected(request)]) > 0

    def is_empty(self):
        """
        Helper method that returns ``True`` if the menu item is empty.
        This method always returns ``False`` for basic items, but can return
        ``True`` if the item is an AppList.
        """
        return False

class AppList(MenuItem, AppListElementMixin):
    """
    A menu item that lists installed apps an their models.
    In addition to the parent :class:`~admin_tools.menu.items.MenuItem`
    properties, the ``AppList`` has two extra properties:

    ``exclude_list``
        A list of apps to exclude, if an app name (e.g. "django.contrib.auth"
        starts with an element of this list (e.g. "django.contrib") it won't
        appear in the menu item.

    ``include_list``
        A list of apps to include, only apps whose name (e.g.
        "django.contrib.auth") starts with one of the strings (e.g.
        "django.contrib") in the list will appear in the menu item.

    If no include/exclude list is provided, **all apps** are shown.

    Here's a small example of building an app list menu item::

        from admin_tools.menu import items, Menu

        class MyMenu(Menu):
            def __init__(self, **kwargs):
                super(MyMenu, self).__init__(**kwargs)
                self.children.append(items.AppList(
                    title='Applications',
                    exclude_list=('django.contrib',)
                )

    The screenshot of what this code produces:

    .. image:: images/applist_menu_item.png

    .. note::

        Note that this menu takes into account user permissions, as a
        consequence, if a user has no rights to change or add a ``Group`` for
        example, the ``django.contrib.auth.Group model`` child item won't be
        displayed in the menu.
    """

    def __init__(self, title=None, **kwargs):
        """
        ``AppListMenuItem`` constructor.
        """
        self.include_list = kwargs.pop('include_list', [])
        self.exclude_list = kwargs.pop('exclude_list', [])
        self.models = list(kwargs.pop('models', []))
        self.exclude = list(kwargs.pop('exclude', []))
        super(AppList, self).__init__(title, **kwargs)

    def init_with_context(self, context):
        """
        Please refer to the :meth:`~admin_tools.menu.items.MenuItem.init_with_context`
        documentation from :class:`~admin_tools.menu.items.MenuItem` class.
        """
        items = self._visible_models(context['request'])
        apps = {}
        for model, perms in items:
            if not perms['change']:
                continue
            app_label = model._meta.app_label
            if app_label not in apps:
                apps[app_label] = {
                    'title': capfirst(app_label.title()),
                    'url': reverse('admin:app_list', args=(app_label,)),
                    'models': []
                }
            apps[app_label]['models'].append({
                'title': capfirst(model._meta.verbose_name_plural),
                'url': self._get_admin_change_url(model)
            })

        apps_sorted = apps.keys()
        apps_sorted.sort()
        for app in apps_sorted:
            app_dict = apps[app]
            item = MenuItem(title=app_dict['title'], url=app_dict['url'])
            # sort model list alphabetically
            apps[app]['models'].sort(lambda x, y: cmp(x['title'], y['title']))
            for model_dict in apps[app]['models']:
                item.children.append(MenuItem(**model_dict))
            self.children.append(item)

    def is_empty(self):
        """
        Helper method that returns ``True`` if the applist menu item has no
        children.

        >>> from admin_tools.menu.items import MenuItem, AppList
        >>> item = AppList(title='My menu item')
        >>> item.is_empty()
        True
        >>> item.children.append(MenuItem(title='foo'))
        >>> item.is_empty()
        False
        >>> item.children = []
        >>> item.is_empty()
        True
        """
        return len(self.children) == 0

class ModelList(MenuItem, AppListElementMixin):
    """
    A menu item that lists a set of models.
    In addition to the parent :class:`~admin_tools.menu.items.MenuItem`
    properties, the ``ModelList`` has two extra properties:

    ``include_list``
        A list of models to include, only models whose name (e.g.
        "blog.comments.Comment") starts with one of the strings (e.g. "blog")
        in the include list will appear in the menu.

    ``exclude_list``
        A list of models to exclude, if a model name (e.g.
        "blog.comments.Comment" starts with an element of this list (e.g.
        "blog.comments") it won't appear in the menu.

    If no include/exclude list is provided, **all models** are shown.

    Here's a small example of building a model list menu item::

        from admin_tools.menu import items, Menu

        class MyMenu(Menu):
            def __init__(self, **kwargs):
                super(MyMenu, self).__init__(**kwargs)
                self.children.append(items.ModelList(
                    title='Applications',
                    exclude_list=('django.contrib',)
                )

    .. note::

        Note that this menu takes into account user permissions, as a
        consequence, if a user has no rights to change or add a ``Group`` for
        example, the ``django.contrib.auth.Group model`` item won't be
        displayed in the menu.
    """

    def __init__(self, **kwargs):
        """
        ``ModelList`` constructor.
        """
        super(ModelList, self).__init__(**kwargs)
        self.include_list = kwargs.get('include_list', [])
        self.exclude_list = kwargs.get('exclude_list', [])
        self.models = list(kwargs.get('models', []))
        self.exclude = list(kwargs.get('exclude', []))

    def init_with_context(self, context):
        """
        Please refer to the :meth:`~admin_tools.menu.items.MenuItem.init_with_context`
        documentation from :class:`~admin_tools.menu.items.MenuItem` class.
        """
        items = self._visible_models(context['request'])
        for model, perms in items:
            if not perms['change']:
                continue
            title = capfirst(model._meta.verbose_name_plural)
            url = self._get_admin_change_url(model)
            item = MenuItem(title=title, url=url)
            self.children.append(item)

    def is_empty(self):
        """
        Helper method that returns ``True`` if the modellist menu item has no
        children.

        >>> from admin_tools.menu.items import MenuItem, ModelList
        >>> item = ModelList(title='My menu item')
        >>> item.is_empty()
        True
        >>> item.children.append(MenuItem(title='foo'))
        >>> item.is_empty()
        False
        >>> item.children = []
        >>> item.is_empty()
        True
        """
        return len(self.children) == 0

class Bookmarks(MenuItem, AppListElementMixin):
    """
    A menu item that lists pages bookmarked by the user. This menu item also
    adds an extra button to the menu that allows the user to bookmark or
    un-bookmark the current page.

    Here's a small example of adding a bookmark menu item::

        from admin_tools.menu import items, Menu

        class MyMenu(Menu):
            def __init__(self, **kwargs):
                super(MyMenu, self).__init__(**kwargs)
                self.children.append(items.Bookmarks('My bookmarks'))

    """
    title = _('Bookmarks')

    def __init__(self, title=None, **kwargs):
        super(Bookmarks, self).__init__(title, **kwargs)
        if 'bookmark' not in self.css_classes:
            self.css_classes.append('bookmark')

    def init_with_context(self, context):
        """
        Please refer to the :meth:`~admin_tools.menu.items.MenuItem.init_with_context`
        documentation from :class:`~admin_tools.menu.items.MenuItem` class.
        """
        from admin_tools.menu.models import Bookmark

        for b in Bookmark.objects.filter(user=context['request'].user):
            self.children.append(MenuItem(mark_safe(b.title), b.url))

        if not len(self.children):
            self.enabled = False

    def is_selected(self, request):
        """
        A bookmark menu item is never considered as active, the real item is.
        """
        return False

<commit_msg>items.ModelList signature now is the same as modules.ModelList<commit_after>from django.core.urlresolvers import reverse
from django.utils.safestring import mark_safe
from django.utils.text import capfirst
from django.utils.translation import ugettext_lazy as _
from admin_tools.utils import AppListElementMixin

class MenuItem(object):
    """
    This is the base class for custom menu items.
    A menu item can have the following properties:

    ``title``
        String that contains the menu item title, make sure you use the
        django gettext functions if your application is multilingual.
        Default value: 'Untitled menu item'.

    ``url``
        String that contains the menu item URL.
        Default value: '#'.

    ``css_classes``
        A list of css classes to be added to the menu item ``li`` class
        attribute. Default value: [].

    ``accesskey``
        The menu item accesskey. Default value: None.

    ``description``
        An optional string that will be used as the ``title`` attribute of
        the menu-item ``a`` tag. Default value: None.

    ``enabled``
        Boolean that determines whether the menu item is enabled or not.
        Disabled items are displayed but are not clickable.
        Default value: True.

    ``template``
        The template to use to render the menu item.
        Default value: 'admin_tools/menu/item.html'.

    ``children``
        A list of children menu items. All children items must be instances of
        the ``MenuItem`` class.
    """

    title = 'Untitled menu item'
    url = '#'
    css_classes = None
    accesskey = None
    description = None
    enabled = True
    template = 'admin_tools/menu/item.html'
    children = None

    def __init__(self, title=None, url=None, **kwargs):

        if title is not None:
            self.title = title

        if url is not None:
            self.url = url

        for key in kwargs:
            if hasattr(self.__class__, key):
                setattr(self, key, kwargs[key])
        self.children = self.children or []
        self.css_classes = self.css_classes or []

    def init_with_context(self, context):
        """
        Like for menus, menu items have a ``init_with_context`` method that is
        called with a ``django.template.RequestContext`` instance as unique
        argument.
        This gives you enough flexibility to build complex items, for example,
        let's build a "history" menu item, that will list the last ten visited
        pages::

            from admin_tools.menu.items import MenuItem

            class HistoryMenuItem(MenuItem):
                title = 'History'

                def init_with_context(self, context):
                    request = context['request']
                    # we use sessions to store the visited pages stack
                    history = request.session.get('history', [])
                    for item in history:
                        self.children.append(MenuItem(
                            title=item['title'],
                            url=item['url']
                        ))
                    # add the current page to the history
                    history.insert(0, {
                        'title': context['title'],
                        'url': request.META['PATH_INFO']
                    })
                    if len(history) > 10:
                        history = history[:10]
                    request.session['history'] = history

        Here's a screenshot of our history item:

        .. image:: images/history_menu_item.png
        """
        pass

    def is_selected(self, request):
        """
        Helper method that returns ``True`` if the menu item is active.
        A menu item is considered as active if it's URL or one of its
        descendants URL is equals to the current URL.
        """
        current_url = request.get_full_path()
        return self.url == current_url or \
            len([c for c in self.children if c.is_selected(request)]) > 0

    def is_empty(self):
        """
        Helper method that returns ``True`` if the menu item is empty.
        This method always returns ``False`` for basic items, but can return
        ``True`` if the item is an AppList.
        """
        return False

class AppList(MenuItem, AppListElementMixin):
    """
    A menu item that lists installed apps an their models.
    In addition to the parent :class:`~admin_tools.menu.items.MenuItem`
    properties, the ``AppList`` has two extra properties:

    ``exclude_list``
        A list of apps to exclude, if an app name (e.g. "django.contrib.auth"
        starts with an element of this list (e.g. "django.contrib") it won't
        appear in the menu item.

    ``include_list``
        A list of apps to include, only apps whose name (e.g.
        "django.contrib.auth") starts with one of the strings (e.g.
        "django.contrib") in the list will appear in the menu item.

    If no include/exclude list is provided, **all apps** are shown.

    Here's a small example of building an app list menu item::

        from admin_tools.menu import items, Menu

        class MyMenu(Menu):
            def __init__(self, **kwargs):
                super(MyMenu, self).__init__(**kwargs)
                self.children.append(items.AppList(
                    title='Applications',
                    exclude_list=('django.contrib',)
                )

    The screenshot of what this code produces:

    .. image:: images/applist_menu_item.png

    .. note::

        Note that this menu takes into account user permissions, as a
        consequence, if a user has no rights to change or add a ``Group`` for
        example, the ``django.contrib.auth.Group model`` child item won't be
        displayed in the menu.
    """

    def __init__(self, title=None, **kwargs):
        """
        ``AppListMenuItem`` constructor.
        """
        self.include_list = kwargs.pop('include_list', [])
        self.exclude_list = kwargs.pop('exclude_list', [])
        self.models = list(kwargs.pop('models', []))
        self.exclude = list(kwargs.pop('exclude', []))
        super(AppList, self).__init__(title, **kwargs)

    def init_with_context(self, context):
        """
        Please refer to the :meth:`~admin_tools.menu.items.MenuItem.init_with_context`
        documentation from :class:`~admin_tools.menu.items.MenuItem` class.
        """
        items = self._visible_models(context['request'])
        apps = {}
        for model, perms in items:
            if not perms['change']:
                continue
            app_label = model._meta.app_label
            if app_label not in apps:
                apps[app_label] = {
                    'title': capfirst(app_label.title()),
                    'url': reverse('admin:app_list', args=(app_label,)),
                    'models': []
                }
            apps[app_label]['models'].append({
                'title': capfirst(model._meta.verbose_name_plural),
                'url': self._get_admin_change_url(model)
            })

        apps_sorted = apps.keys()
        apps_sorted.sort()
        for app in apps_sorted:
            app_dict = apps[app]
            item = MenuItem(title=app_dict['title'], url=app_dict['url'])
            # sort model list alphabetically
            apps[app]['models'].sort(lambda x, y: cmp(x['title'], y['title']))
            for model_dict in apps[app]['models']:
                item.children.append(MenuItem(**model_dict))
            self.children.append(item)

    def is_empty(self):
        """
        Helper method that returns ``True`` if the applist menu item has no
        children.

        >>> from admin_tools.menu.items import MenuItem, AppList
        >>> item = AppList(title='My menu item')
        >>> item.is_empty()
        True
        >>> item.children.append(MenuItem(title='foo'))
        >>> item.is_empty()
        False
        >>> item.children = []
        >>> item.is_empty()
        True
        """
        return len(self.children) == 0

class ModelList(MenuItem, AppListElementMixin):
    """
    A menu item that lists a set of models.
    In addition to the parent :class:`~admin_tools.menu.items.MenuItem`
    properties, the ``ModelList`` has two extra properties:

    ``models``
        A list of models to include, only models whose name (e.g.
        "blog.comments.Comment") match one of the strings (e.g. "blog.*")
        in the include list will appear in the dashboard module.

    ``exclude``
        A list of models to exclude, if a model name (e.g.
        "blog.comments.Comment" match an element of this list (e.g.
        "blog.comments.*") it won't appear in the dashboard module.

    Here's a small example of building a model list menu item::

        from admin_tools.menu import items, Menu

        class MyMenu(Menu):
            def __init__(self, **kwargs):
                super(MyMenu, self).__init__(**kwargs)
                self.children += [
                    items.ModelList('Authentication', ['django.contrib.auth.*',])
                ]

    .. note::

        Note that this menu takes into account user permissions, as a
        consequence, if a user has no rights to change or add a ``Group`` for
        example, the ``django.contrib.auth.Group model`` item won't be
        displayed in the menu.
    """

    def __init__(self, title=None, models=None, exclude=None, **kwargs):
        """
        ``ModelList`` constructor.
        """
        self.models = list(models or [])
        self.exclude = list(exclude or [])
        self.include_list = kwargs.pop('include_list', []) # deprecated
        self.exclude_list = kwargs.pop('exclude_list', []) # deprecated

        super(ModelList, self).__init__(title, **kwargs)

    def init_with_context(self, context):
        """
        Please refer to the :meth:`~admin_tools.menu.items.MenuItem.init_with_context`
        documentation from :class:`~admin_tools.menu.items.MenuItem` class.
        """
        items = self._visible_models(context['request'])
        for model, perms in items:
            if not perms['change']:
                continue
            title = capfirst(model._meta.verbose_name_plural)
            url = self._get_admin_change_url(model)
            item = MenuItem(title=title, url=url)
            self.children.append(item)

    def is_empty(self):
        """
        Helper method that returns ``True`` if the modellist menu item has no
        children.

        >>> from admin_tools.menu.items import MenuItem, ModelList
        >>> item = ModelList(title='My menu item')
        >>> item.is_empty()
        True
        >>> item.children.append(MenuItem(title='foo'))
        >>> item.is_empty()
        False
        >>> item.children = []
        >>> item.is_empty()
        True
        """
        return len(self.children) == 0

class Bookmarks(MenuItem, AppListElementMixin):
    """
    A menu item that lists pages bookmarked by the user. This menu item also
    adds an extra button to the menu that allows the user to bookmark or
    un-bookmark the current page.

    Here's a small example of adding a bookmark menu item::

        from admin_tools.menu import items, Menu

        class MyMenu(Menu):
            def __init__(self, **kwargs):
                super(MyMenu, self).__init__(**kwargs)
                self.children.append(items.Bookmarks('My bookmarks'))

    """
    title = _('Bookmarks')

    def __init__(self, title=None, **kwargs):
        super(Bookmarks, self).__init__(title, **kwargs)
        if 'bookmark' not in self.css_classes:
            self.css_classes.append('bookmark')

    def init_with_context(self, context):
        """
        Please refer to the :meth:`~admin_tools.menu.items.MenuItem.init_with_context`
        documentation from :class:`~admin_tools.menu.items.MenuItem` class.
        """
        from admin_tools.menu.models import Bookmark

        for b in Bookmark.objects.filter(user=context['request'].user):
            self.children.append(MenuItem(mark_safe(b.title), b.url))

        if not len(self.children):
            self.enabled = False

    def is_selected(self, request):
        """
        A bookmark menu item is never considered as active, the real item is.
        """
        return False

<|endoftext|>


<commit_before>import logging

logger = logging.getLogger(__name__)

def get_clicks_report(client, customer_id, *args, **kwargs
                      ):
    include_fields = kwargs.pop('include_fields', [])
    exclude_fields = kwargs.pop('exclude_fields', ['ConversionTypeName'])
    exclude_terms = kwargs.pop('exclude_terms', ['Significance'])
    exclude_behavior = kwargs.pop('exclude_behavior', ['Segment'])
    kwargs['include_zero_impressions'] = False
    report = client.get_report(
        'CLICK_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        exclude_behavior,
        include_fields,
        *args, **kwargs
    )
    return report

def get_negative_keywords_report(client, customer_id, *args, **kwargs
                                 ):
    exclude_fields = ['ConversionTypeName']
    exclude_terms = []
    report = client.get_report(
        'CAMPAIGN_NEGATIVE_KEYWORDS_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        ['Segment'],
        [],
        *args, **kwargs
    )
    return report

def get_criteria_report(client, customer_id, *args, **kwargs
                        ):
    exclude_fields = ['ConversionTypeName']
    exclude_terms = ['Significance', 'ActiveView', 'Average']
    report = client.get_report(
        'CRITERIA_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        ['Segment'],
        [],
        *args, **kwargs
    )
    return report

def get_ad_performance_report(client, customer_id, *args, **kwargs
                              ):
    logger.debug('Running get_ad_performance_report...')
    include_fields = kwargs.pop('include_fields', [])
    exclude_fields = kwargs.pop('exclude_fields', ['BusinessName',
                                                   'ConversionTypeName',
                                                   'CriterionType',
                                                   'CriterionId',
                                                   'ClickType',
                                                   'ConversionCategoryName',
                                                   'ConversionTrackerId',
                                                   'IsNegative'])
    exclude_terms = kwargs.pop('exclude_terms', ['Significance', 'ActiveView', 'Average'])
    exclude_behavior = kwargs.pop('exclude_behavior', ['Segment'])
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',
                'ExternalCustomerId',
                'CampaignId',
                'CampaignName',
                'CampaignStatus',
                'AdGroupId',
                'AdGroupName',
                'AdGroupStatus',
                'Id',
                'Impressions',
                'Clicks',
                'Conversions',
                'Cost',
                'Status',
                'CreativeUrlCustomParameters',
                'CreativeTrackingUrlTemplate',
                'CreativeDestinationUrl',
                'CreativeFinalUrls',
                'CreativeFinalMobileUrls',
                'CreativeFinalAppUrls',
                'Headline',
                'HeadlinePart1',
                'HeadlinePart2',
                'Description',
                'Description1',
                'Description2',
                'Path1',
                'Path2',
                'DisplayUrl',
            ]
    report = client.get_report(
        'AD_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        exclude_behavior,
        include_fields,
        *args, **kwargs
    )
    return report

def get_keywords_report(client, customer_id, *args, **kwargs
                        ):
    exclude_fields = ['ConversionTypeName']
    exclude_terms = ['Significance']
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',
                'ExternalCustomerId',
                'CampaignId',
                'CampaignName',
                'CampaignStatus',
                'AdGroupId',
                'AdGroupName',
                'AdGroupStatus',
                'Id',
                'Impressions',
                'Clicks',
                'Conversions',
                'Cost',
                'Status',
                'KeywordMatchType',
                'Criteria',
                'BiddingStrategySource',
                'BiddingStrategyType',
                'SearchImpressionShare',
                'CpcBid',
                'CreativeQualityScore',
                'PostClickQualityScore',
                'SearchPredictedCtr',
                'QualityScore',
            ]
    report = client.get_report(
        'KEYWORDS_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        ['Segment'],
        [],
        *args, **kwargs
    )
    return report

def get_search_terms_report(client, customer_id, *args, **kwargs
                            ):
    exclude_fields = ['ConversionTypeName']
    exclude_terms = ['Significance']
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',
                'ExternalCustomerId',
                'CampaignId',
                'CampaignName',
                'CampaignStatus',
                'AdGroupId',
                'AdGroupName',
                'AdGroupStatus',
                'Impressions',
                'Clicks',
                'Conversions',
                'Cost',
                'Status',
                'KeywordId',
                'KeywordTextMatchingQuery',
                'Query',
            ]
    report = client.get_report(
        'SEARCH_QUERY_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        ['Segment'],
        [],
        *args, **kwargs
    )
    return report

def get_campaigns_report(client, customer_id, *args, **kwargs
                         ):
    include_fields = kwargs.pop('include_fields', [])
    exclude_fields = kwargs.pop('exclude_fields', ['ConversionTypeName'])
    exclude_terms = kwargs.pop('exclude_terms', ['Significance'])
    exclude_behavior = kwargs.pop('exclude_behavior', ['Segment'])
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',
                'ExternalCustomerId',
                'CampaignId',
                'CampaignName',
                'CampaignStatus',
                'Impressions',
                'Clicks',
                'Conversions',
                'Cost',
                'Status',
                'BiddingStrategyType',
                'SearchImpressionShare',
            ]
    report = client.get_report(
        'CAMPAIGN_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        exclude_behavior,
        include_fields,
        *args, **kwargs
    )
    return report

def get_labels_report(client, customer_id, *args, **kwargs
                      ):
    """
    Get report from AdWords account 'customer_id' and save to Redshift 'target_name' table
    """
    include_fields = kwargs.pop('include_fields', [])
    exclude_fields = kwargs.pop('exclude_fields', [])
    exclude_terms = kwargs.pop('exclude_terms', [])
    exclude_behavior = kwargs.pop('exclude_behavior', ['Segment'])
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',  # The descriptive name of the Customer account.
                'ExternalCustomerId',  # The Customer ID.
                'LabelId',
                'LabelName',
            ]
    report = client.get_report(
        'LABEL_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        exclude_behavior,
        include_fields,
        *args, **kwargs
    )
    return report

def get_budget_report(
        client, customer_id, *args, **kwargs
):
    """
    Get report from AdWords account 'customer_id' and save to Redshift 'target_name' table
    """
    include_fields = kwargs.pop('include_fields', [])
    exclude_fields = kwargs.pop('exclude_fields', ['ConversionTypeName'])
    exclude_terms = kwargs.pop('exclude_terms', ['Significance'])
    exclude_behavior = kwargs.pop('exclude_behavior', ['Segment'])
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',  # The descriptive name of the Customer account.
                'ExternalCustomerId',  # The Customer ID.
                'BudgetId',
                'BudgetName',
                'BudgetReferenceCount',  # The number of campaigns actively using the budget.
                'Amount',  # The daily budget
                'IsBudgetExplicitlyShared',
                # Shared budget (true) or specific to the campaign (false)
            ]
    report = client.get_report(
        'BUDGET_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        exclude_behavior,
        include_fields,
        *args, **kwargs
    )
    return report

def get_adgroups_report(client, customer_id, *args, **kwargs
                        ):
    include_fields = kwargs.pop('include_fields', [])
    exclude_fields = kwargs.pop('exclude_fields', ['ConversionTypeName'])
    exclude_terms = kwargs.pop('exclude_terms', ['Significance'])
    exclude_behavior = kwargs.pop('exclude_behavior', ['Segment'])
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',
                'ExternalCustomerId',
                'CampaignId',
                'CampaignName',
                'CampaignStatus',
                'AdGroupId',
                'AdGroupName',
                'AdGroupStatus',
                'Impressions',
                'Clicks',
                'Conversions',
                'Cost',
                'Status',
                'BiddingStrategySource',
                'BiddingStrategyType',
                'SearchImpressionShare',
                'CpcBid',
                'Labels',
            ]
    report = client.get_report(
        'ADGROUP_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        exclude_behavior,
        include_fields,
        *args, **kwargs
    )
    return report

def get_campaigns_location_report(client, customer_id, *args, **kwargs
                                  ):
    exclude_fields = []
    exclude_terms = ['Significance']
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',
                'ExternalCustomerId',
                'CampaignId',
                'CampaignName',
                'CampaignStatus',
                'Id',
                'IsNegative',
                'BidModifier',
                'Impressions',
                'Clicks',
                'Conversions',
                'Cost',
            ]
    report = client.get_report(
        'CAMPAIGN_LOCATION_TARGET_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        ['Segment'],
        [],
        *args, **kwargs
    )
    return report
<commit_msg>better style<commit_after>import logging

logger = logging.getLogger(__name__)

def get_clicks_report(client, customer_id, *args, **kwargs):
    include_fields = kwargs.pop('include_fields', [])
    exclude_fields = kwargs.pop('exclude_fields', ['ConversionTypeName'])
    exclude_terms = kwargs.pop('exclude_terms', ['Significance'])
    exclude_behavior = kwargs.pop('exclude_behavior', ['Segment'])
    kwargs['include_zero_impressions'] = False
    report = client.get_report(
        'CLICK_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        exclude_behavior,
        include_fields,
        *args, **kwargs
    )
    return report

def get_negative_keywords_report(client, customer_id, *args, **kwargs):
    exclude_fields = ['ConversionTypeName']
    exclude_terms = []
    report = client.get_report(
        'CAMPAIGN_NEGATIVE_KEYWORDS_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        ['Segment'],
        [],
        *args, **kwargs
    )
    return report

def get_criteria_report(client, customer_id, *args, **kwargs):
    exclude_fields = ['ConversionTypeName']
    exclude_terms = ['Significance', 'ActiveView', 'Average']
    report = client.get_report(
        'CRITERIA_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        ['Segment'],
        [],
        *args, **kwargs
    )
    return report

def get_ad_performance_report(client, customer_id, *args, **kwargs):
    logger.debug('Running get_ad_performance_report...')
    include_fields = kwargs.pop('include_fields', [])
    exclude_fields = kwargs.pop('exclude_fields', ['BusinessName',
                                                   'ConversionTypeName',
                                                   'CriterionType',
                                                   'CriterionId',
                                                   'ClickType',
                                                   'ConversionCategoryName',
                                                   'ConversionTrackerId',
                                                   'IsNegative'])
    exclude_terms = kwargs.pop('exclude_terms', ['Significance', 'ActiveView', 'Average'])
    exclude_behavior = kwargs.pop('exclude_behavior', ['Segment'])
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',
                'ExternalCustomerId',
                'CampaignId',
                'CampaignName',
                'CampaignStatus',
                'AdGroupId',
                'AdGroupName',
                'AdGroupStatus',
                'Id',
                'Impressions',
                'Clicks',
                'Conversions',
                'Cost',
                'Status',
                'CreativeUrlCustomParameters',
                'CreativeTrackingUrlTemplate',
                'CreativeDestinationUrl',
                'CreativeFinalUrls',
                'CreativeFinalMobileUrls',
                'CreativeFinalAppUrls',
                'Headline',
                'HeadlinePart1',
                'HeadlinePart2',
                'Description',
                'Description1',
                'Description2',
                'Path1',
                'Path2',
                'DisplayUrl',
            ]
    report = client.get_report(
        'AD_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        exclude_behavior,
        include_fields,
        *args, **kwargs
    )
    return report

def get_keywords_report(client, customer_id, *args, **kwargs):
    exclude_fields = ['ConversionTypeName']
    exclude_terms = ['Significance']
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',
                'ExternalCustomerId',
                'CampaignId',
                'CampaignName',
                'CampaignStatus',
                'AdGroupId',
                'AdGroupName',
                'AdGroupStatus',
                'Id',
                'Impressions',
                'Clicks',
                'Conversions',
                'Cost',
                'Status',
                'KeywordMatchType',
                'Criteria',
                'BiddingStrategySource',
                'BiddingStrategyType',
                'SearchImpressionShare',
                'CpcBid',
                'CreativeQualityScore',
                'PostClickQualityScore',
                'SearchPredictedCtr',
                'QualityScore',
            ]
    report = client.get_report(
        'KEYWORDS_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        ['Segment'],
        [],
        *args, **kwargs
    )
    return report

def get_search_terms_report(client, customer_id, *args, **kwargs):
    exclude_fields = ['ConversionTypeName']
    exclude_terms = ['Significance']
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',
                'ExternalCustomerId',
                'CampaignId',
                'CampaignName',
                'CampaignStatus',
                'AdGroupId',
                'AdGroupName',
                'AdGroupStatus',
                'Impressions',
                'Clicks',
                'Conversions',
                'Cost',
                'Status',
                'KeywordId',
                'KeywordTextMatchingQuery',
                'Query',
            ]
    report = client.get_report(
        'SEARCH_QUERY_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        ['Segment'],
        [],
        *args, **kwargs
    )
    return report

def get_campaigns_report(client, customer_id, *args, **kwargs):
    include_fields = kwargs.pop('include_fields', [])
    exclude_fields = kwargs.pop('exclude_fields', ['ConversionTypeName'])
    exclude_terms = kwargs.pop('exclude_terms', ['Significance'])
    exclude_behavior = kwargs.pop('exclude_behavior', ['Segment'])
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',
                'ExternalCustomerId',
                'CampaignId',
                'CampaignName',
                'CampaignStatus',
                'Impressions',
                'Clicks',
                'Conversions',
                'Cost',
                'Status',
                'BiddingStrategyType',
                'SearchImpressionShare',
            ]
    report = client.get_report(
        'CAMPAIGN_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        exclude_behavior,
        include_fields,
        *args, **kwargs
    )
    return report

def get_labels_report(client, customer_id, *args, **kwargs):
    """
    Get report from AdWords account 'customer_id' and save to Redshift 'target_name' table
    """
    include_fields = kwargs.pop('include_fields', [])
    exclude_fields = kwargs.pop('exclude_fields', [])
    exclude_terms = kwargs.pop('exclude_terms', [])
    exclude_behavior = kwargs.pop('exclude_behavior', ['Segment'])
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',  # The descriptive name of the Customer account.
                'ExternalCustomerId',  # The Customer ID.
                'LabelId',
                'LabelName',
            ]
    report = client.get_report(
        'LABEL_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        exclude_behavior,
        include_fields,
        *args, **kwargs
    )
    return report

def get_budget_report(
        client, customer_id, *args, **kwargs):
    """
    Get report from AdWords account 'customer_id' and save to Redshift 'target_name' table
    """
    include_fields = kwargs.pop('include_fields', [])
    exclude_fields = kwargs.pop('exclude_fields', ['ConversionTypeName'])
    exclude_terms = kwargs.pop('exclude_terms', ['Significance'])
    exclude_behavior = kwargs.pop('exclude_behavior', ['Segment'])
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',  # The descriptive name of the Customer account.
                'ExternalCustomerId',  # The Customer ID.
                'BudgetId',
                'BudgetName',
                'BudgetReferenceCount',  # The number of campaigns actively using the budget.
                'Amount',  # The daily budget
                'IsBudgetExplicitlyShared',
                # Shared budget (true) or specific to the campaign (false)
            ]
    report = client.get_report(
        'BUDGET_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        exclude_behavior,
        include_fields,
        *args, **kwargs
    )
    return report

def get_adgroups_report(client, customer_id, *args, **kwargs):
    include_fields = kwargs.pop('include_fields', [])
    exclude_fields = kwargs.pop('exclude_fields', ['ConversionTypeName'])
    exclude_terms = kwargs.pop('exclude_terms', ['Significance'])
    exclude_behavior = kwargs.pop('exclude_behavior', ['Segment'])
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',
                'ExternalCustomerId',
                'CampaignId',
                'CampaignName',
                'CampaignStatus',
                'AdGroupId',
                'AdGroupName',
                'AdGroupStatus',
                'Impressions',
                'Clicks',
                'Conversions',
                'Cost',
                'Status',
                'BiddingStrategySource',
                'BiddingStrategyType',
                'SearchImpressionShare',
                'CpcBid',
                'Labels',
            ]
    report = client.get_report(
        'ADGROUP_PERFORMANCE_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        exclude_behavior,
        include_fields,
        *args, **kwargs
    )
    return report

def get_campaigns_location_report(client, customer_id, *args, **kwargs):
    exclude_fields = []
    exclude_terms = ['Significance']
    use_fields = kwargs.pop('fields', False)
    if use_fields:
        try:
            kwargs['fields'] = list(use_fields)
        except TypeError:
            kwargs['fields'] = [
                'AccountDescriptiveName',
                'ExternalCustomerId',
                'CampaignId',
                'CampaignName',
                'CampaignStatus',
                'Id',
                'IsNegative',
                'BidModifier',
                'Impressions',
                'Clicks',
                'Conversions',
                'Cost',
            ]
    report = client.get_report(
        'CAMPAIGN_LOCATION_TARGET_REPORT',
        customer_id,
        exclude_fields,
        exclude_terms,
        ['Segment'],
        [],
        *args, **kwargs
    )
    return report
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import json
import os
import re
import sys
from datetime import datetime, timedelta

import mwparserfromhell
import pywikibot
from pywikibot.data.api import Request

from config import *

os.environ['PYWIKIBOT2_DIR'] = os.path.dirname(os.path.realpath(__file__))

site = pywikibot.Site()
site.login()

config_page = pywikibot.Page(site, config_page_name)
cfg = config_page.text
cfg = json.loads(cfg)
print(json.dumps(cfg, indent=4, ensure_ascii=False))

if not cfg["enable"]:
    exit("disabled\n")

def converttitle(title):
    r = Request(site=site, parameters={
        'action': 'query',
        'titles': title,
        "redirects": 1,
        "converttitles": 1
    })
    data = r.submit()
    title = list(data['query']['pages'].values())[0]['title']
    mode = []
    if 'redirects' in data['query']:  # 重定向
        mode.append('redirects')
    if 'converted' in data['query']:  # 繁簡轉換
        mode.append('converted')
    if 'normalized' in data['query']:  # 命名空間
        mode.append('normalized')
    return {'title': title, 'mode': mode}

def appendComment(text, mode):
    text = text.strip()
    comment = []
    if 'redirects' in mode and isinstance(cfg['comment_redirects'], str):
        comment.append(cfg['comment_redirects'])
    if 'converted' in mode and isinstance(cfg['comment_converted'], str):
        comment.append(cfg['comment_converted'])
    if 'normalized' in mode and isinstance(cfg['comment_normalized'], str):
        comment.append(cfg['comment_normalized'])
    if len(comment) > 0:
        text += '\n' + cfg['comment'].format(''.join(comment)) + '\n\n'
    return text

def fix(pagename):
    if not pagename.startswith("Wikipedia:頁面存廢討論/記錄/"):
        pagename = "Wikipedia:頁面存廢討論/記錄/" + pagename

    print("-"*50)
    print("running for "+pagename)

    afdpage = pywikibot.Page(site, pagename)
    text = afdpage.text

    wikicode = mwparserfromhell.parse(text)

    for section in wikicode.get_sections()[1:]:
        title = str(section.get(0).title)
        print(title)
        if re.search(r"{{\s*(delh|TalkendH)\s*\|", str(section), re.IGNORECASE) != None:
            print("  closed, skip")
            continue

        m = re.search(r"^\[\[([^\]]+)\]\]$", title, re.IGNORECASE)
        if m != None:
            title = m.group(1)
            start = ""
            if title[0] == ":":
                start = ":"
                title = title[1:]

            mode = []

            convert = converttitle(title)
            title = convert['title']
            mode += convert['mode']

            title = "[["+start+title+"]]"

            if str(section.get(0).title) != title:
                if str(section.get(0).title).replace("_", " ") != title:
                    section.insert(
                        1, "\n{{formerly|"+str(section.get(0).title)+"}}")
                print("  set new title = "+title)
                section.get(0).title = title
                newtext = appendComment(str(section), mode)
                section.replace(section, newtext)
            continue

        m = re.search(
            r"^(\[\[[^\]]+\]\][、，])+\[\[[^\]]+\]\]$", title, re.IGNORECASE)
        if m != None:
            titlelist = m.group(0).replace("]]，[[", "]]、[[").split("、")
            newtitlelist = []
            mode = []
            for title in titlelist:
                if title.startswith("[[") and title.endswith("]]"):
                    title = title[2:-2]

                    if title[0] == ":":
                        title = title[1:]

                    convert = converttitle(title)
                    title = convert['title']
                    mode += convert['mode']

                    newtitlelist.append(title)
                else:
                    print("  wrong title: "+title)
                    return
            title = "{{al|"+"|".join(newtitlelist)+"}}"
            if str(section.get(0).title) != title:
                print("  set new title = "+title)
                section.get(0).title = title
                newtext = appendComment(str(section), mode)
                section.replace(section, newtext)
            continue

        m = re.search(r"^{{al\|([^\]]+\|)+[^\]]+}}$", title, re.IGNORECASE)
        if m != None:
            titlelist = m.group(0)[5:-2].split("|")
            newtitlelist = []
            mode = []
            for title in titlelist:
                if title[0] == ":":
                    title = title[1:]

                convert = converttitle(title)
                title = convert['title']
                mode += convert['mode']

                newtitlelist.append(title)
            title = "{{al|"+"|".join(newtitlelist)+"}}"
            if str(section.get(0).title) != title:
                print("  set new title = "+title)
                section.get(0).title = title
                newtext = appendComment(str(section), mode)
                section.replace(section, newtext)
            continue

        print("  unknown format, skip")

    text = str(wikicode)

    if afdpage.text == text:
        print("  nothing changed")
        return

    pywikibot.showDiff(afdpage.text, text)
    summary = cfg["summary"]
    print(summary)
    afdpage.text = text
    afdpage.save(summary=summary, minor=False)

if len(sys.argv) >= 2:
    pagename = sys.argv[1]
    fix(pagename)
else:
    print("run past {} days".format(cfg["run_past_days"]))
    for delta in range(cfg["run_past_days"]):
        rundate = datetime.now() - timedelta(days=delta)
        pagename = rundate.strftime('%Y/%m/%d')
        fix(pagename)
<commit_msg>afd-section-title: Change auto-prepend pagename rule<commit_after># -*- coding: utf-8 -*-
import json
import os
import re
import sys
from datetime import datetime, timedelta

import mwparserfromhell
import pywikibot
from pywikibot.data.api import Request

from config import *

os.environ['PYWIKIBOT2_DIR'] = os.path.dirname(os.path.realpath(__file__))

site = pywikibot.Site()
site.login()

config_page = pywikibot.Page(site, config_page_name)
cfg = config_page.text
cfg = json.loads(cfg)
print(json.dumps(cfg, indent=4, ensure_ascii=False))

if not cfg["enable"]:
    exit("disabled\n")

def converttitle(title):
    r = Request(site=site, parameters={
        'action': 'query',
        'titles': title,
        "redirects": 1,
        "converttitles": 1
    })
    data = r.submit()
    title = list(data['query']['pages'].values())[0]['title']
    mode = []
    if 'redirects' in data['query']:  # 重定向
        mode.append('redirects')
    if 'converted' in data['query']:  # 繁簡轉換
        mode.append('converted')
    if 'normalized' in data['query']:  # 命名空間
        mode.append('normalized')
    return {'title': title, 'mode': mode}

def appendComment(text, mode):
    text = text.strip()
    comment = []
    if 'redirects' in mode and isinstance(cfg['comment_redirects'], str):
        comment.append(cfg['comment_redirects'])
    if 'converted' in mode and isinstance(cfg['comment_converted'], str):
        comment.append(cfg['comment_converted'])
    if 'normalized' in mode and isinstance(cfg['comment_normalized'], str):
        comment.append(cfg['comment_normalized'])
    if len(comment) > 0:
        text += '\n' + cfg['comment'].format(''.join(comment)) + '\n\n'
    return text

def fix(pagename):
    if re.search(r'\d{4}/\d{2}/\d{2}', pagename):
        pagename = "Wikipedia:頁面存廢討論/記錄/" + pagename

    print("-"*50)
    print("running for "+pagename)

    afdpage = pywikibot.Page(site, pagename)
    text = afdpage.text

    wikicode = mwparserfromhell.parse(text)

    for section in wikicode.get_sections()[1:]:
        title = str(section.get(0).title)
        print(title)
        if re.search(r"{{\s*(delh|TalkendH)\s*\|", str(section), re.IGNORECASE) != None:
            print("  closed, skip")
            continue

        m = re.search(r"^\[\[([^\]]+)\]\]$", title, re.IGNORECASE)
        if m != None:
            title = m.group(1)
            start = ""
            if title[0] == ":":
                start = ":"
                title = title[1:]

            mode = []

            convert = converttitle(title)
            title = convert['title']
            mode += convert['mode']

            title = "[["+start+title+"]]"

            if str(section.get(0).title) != title:
                if str(section.get(0).title).replace("_", " ") != title:
                    section.insert(
                        1, "\n{{formerly|"+str(section.get(0).title)+"}}")
                print("  set new title = "+title)
                section.get(0).title = title
                newtext = appendComment(str(section), mode)
                section.replace(section, newtext)
            continue

        m = re.search(
            r"^(\[\[[^\]]+\]\][、，])+\[\[[^\]]+\]\]$", title, re.IGNORECASE)
        if m != None:
            titlelist = m.group(0).replace("]]，[[", "]]、[[").split("、")
            newtitlelist = []
            mode = []
            for title in titlelist:
                if title.startswith("[[") and title.endswith("]]"):
                    title = title[2:-2]

                    if title[0] == ":":
                        title = title[1:]

                    convert = converttitle(title)
                    title = convert['title']
                    mode += convert['mode']

                    newtitlelist.append(title)
                else:
                    print("  wrong title: "+title)
                    return
            title = "{{al|"+"|".join(newtitlelist)+"}}"
            if str(section.get(0).title) != title:
                print("  set new title = "+title)
                section.get(0).title = title
                newtext = appendComment(str(section), mode)
                section.replace(section, newtext)
            continue

        m = re.search(r"^{{al\|([^\]]+\|)+[^\]]+}}$", title, re.IGNORECASE)
        if m != None:
            titlelist = m.group(0)[5:-2].split("|")
            newtitlelist = []
            mode = []
            for title in titlelist:
                if title[0] == ":":
                    title = title[1:]

                convert = converttitle(title)
                title = convert['title']
                mode += convert['mode']

                newtitlelist.append(title)
            title = "{{al|"+"|".join(newtitlelist)+"}}"
            if str(section.get(0).title) != title:
                print("  set new title = "+title)
                section.get(0).title = title
                newtext = appendComment(str(section), mode)
                section.replace(section, newtext)
            continue

        print("  unknown format, skip")

    text = str(wikicode)

    if afdpage.text == text:
        print("  nothing changed")
        return

    pywikibot.showDiff(afdpage.text, text)
    summary = cfg["summary"]
    print(summary)
    afdpage.text = text
    afdpage.save(summary=summary, minor=False)

if len(sys.argv) >= 2:
    pagename = sys.argv[1]
    fix(pagename)
else:
    print("run past {} days".format(cfg["run_past_days"]))
    for delta in range(cfg["run_past_days"]):
        rundate = datetime.now() - timedelta(days=delta)
        pagename = rundate.strftime('%Y/%m/%d')
        fix(pagename)
<|endoftext|>


<commit_before>from django.shortcuts import render, get_object_or_404
from django.views.generic import DetailView

from agir.events.models import Event
from agir.lib.views import IframableMixin
from agir.municipales.models import CommunePage

class CommuneView(IframableMixin, DetailView):
    model = CommunePage
    context_object_name = "commune"

    def get_object(self, queryset=None):
        return get_object_or_404(
            self.model,
            code_departement=self.kwargs["code_departement"],
            slug=self.kwargs["slug"],
        )

    def get_template_names(self):
        if self.request.GET.get("iframe"):
            return ["municipales/commune_details_iframe.html"]
        return ["municipales/commune_details.html"]

    def get_context_data(self, **kwargs):
        kwargs["events"] = Event.objects.upcoming().filter(
            coordinates__coveredby=self.object.coordinates
        )

        return super().get_context_data(**kwargs)
<commit_msg>fix: events order on commune pages<commit_after>from django.shortcuts import render, get_object_or_404
from django.views.generic import DetailView

from agir.events.models import Event
from agir.lib.views import IframableMixin
from agir.municipales.models import CommunePage

class CommuneView(IframableMixin, DetailView):
    model = CommunePage
    context_object_name = "commune"

    def get_object(self, queryset=None):
        return get_object_or_404(
            self.model,
            code_departement=self.kwargs["code_departement"],
            slug=self.kwargs["slug"],
        )

    def get_template_names(self):
        if self.request.GET.get("iframe"):
            return ["municipales/commune_details_iframe.html"]
        return ["municipales/commune_details.html"]

    def get_context_data(self, **kwargs):
        kwargs["events"] = (
            Event.objects.upcoming()
            .filter(coordinates__coveredby=self.object.coordinates)
            .order_by("start_time")
        )

        return super().get_context_data(**kwargs)
<|endoftext|>


<commit_before>from abc import abstractmethod, ABCMeta
from enum import IntEnum

from RULEngine.Game.Referee import RefereeCommand
from ai.Algorithm.IntelligentModule import IntelligentModule

from ai.Algorithm.evaluation_module import *

class AutoPlay(IntelligentModule, metaclass=ABCMeta):
    """
        Classe mère des modules de jeu automatique.
        Un module de jeu est un module intelligent servant à faire la sélection
        des stratégies en prenant en compte différents aspects du jeu, notamment le referee
        et la position des robots et de la balle.
    """
    def __init__(self, worldstate):
        super().__init__(worldstate)
        self.selected_strategy = None
        self.current_state = None
        self.next_state = None

    def get_selected_strategy(self):
        return self.selected_strategy

    @property
    def info(self):
        return {
            "selected_strategy": str(self.selected_strategy),
            "current_state": str(self.current_state)
        }

    @abstractmethod
    def update(self):
        """ Effectue la mise à jour du module """
        pass

    @abstractmethod
    def str(self):
        """
            La représentation en string d'un module intelligent devrait
            permettre de facilement envoyer son information dans un fichier de
            log.
        """

class SimpleAutoPlayState(IntEnum):
    HALT = 0
    STOP = 1
    GOAL_US = 2
    GOAL_THEM = 3
    BALL_PLACEMENT_THEM = 4
    BALL_PLACEMENT_US = 5
    NORMAL_OFFENSE = 6
    NORMAL_DEFENSE = 7
    TIMEOUT = 8
    PREPARE_KICKOFF_OFFENSE = 9
    PREPARE_KICKOFF_DEFENSE = 10
    OFFENSE_KICKOFF = 11
    DEFENSE_KICKOFF = 12
    PREPARE_PENALTY_OFFENSE = 13
    PREPARE_PENALTY_DEFENSE = 14
    OFFENSE_PENALTY = 15
    DEFENSE_PENALTY = 16
    DIRECT_FREE_OFFENSE = 17
    DIRECT_FREE_DEFENSE = 18
    INDIRECT_FREE_OFFENSE = 19
    INDIRECT_FREE_DEFENSE = 20

class SimpleAutoPlay(AutoPlay):
    """
        Classe simple implémentant la sélection de stratégies.
    """
    def __init__(self, worldstate):
        super().__init__(worldstate)
        self.last_ref_command = RefereeCommand.HALT

    def update(self):
        self.next_state = self._select_next_state()

        if self.next_state is None:
            self.next_state = SimpleAutoPlayState.HALT
            self.selected_strategy = self._get_new_strategy(self.next_state)

        elif self.next_state != self.current_state:
            self.selected_strategy = self._get_new_strategy(self.next_state)

        self.current_state = self.next_state

        self.ws.play_state.set_strategy(self.selected_strategy)

    def str(self):
        pass

    def _analyse_game(self):
        # TODO handle kickoff and penalty transition
        if is_ball_our_side():
            return SimpleAutoPlayState.NORMAL_DEFENSE
        else:
            return SimpleAutoPlayState.NORMAL_OFFENSE

    def _normal_start(self):
        return {
            RefereeCommand.PREPARE_KICKOFF_US: SimpleAutoPlayState.OFFENSE_KICKOFF,
            RefereeCommand.PREPARE_KICKOFF_THEM: SimpleAutoPlayState.DEFENSE_KICKOFF,
            RefereeCommand.PREPARE_PENALTY_US: SimpleAutoPlayState.OFFENSE_PENALTY,
            RefereeCommand.PREPARE_PENALTY_THEM: SimpleAutoPlayState.DEFENSE_PENALTY,
            RefereeCommand.NORMAL_START: self._analyse_game()
        }.get(self.last_ref_command, RefereeCommand.NORMAL_START)

    def _select_next_state(self):
        referee = self.ws.game_state.game.referee
        next_state = self.current_state
        # On command change
        if self.last_ref_command != referee.command:
            next_state = {
                RefereeCommand.HALT: SimpleAutoPlayState.HALT,

                RefereeCommand.STOP: SimpleAutoPlayState.STOP,
                RefereeCommand.GOAL_US: self.current_state,
                RefereeCommand.GOAL_THEM: self.current_state,
                RefereeCommand.BALL_PLACEMENT_THEM: SimpleAutoPlayState.STOP,

                RefereeCommand.BALL_PLACEMENT_US: SimpleAutoPlayState.HALT, #TODO send ball new position to strategy...

                RefereeCommand.FORCE_START: self._analyse_game(),
                RefereeCommand.NORMAL_START: self._normal_start(),

                RefereeCommand.TIMEOUT_THEM: SimpleAutoPlayState.TIMEOUT,
                RefereeCommand.TIMEOUT_US: SimpleAutoPlayState.TIMEOUT,

                RefereeCommand.PREPARE_KICKOFF_US: SimpleAutoPlayState.PREPARE_KICKOFF_OFFENSE,
                RefereeCommand.PREPARE_KICKOFF_THEM: SimpleAutoPlayState.PREPARE_KICKOFF_DEFENSE,
                RefereeCommand.PREPARE_PENALTY_US: SimpleAutoPlayState.PREPARE_PENALTY_OFFENSE,
                RefereeCommand.PREPARE_PENALTY_THEM: SimpleAutoPlayState.PREPARE_PENALTY_DEFENSE,

                RefereeCommand.DIRECT_FREE_US: SimpleAutoPlayState.DIRECT_FREE_OFFENSE,
                RefereeCommand.DIRECT_FREE_THEM: SimpleAutoPlayState.DIRECT_FREE_DEFENSE,
                RefereeCommand.INDIRECT_FREE_US: SimpleAutoPlayState.INDIRECT_FREE_OFFENSE,
                RefereeCommand.INDIRECT_FREE_THEM: SimpleAutoPlayState.INDIRECT_FREE_DEFENSE

            }.get(referee.command, RefereeCommand.HALT)

        # During the game
        elif referee.command == RefereeCommand.FORCE_START or\
            referee.command == RefereeCommand.NORMAL_START:
            next_state = self._analyse_game()

        self.last_ref_command = referee.command
        return next_state

    def _get_new_strategy(self, state):
        name = self._get_strategy_name(state)
        return self.ws.play_state.get_new_strategy(name)(self.ws.game_state)

    def _get_strategy_name(self, state):
        return {
            # Robots must be stopped
            SimpleAutoPlayState.HALT: 'DoNothing',

            # Robots must stay 50 cm from the ball
            SimpleAutoPlayState.STOP: 'DoNothing',
            SimpleAutoPlayState.GOAL_US: 'DoNothing',
            SimpleAutoPlayState.GOAL_THEM: 'DoNothing',
            SimpleAutoPlayState.BALL_PLACEMENT_THEM: 'DoNothing',

            # Place the ball to the designated position
            SimpleAutoPlayState.BALL_PLACEMENT_US: 'DoNothing',

            SimpleAutoPlayState.NORMAL_OFFENSE: 'Offense',
            SimpleAutoPlayState.NORMAL_DEFENSE: 'DoNothing',

            SimpleAutoPlayState.TIMEOUT: 'DoNothing',

            # Kickoff
            SimpleAutoPlayState.PREPARE_KICKOFF_OFFENSE: 'PrepareKickOffOffense',
            SimpleAutoPlayState.PREPARE_KICKOFF_DEFENSE: 'DoNothing',
            SimpleAutoPlayState.OFFENSE_KICKOFF: 'DoNothing',
            SimpleAutoPlayState.DEFENSE_KICKOFF: 'DoNothing',

            # Penalty
            SimpleAutoPlayState.PREPARE_PENALTY_OFFENSE: 'DoNothing',
            SimpleAutoPlayState.PREPARE_PENALTY_DEFENSE: 'DoNothing',
            SimpleAutoPlayState.OFFENSE_PENALTY: 'DoNothing',
            SimpleAutoPlayState.DEFENSE_PENALTY: 'DoNothing',

            # Freekicks
            SimpleAutoPlayState.DIRECT_FREE_DEFENSE: 'DoNothing',
            SimpleAutoPlayState.DIRECT_FREE_OFFENSE: 'DoNothing',
            SimpleAutoPlayState.INDIRECT_FREE_DEFENSE: 'DoNothing',
            SimpleAutoPlayState.INDIRECT_FREE_OFFENSE: 'DoNothing'

        }.get(state, SimpleAutoPlayState.HALT)<commit_msg>Adding some strategy to auplay<commit_after>from abc import abstractmethod, ABCMeta
from enum import IntEnum

from RULEngine.Game.Referee import RefereeCommand
from ai.Algorithm.IntelligentModule import IntelligentModule

from ai.Algorithm.evaluation_module import *

class AutoPlay(IntelligentModule, metaclass=ABCMeta):
    """
        Classe mère des modules de jeu automatique.
        Un module de jeu est un module intelligent servant à faire la sélection
        des stratégies en prenant en compte différents aspects du jeu, notamment le referee
        et la position des robots et de la balle.
    """
    def __init__(self, worldstate):
        super().__init__(worldstate)
        self.selected_strategy = None
        self.current_state = None
        self.next_state = None

    def get_selected_strategy(self):
        return self.selected_strategy

    @property
    def info(self):
        return {
            "selected_strategy": str(self.selected_strategy),
            "current_state": str(self.current_state)
        }

    @abstractmethod
    def update(self):
        """ Effectue la mise à jour du module """
        pass

    @abstractmethod
    def str(self):
        """
            La représentation en string d'un module intelligent devrait
            permettre de facilement envoyer son information dans un fichier de
            log.
        """

class SimpleAutoPlayState(IntEnum):
    HALT = 0
    STOP = 1
    GOAL_US = 2
    GOAL_THEM = 3
    BALL_PLACEMENT_THEM = 4
    BALL_PLACEMENT_US = 5
    NORMAL_OFFENSE = 6
    NORMAL_DEFENSE = 7
    TIMEOUT = 8
    PREPARE_KICKOFF_OFFENSE = 9
    PREPARE_KICKOFF_DEFENSE = 10
    OFFENSE_KICKOFF = 11
    DEFENSE_KICKOFF = 12
    PREPARE_PENALTY_OFFENSE = 13
    PREPARE_PENALTY_DEFENSE = 14
    OFFENSE_PENALTY = 15
    DEFENSE_PENALTY = 16
    DIRECT_FREE_OFFENSE = 17
    DIRECT_FREE_DEFENSE = 18
    INDIRECT_FREE_OFFENSE = 19
    INDIRECT_FREE_DEFENSE = 20

class SimpleAutoPlay(AutoPlay):
    """
        Classe simple implémentant la sélection de stratégies.
    """
    def __init__(self, worldstate):
        super().__init__(worldstate)
        self.last_ref_command = RefereeCommand.HALT

    def update(self):
        self.next_state = self._select_next_state()

        if self.next_state is None:
            self.next_state = SimpleAutoPlayState.HALT
            self.selected_strategy = self._get_new_strategy(self.next_state)

        elif self.next_state != self.current_state:
            self.selected_strategy = self._get_new_strategy(self.next_state)

        self.current_state = self.next_state

        self.ws.play_state.set_strategy(self.selected_strategy)

    def str(self):
        pass

    def _analyse_game(self):
        if is_ball_our_side():
            return SimpleAutoPlayState.NORMAL_DEFENSE
        else:
            return SimpleAutoPlayState.NORMAL_OFFENSE

    def _normal_start(self):
        return {
            RefereeCommand.PREPARE_KICKOFF_US: SimpleAutoPlayState.OFFENSE_KICKOFF,
            RefereeCommand.PREPARE_KICKOFF_THEM: SimpleAutoPlayState.DEFENSE_KICKOFF,
            RefereeCommand.PREPARE_PENALTY_US: SimpleAutoPlayState.OFFENSE_PENALTY,
            RefereeCommand.PREPARE_PENALTY_THEM: SimpleAutoPlayState.DEFENSE_PENALTY,
            RefereeCommand.NORMAL_START: self._analyse_game()
        }.get(self.last_ref_command, RefereeCommand.NORMAL_START)

    def _select_next_state(self):
        referee = self.ws.game_state.game.referee
        next_state = self.current_state
        # On command change
        if self.last_ref_command != referee.command:
            next_state = {
                RefereeCommand.HALT: SimpleAutoPlayState.HALT,

                RefereeCommand.STOP: SimpleAutoPlayState.STOP,
                RefereeCommand.GOAL_US: self.current_state,
                RefereeCommand.GOAL_THEM: self.current_state,
                RefereeCommand.BALL_PLACEMENT_THEM: SimpleAutoPlayState.STOP,

                RefereeCommand.BALL_PLACEMENT_US: SimpleAutoPlayState.HALT, #TODO send ball new position to strategy...

                RefereeCommand.FORCE_START: self._analyse_game(),
                RefereeCommand.NORMAL_START: self._normal_start(),

                RefereeCommand.TIMEOUT_THEM: SimpleAutoPlayState.TIMEOUT,
                RefereeCommand.TIMEOUT_US: SimpleAutoPlayState.TIMEOUT,

                RefereeCommand.PREPARE_KICKOFF_US: SimpleAutoPlayState.PREPARE_KICKOFF_OFFENSE,
                RefereeCommand.PREPARE_KICKOFF_THEM: SimpleAutoPlayState.PREPARE_KICKOFF_DEFENSE,
                RefereeCommand.PREPARE_PENALTY_US: SimpleAutoPlayState.PREPARE_PENALTY_OFFENSE,
                RefereeCommand.PREPARE_PENALTY_THEM: SimpleAutoPlayState.PREPARE_PENALTY_DEFENSE,

                RefereeCommand.DIRECT_FREE_US: SimpleAutoPlayState.DIRECT_FREE_OFFENSE,
                RefereeCommand.DIRECT_FREE_THEM: SimpleAutoPlayState.DIRECT_FREE_DEFENSE,
                RefereeCommand.INDIRECT_FREE_US: SimpleAutoPlayState.INDIRECT_FREE_OFFENSE,
                RefereeCommand.INDIRECT_FREE_THEM: SimpleAutoPlayState.INDIRECT_FREE_DEFENSE

            }.get(referee.command, RefereeCommand.HALT)

        # During the game
        elif referee.command == RefereeCommand.FORCE_START or\
            referee.command == RefereeCommand.NORMAL_START:
            next_state = self._analyse_game()

        self.last_ref_command = referee.command
        return next_state

    def _get_new_strategy(self, state):
        name = self._get_strategy_name(state)
        return self.ws.play_state.get_new_strategy(name)(self.ws.game_state)

    def _get_strategy_name(self, state):
        return {
            # Robots must be stopped
            SimpleAutoPlayState.HALT: 'DoNothing',

            # Robots must stay 50 cm from the ball
            SimpleAutoPlayState.STOP: 'DoNothing',
            SimpleAutoPlayState.GOAL_US: 'DoNothing',
            SimpleAutoPlayState.GOAL_THEM: 'DoNothing',
            SimpleAutoPlayState.BALL_PLACEMENT_THEM: 'DoNothing',

            # Place the ball to the designated position
            SimpleAutoPlayState.BALL_PLACEMENT_US: 'DoNothing',

            SimpleAutoPlayState.NORMAL_OFFENSE: 'Offense',
            SimpleAutoPlayState.NORMAL_DEFENSE: 'DefenseWall',

            SimpleAutoPlayState.TIMEOUT: 'DoNothing',

            # Kickoff
            SimpleAutoPlayState.PREPARE_KICKOFF_OFFENSE: 'PrepareKickOffOffense',
            SimpleAutoPlayState.PREPARE_KICKOFF_DEFENSE: 'DoNothing',
            SimpleAutoPlayState.OFFENSE_KICKOFF: 'Offense',
            SimpleAutoPlayState.DEFENSE_KICKOFF: 'DefenseWall',

            # Penalty
            SimpleAutoPlayState.PREPARE_PENALTY_OFFENSE: 'DoNothing',
            SimpleAutoPlayState.PREPARE_PENALTY_DEFENSE: 'DoNothing',
            SimpleAutoPlayState.OFFENSE_PENALTY: 'DoNothing',
            SimpleAutoPlayState.DEFENSE_PENALTY: 'DefenseWall',

            # Freekicks
            SimpleAutoPlayState.DIRECT_FREE_DEFENSE: 'DefenseWall',
            SimpleAutoPlayState.DIRECT_FREE_OFFENSE: 'DoNothing',
            SimpleAutoPlayState.INDIRECT_FREE_DEFENSE: 'DefenseWall',
            SimpleAutoPlayState.INDIRECT_FREE_OFFENSE: 'DoNothing'

        }.get(state, SimpleAutoPlayState.HALT)<|endoftext|>


<commit_before>import aiohttp_jinja2
import jinja2

from .admin import Admin
from .base import BaseView
from .consts import APP_KEY, TEMPLATE_APP_KEY
from .consts import PROJ_ROOT, TEMPLATES_ROOT

__all__ = ['Admin', 'BaseView', 'setup']
__version__ = '0.0.1'

def setup(app, name=None, url=None, index_view=None,
          translations_path=None, endpoint=None, static_url_path=None,
          base_template=None, template_mode=None, category_icon_classes=None):

    app.router.add_static('/static/admin',
                          path=str(PROJ_ROOT / 'static'),
                          name='admin.static')

    loader = jinja2.FileSystemLoader([str(TEMPLATES_ROOT)])
    aiohttp_jinja2.setup(app, loader=loader, app_key=TEMPLATE_APP_KEY)

    admin = Admin(app, name=name, url=url, index_view=index_view,
                  translations_path=translations_path, endpoint=endpoint,
                  static_url_path=static_url_path,
                  base_template=base_template, template_mode=template_mode,
                  category_icon_classes=category_icon_classes)
    app[APP_KEY] = admin
    return admin
<commit_msg>now we can provide folter for additional admin templates<commit_after>import aiohttp_jinja2
import jinja2

from .admin import Admin
from .base import BaseView
from .consts import APP_KEY, TEMPLATE_APP_KEY
from .consts import PROJ_ROOT, TEMPLATES_ROOT

__all__ = ['Admin', 'BaseView', 'setup']
__version__ = '0.0.1'

def setup(app, name=None, url=None, index_view=None,
          translations_path=None, endpoint=None, static_url_path=None,
          base_template=None, template_mode='bootstrap3',
          category_icon_classes=None,
          template_folder=None):

    app.router.add_static('/static/admin',
                          path=str(PROJ_ROOT / 'static'),
                          name='admin.static')

    # gather template folders: default and provided
    if not isinstance(template_folder, list):
        template_folder = [template_folder]
    template_root = str(TEMPLATES_ROOT / template_mode)
    if template_folder is None:
        template_folders = [template_root]
    else:
        template_folders = [template_root] + template_folder

    # init aiohttp_jinja plugin
    loader = jinja2.FileSystemLoader(template_folders)
    aiohttp_jinja2.setup(app, loader=loader, app_key=TEMPLATE_APP_KEY)

    admin = Admin(app, name=name, url=url, index_view=index_view,
                  translations_path=translations_path, endpoint=endpoint,
                  static_url_path=static_url_path,
                  base_template=base_template, template_mode=template_mode,
                  category_icon_classes=category_icon_classes)
    app[APP_KEY] = admin
    return admin
<|endoftext|>


<commit_before>import logging
from collections import Mapping

from .. import utils

logger = logging.getLogger(__name__)

async def load_entities(conf, context=None, loop=None, entities=None, path=()):
    from .context import Context

    ents = {}
    if entities is None:
        entities = ents

    if conf.get('app.cls'):
        cls = utils.import_name(conf['app.cls'])
        context.app = await cls.factory(
            config=conf, context=context, loop=loop)

    elif context is None:
        context = Context(conf, loop=loop)

    for k, v in conf.items():
        if not isinstance(v, Mapping):
            pass
        elif k in ('logging', 'app'):
            pass
        elif 'cls' in v:
            if 'name' not in v:
                v['name'] = '.'.join(path + (k,))
            cls = utils.import_name(v['cls'])
            logger.debug('Imported "{}" as {}'.format(v['cls'], cls))
            conf[k] = cls(v, context=context, loop=loop)
            entities[path + (k,)] = conf[k]
        else:
            conf[k] = await load_entities(
                conf[k],
                context=context,
                loop=loop,
                entities=entities,
                path=path + (k,))

    for i in ents.values():
        await i.init()

    return conf
<commit_msg>compatible spike<commit_after>import logging
from collections import Mapping

from .. import utils

logger = logging.getLogger(__name__)

async def load_entities(conf, context=None, loop=None, entities=None, path=()):
    from .context import Context

    ents = {}
    if entities is None:
        entities = ents

    if conf.get('app.cls'):
        cls = utils.import_name(conf['app.cls'])
        conf['app'] = await cls.factory(
            config=conf, context=context, loop=loop)

    elif context is None:
        context = Context(conf, loop=loop)

    for k, v in conf.items():
        if not isinstance(v, Mapping):
            pass
        elif k in ('logging', 'app'):
            pass
        elif 'cls' in v:
            if 'name' not in v:
                v['name'] = '.'.join(path + (k,))
            cls = utils.import_name(v['cls'])
            logger.debug('Imported "{}" as {}'.format(v['cls'], cls))
            conf[k] = cls(v, context=context, loop=loop)
            entities[path + (k,)] = conf[k]
        else:
            conf[k] = await load_entities(
                conf[k],
                context=context,
                loop=loop,
                entities=entities,
                path=path + (k,))

    for i in ents.values():
        await i.init()

    return conf
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.conf import settings
from django.contrib import messages
from django.shortcuts import redirect, get_object_or_404
from django.utils.translation import ugettext_lazy as _
from django.views.generic import FormView, DetailView

from pyrate.services import mailchimp

from .utils import get_language_for_code
from .forms import SubscriptionPluginForm
from .models import SubscriptionPlugin, Campaign

ERROR_MESSAGES = {
    104: _('Invalid API-Key'),
    200: _('The selected list does not exist.'),
    214: _('You are already subscribed to our list.'),
    230: _('You are already subscribed to our list.'),
}

class SubscriptionView(FormView):

    form_class = SubscriptionPluginForm
    template_name = 'aldryn_mailchimp/subscription.html'

    def form_valid(self, form):
        h = mailchimp.MailchimpPyrate(settings.MAILCHIMP_API_KEY)
        plugin = get_object_or_404(SubscriptionPlugin, pk=form.cleaned_data['plugin_id'])

        merge_vars = None
        if plugin.assign_language:
            language = get_language_for_code(self.request.LANGUAGE_CODE)
            if language:
                merge_vars = {'mc_language': language}

        try:
            h.subscribe_to_list(list_id=plugin.list_id, user_email=form.cleaned_data['email'], merge_vars=merge_vars)
        except Exception, e:
            if hasattr(e, 'code') and e.code in ERROR_MESSAGES:
                message = ERROR_MESSAGES.get(e.code)
            else:
                message = _(u'Oops, something must have gone wrong. Please try again later.')
            messages.error(self.request, message)
        else:
            messages.success(self.request, _(u'You have successfully subscribed to our mailing list.'))
        return redirect(form.cleaned_data['redirect_url'])

class CampaignDetail(DetailView):
    model = Campaign

    @property
    def template_name_suffix(self):
        default = '_detail'
        iframe = '_detail_iframe'
        return iframe if 'iframe' in self.request.GET else default

    def get_queryset(self):
        return self.model.objects.published()

campaign_detail = CampaignDetail.as_view()
<commit_msg>redirect when form is invalid<commit_after># -*- coding: utf-8 -*-
from django.conf import settings
from django.contrib import messages
from django.http import HttpResponseBadRequest, HttpResponseRedirect
from django.shortcuts import redirect, get_object_or_404
from django.utils.translation import ugettext_lazy as _
from django.views.generic import FormView, DetailView

from pyrate.services import mailchimp

from .utils import get_language_for_code
from .forms import SubscriptionPluginForm
from .models import SubscriptionPlugin, Campaign

ERROR_MESSAGES = {
    104: _('Invalid API-Key'),
    200: _('The selected list does not exist.'),
    214: _('You are already subscribed to our list.'),
    230: _('You are already subscribed to our list.'),
}

class SubscriptionView(FormView):

    form_class = SubscriptionPluginForm
    template_name = 'aldryn_mailchimp/subscription.html'

    def form_valid(self, form):
        h = mailchimp.MailchimpPyrate(settings.MAILCHIMP_API_KEY)
        plugin = get_object_or_404(SubscriptionPlugin, pk=form.cleaned_data['plugin_id'])

        merge_vars = None
        if plugin.assign_language:
            language = get_language_for_code(self.request.LANGUAGE_CODE)
            if language:
                merge_vars = {'mc_language': language}

        try:
            h.subscribe_to_list(list_id=plugin.list_id, user_email=form.cleaned_data['email'], merge_vars=merge_vars)
        except Exception, e:
            if hasattr(e, 'code') and e.code in ERROR_MESSAGES:
                message = ERROR_MESSAGES.get(e.code)
            else:
                message = _(u'Oops, something must have gone wrong. Please try again later.')
            messages.error(self.request, message)
        else:
            messages.success(self.request, _(u'You have successfully subscribed to our mailing list.'))
        return redirect(form.cleaned_data['redirect_url'])

    def form_invalid(self, form):
        redirect_url = form.data.get('redirect_url')

        if redirect_url:
            message = _(u'Please enter a valid email.')
            messages.error(self.request, message)
            response = HttpResponseRedirect(redirect_url)
        else:
            # user has tampered with the redirect_url field.
            response = HttpResponseBadRequest()
        return response

class CampaignDetail(DetailView):
    model = Campaign

    @property
    def template_name_suffix(self):
        default = '_detail'
        iframe = '_detail_iframe'
        return iframe if 'iframe' in self.request.GET else default

    def get_queryset(self):
        return self.model.objects.published()

campaign_detail = CampaignDetail.as_view()
<|endoftext|>


<commit_before>import logging
from banal import is_mapping, ensure_list
from followthemoney import model
from followthemoney.exc import InvalidData
from followthemoney.pragma import remove_checksums

from aleph.model import Entity, Document
from aleph.queues import ingest_entity
from aleph.analysis import analyze_entity
from aleph.queues import queue_task, OP_INDEX
from aleph.index.entities import index_bulk
from aleph.logic.entities import refresh_entity_id
from aleph.logic.collections import refresh_collection
from aleph.logic.aggregator import get_aggregator

log = logging.getLogger(__name__)

def _collection_proxies(collection):
    for entity in Entity.by_collection(collection.id).yield_per(5000):
        yield entity.to_proxy()
    for document in Document.by_collection(collection.id).yield_per(5000):
        yield document.to_proxy()

def process_collection(stage, collection, ingest=True, sync=False):
    """Trigger a full re-parse of all documents and re-build the
    search index from the aggregator."""
    aggregator = get_aggregator(collection)
    for proxy in _collection_proxies(collection):
        if ingest and proxy.schema.is_a(Document.SCHEMA):
            ingest_entity(collection, proxy,
                          job_id=stage.job.id,
                          sync=sync)
        else:
            aggregator.put(proxy, fragment='db')
            queue_task(collection, OP_INDEX,
                       job_id=stage.job.id,
                       payload={'entity_id': proxy.id},
                       context={'sync': sync})
    aggregator.close()

def _process_entity(entity, sync=False):
    """Perform pre-index processing on an entity, includes running the
    NLP pipeline."""
    analyze_entity(entity)
    if sync:
        refresh_entity_id(entity.id)
    # log.debug("Index: %r", entity)
    return entity

def _fetch_entities(stage, collection, entity_ids=None, batch=100):
    aggregator = get_aggregator(collection)
    if entity_ids is not None:
        entity_ids = ensure_list(entity_ids)
        # WEIRD: Instead of indexing a single entity, this will try
        # pull a whole batch of them off the queue and do it at once.
        tasks = stage.get_tasks(limit=max(1, batch - len(entity_ids)))
        for task in tasks:
            entity_ids.extend(ensure_list(task.payload.get('entity_ids')))
        # FIXME: this doesn't retain mapping_id properly.
        stage.mark_done(len(tasks))

    yield from aggregator.iterate(entity_id=entity_ids)
    aggregator.close()

def index_aggregate(stage, collection, sync=False, entity_ids=None,
                    mapping_id=None):
    """Project the contents of the collections aggregator into the index."""
    entities = _fetch_entities(stage, collection, entity_ids=entity_ids)
    entities = (_process_entity(e, sync=sync) for e in entities)
    extra = {'job_id': stage.job.id, 'mapping_id': mapping_id}
    index_bulk(collection, entities, extra)
    refresh_collection(collection.id)

def bulk_write(collection, entities, job_id=None, unsafe=False):
    """Write a set of entities - given as dicts - to the index."""
    # This is called mainly by the /api/2/collections/X/_bulk API.
    def _generate():
        for data in entities:
            if not is_mapping(data):
                raise InvalidData("Failed to read input data", errors=data)
            entity = model.get_proxy(data)
            if entity.id is None:
                raise InvalidData("No ID for entity", errors=entity.to_dict())
            if not unsafe:
                entity = remove_checksums(entity)
            yield _process_entity(entity)

    index_bulk(collection, _generate(), {'job_id': job_id})
    refresh_collection(collection.id)
<commit_msg>Support really sync indexing<commit_after>import logging
from banal import is_mapping, ensure_list
from followthemoney import model
from followthemoney.exc import InvalidData
from followthemoney.pragma import remove_checksums

from aleph.model import Entity, Document
from aleph.queues import ingest_entity
from aleph.analysis import analyze_entity
from aleph.queues import queue_task, OP_INDEX
from aleph.index.entities import index_bulk
from aleph.logic.entities import refresh_entity_id
from aleph.logic.collections import refresh_collection
from aleph.logic.aggregator import get_aggregator

log = logging.getLogger(__name__)

def _collection_proxies(collection):
    for entity in Entity.by_collection(collection.id).yield_per(5000):
        yield entity.to_proxy()
    for document in Document.by_collection(collection.id).yield_per(5000):
        yield document.to_proxy()

def process_collection(stage, collection, ingest=True, sync=False):
    """Trigger a full re-parse of all documents and re-build the
    search index from the aggregator."""
    aggregator = get_aggregator(collection)
    for proxy in _collection_proxies(collection):
        if ingest and proxy.schema.is_a(Document.SCHEMA):
            ingest_entity(collection, proxy,
                          job_id=stage.job.id,
                          sync=sync)
        else:
            aggregator.put(proxy, fragment='db')
            queue_task(collection, OP_INDEX,
                       job_id=stage.job.id,
                       payload={'entity_ids': [proxy.id]},
                       context={'sync': sync})
    aggregator.close()

def _process_entity(entity, sync=False):
    """Perform pre-index processing on an entity, includes running the
    NLP pipeline."""
    analyze_entity(entity)
    if sync:
        refresh_entity_id(entity.id)
    # log.debug("Index: %r", entity)
    return entity

def _fetch_entities(stage, collection, entity_ids=None, batch=100):
    aggregator = get_aggregator(collection)
    if entity_ids is not None:
        entity_ids = ensure_list(entity_ids)
        # WEIRD: Instead of indexing a single entity, this will try
        # pull a whole batch of them off the queue and do it at once.
        tasks = stage.get_tasks(limit=max(1, batch - len(entity_ids)))
        for task in tasks:
            entity_ids.extend(ensure_list(task.payload.get('entity_ids')))
        # FIXME: this doesn't retain mapping_id properly.
        stage.mark_done(len(tasks))

    yield from aggregator.iterate(entity_id=entity_ids)
    aggregator.close()

def index_aggregate(stage, collection, sync=False, entity_ids=None,
                    mapping_id=None):
    """Project the contents of the collections aggregator into the index."""
    entities = _fetch_entities(stage, collection, entity_ids=entity_ids)
    entities = (_process_entity(e, sync=sync) for e in entities)
    extra = {'job_id': stage.job.id, 'mapping_id': mapping_id}
    index_bulk(collection, entities, extra, sync=sync)
    refresh_collection(collection.id, sync=sync)

def bulk_write(collection, entities, job_id=None, unsafe=False):
    """Write a set of entities - given as dicts - to the index."""
    # This is called mainly by the /api/2/collections/X/_bulk API.
    def _generate():
        for data in entities:
            if not is_mapping(data):
                raise InvalidData("Failed to read input data", errors=data)
            entity = model.get_proxy(data)
            if entity.id is None:
                raise InvalidData("No ID for entity", errors=entity.to_dict())
            if not unsafe:
                entity = remove_checksums(entity)
            yield _process_entity(entity)

    index_bulk(collection, _generate(), {'job_id': job_id})
    refresh_collection(collection.id)
<|endoftext|>