<commit_before>import uuid

from django.db.models.fields import FieldDoesNotExist
from django.db.models.fields.related import ForeignObjectRel
from django.db.models.fields.related import OneToOneRel

from django_filters import Filter, FilterSet, NumberFilter, DateFilter
from rest_framework.filters import OrderingFilter

from api.generics.filters import CommaSeparatedCharFilter
from iati.models import Activity, Budget, RelatedActivity

class CommaSeparatedDateRangeFilter(Filter):

    def filter(self, qs, value):

        if value in ([], (), {}, None, ''):
            return qs

        value = value.split(',')

        return super(CommaSeparatedCharFilter, self).filter(qs, value)

class TogetherFilter(Filter):
    """
    Used with TogetherFilterSet, always gets called regardless of GET args
    """

    def __init__(self, filters=None, values=None, **kwargs):
        self.filter_classes = filters
        self.values = values

        super(TogetherFilter, self).__init__(**kwargs)

    def filter(self, qs, values):
        if self.filter_classes:
            filters = { "%s__%s" % (c[0].name, c[0].lookup_type) : c[1] for c in zip(self.filter_classes, values)}
            qs = qs.filter(**filters).distinct()

            return qs

class TogetherFilterSet(FilterSet):
    def __init__(self, data=None, queryset=None, prefix=None, strict=None):
        """
        Adds a together_exclusive meta option that selects fields that have to
        be called in the same django filter() call when both present
        """

        meta = getattr(self, 'Meta', None)

        # fields that must be filtered in the same filter call
        self.together_exclusive = getattr(meta, 'together_exclusive', None)

        data = data.copy()

        for filterlist in self.together_exclusive:
            if set(filterlist).issubset(data.keys()):

                filter_values = [data.pop(filteritem)[0] for filteritem in filterlist]
                filter_classes = [self.declared_filters.get(filteritem, None) for filteritem in filterlist]

                uid = uuid.uuid4()

                self.base_filters[uid] = TogetherFilter(filters=filter_classes)
                data.appendlist(uid, filter_values)

        super(FilterSet, self).__init__(data, queryset, prefix, strict)

class ActivityFilter(TogetherFilterSet):

    activity_id = CommaSeparatedCharFilter(
        name='id',
        lookup_type='in')

    activity_scope = CommaSeparatedCharFilter(
        name='scope__code',
        lookup_type='in')

    recipient_country = CommaSeparatedCharFilter(
        lookup_type='in',
        name='recipient_country')

    recipient_region = CommaSeparatedCharFilter(
        lookup_type='in',
        name='recipient_region')

    recipient_region_not_in = CommaSeparatedCharFilter(
        lookup_type='in',
        name='recipient_region',
        exclude=True)

    planned_start_date_lte = DateFilter(
        lookup_type='lte',
        name='planned_start')

    planned_start_date_gte = DateFilter(
        lookup_type='gte',
        name='planned_start')

    actual_start_date_lte = DateFilter(
        lookup_type='lte',
        name='actual_start')

    actual_start_date_gte = DateFilter(
        lookup_type='gte',
        name='actual_start')

    planned_end_date_lte = DateFilter(
        lookup_type='lte',
        name='planned_end')

    planned_end_date_gte = DateFilter(
        lookup_type='gte',
        name='planned_end')

    actual_end_date_lte = DateFilter(
        lookup_type='lte',
        name='actual_end')

    actual_end_date_gte = DateFilter(
        lookup_type='gte',
        name='actual_end')

    sector = CommaSeparatedCharFilter(
        lookup_type='in',
        name='sector')

    total_budget_value_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregations__total_budget_value')

    total_budget_value_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregations__total_budget_value')

    total_child_budget_value_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregations__total_child_budget_value')

    total_child_budget_value_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregations__total_child_budget_value')

    sector_category = CommaSeparatedCharFilter(
        lookup_type='in',
        name='activitysector__sector__category__code')

    participating_organisation = CommaSeparatedCharFilter(
        lookup_type='in',
        name='participating_organisations__ref')

    reporting_organisation = CommaSeparatedCharFilter(
        lookup_type='in',
        name='reporting_organisations__ref')

    xml_source_ref = CommaSeparatedCharFilter(
        lookup_type='in',
        name='xml_source_ref')

    activity_status = CommaSeparatedCharFilter(
        lookup_type='in',
        name='activity_status',
        distinct=True)

    hierarchy = CommaSeparatedCharFilter(
        lookup_type='in',
        name='hierarchy')

    related_activity_id = CommaSeparatedCharFilter(
        lookup_type='in',
        name='relatedactivity__ref_activity__id', distinct=True)

    related_activity_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='relatedactivity__type__code',
        distinct=True)

    related_activity_recipient_country = CommaSeparatedCharFilter(
        lookup_type='in',
        name='relatedactivity__ref_activity__recipient_country',
        distinct=True)

    related_activity_recipient_region = CommaSeparatedCharFilter(
        lookup_type='in',
        name='relatedactivity__ref_activity__recipient_region',
        distinct=True)

    related_activity_sector = CommaSeparatedCharFilter(
        lookup_type='in',
        name='relatedactivity__ref_activity__sector',
        distinct=True)

    related_activity_sector_category = CommaSeparatedCharFilter(
        lookup_type='in',
        name='relatedactivity__ref_activity__sector__category',
        distinct=True)

    budget_period_start = DateFilter(
        lookup_type='gte',
        name='budget__period_start')

    budget_period_end = DateFilter(
        lookup_type='lte',
        name='budget__period_end')

    budget_currency = CommaSeparatedCharFilter(
        lookup_type='in',
        name='budget__currency__code')

    transaction_provider_activity = CommaSeparatedCharFilter(
        lookup_type='in',
        name='transaction__provider_organisation__provider_activity_ref',
        distinct=True)

    transaction_date_year = NumberFilter(
        lookup_type='year',
        name='transaction__transaction_date',
        distinct=True)

    class Meta:
        model = Activity
        together_exclusive = [('budget_period_start', 'budget_period_end')]

class BudgetFilter(FilterSet):

    budget_period_start = DateFilter(
        lookup_type='gte',
        name='period_start')

    budget_period_end = DateFilter(
        lookup_type='lte',
        name='period_end')

    class Meta:
        model = Budget

class RelatedActivityFilter(FilterSet):

    related_activity_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='type__code')

    class Meta:
        model = RelatedActivity

class RelatedOrderingFilter(OrderingFilter):
    """
    Extends OrderingFilter to support ordering by fields in related models
    using the Django ORM __ notation

    Also provides support for mapping of fields,
    in remove_invalid_fields a mapping is maintained
    to make 'user-friendly' names possible
    """

    def is_valid_field(self, model, field):
        """
        Return true if the field exists within the model (or in the related
        model specified using the Django ORM __ notation)
        """
        components = field.split('__', 1)
        try:
            field, parent_model, direct, m2m = model._meta.get_field_by_name(components[0])

            if isinstance(field, OneToOneRel):
                return self.is_valid_field(field.related_model, components[1])

            # reverse relation
            if isinstance(field, ForeignObjectRel):
                return self.is_valid_field(field.model, components[1])

            # foreign key
            if field.rel and len(components) == 2:
                return self.is_valid_field(field.rel.to, components[1])
            return True
        except FieldDoesNotExist:
            return False

    def remove_invalid_fields(self, queryset, ordering, view):

        mapped_fields = {
            'title': 'title__narratives__content',
            'activity_budget_value': 'activity_aggregation__budget_value',
            'activity_plus_child_budget_value': 'activity_plus_child_aggregation__budget_value',
            'planned_start_date': 'planned_start',
            'actual_start_date': 'actual_start',
            'planned_end_date': 'planned_end',
            'actual_end_date': 'actual_end',
            'start_date': 'start_date',
            'end_date': 'end_date',
        }

        for i, term in enumerate(ordering):
            if term.lstrip('-') in mapped_fields:
                ordering[i] = ordering[i].replace(term.lstrip('-'), mapped_fields[term.lstrip('-')])

        return [term for term in ordering
                if self.is_valid_field(queryset.model, term.lstrip('-'))]
<commit_msg>participating_organisation_role filter<commit_after>import uuid

from django.db.models.fields import FieldDoesNotExist
from django.db.models.fields.related import ForeignObjectRel
from django.db.models.fields.related import OneToOneRel

from django_filters import Filter, FilterSet, NumberFilter, DateFilter
from rest_framework.filters import OrderingFilter

from api.generics.filters import CommaSeparatedCharFilter
from iati.models import Activity, Budget, RelatedActivity

class CommaSeparatedDateRangeFilter(Filter):

    def filter(self, qs, value):

        if value in ([], (), {}, None, ''):
            return qs

        value = value.split(',')

        return super(CommaSeparatedCharFilter, self).filter(qs, value)

class TogetherFilter(Filter):
    """
    Used with TogetherFilterSet, always gets called regardless of GET args
    """

    def __init__(self, filters=None, values=None, **kwargs):
        self.filter_classes = filters
        self.values = values

        super(TogetherFilter, self).__init__(**kwargs)

    def filter(self, qs, values):
        if self.filter_classes:
            filters = { "%s__%s" % (c[0].name, c[0].lookup_type) : c[1] for c in zip(self.filter_classes, values)}
            qs = qs.filter(**filters).distinct()

            return qs

class TogetherFilterSet(FilterSet):
    def __init__(self, data=None, queryset=None, prefix=None, strict=None):
        """
        Adds a together_exclusive meta option that selects fields that have to
        be called in the same django filter() call when both present
        """

        meta = getattr(self, 'Meta', None)

        # fields that must be filtered in the same filter call
        self.together_exclusive = getattr(meta, 'together_exclusive', None)

        data = data.copy()

        for filterlist in self.together_exclusive:
            if set(filterlist).issubset(data.keys()):

                filter_values = [data.pop(filteritem)[0] for filteritem in filterlist]
                filter_classes = [self.declared_filters.get(filteritem, None) for filteritem in filterlist]

                uid = uuid.uuid4()

                self.base_filters[uid] = TogetherFilter(filters=filter_classes)
                data.appendlist(uid, filter_values)

        super(FilterSet, self).__init__(data, queryset, prefix, strict)

class ActivityFilter(TogetherFilterSet):

    activity_id = CommaSeparatedCharFilter(
        name='id',
        lookup_type='in')

    activity_scope = CommaSeparatedCharFilter(
        name='scope__code',
        lookup_type='in')

    recipient_country = CommaSeparatedCharFilter(
        lookup_type='in',
        name='recipient_country')

    recipient_region = CommaSeparatedCharFilter(
        lookup_type='in',
        name='recipient_region')

    recipient_region_not_in = CommaSeparatedCharFilter(
        lookup_type='in',
        name='recipient_region',
        exclude=True)

    planned_start_date_lte = DateFilter(
        lookup_type='lte',
        name='planned_start')

    planned_start_date_gte = DateFilter(
        lookup_type='gte',
        name='planned_start')

    actual_start_date_lte = DateFilter(
        lookup_type='lte',
        name='actual_start')

    actual_start_date_gte = DateFilter(
        lookup_type='gte',
        name='actual_start')

    planned_end_date_lte = DateFilter(
        lookup_type='lte',
        name='planned_end')

    planned_end_date_gte = DateFilter(
        lookup_type='gte',
        name='planned_end')

    actual_end_date_lte = DateFilter(
        lookup_type='lte',
        name='actual_end')

    actual_end_date_gte = DateFilter(
        lookup_type='gte',
        name='actual_end')

    sector = CommaSeparatedCharFilter(
        lookup_type='in',
        name='sector')

    total_budget_value_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregations__total_budget_value')

    total_budget_value_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregations__total_budget_value')

    total_child_budget_value_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregations__total_child_budget_value')

    total_child_budget_value_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregations__total_child_budget_value')

    sector_category = CommaSeparatedCharFilter(
        lookup_type='in',
        name='activitysector__sector__category__code')

    participating_organisation = CommaSeparatedCharFilter(
        lookup_type='in',
        name='participating_organisations__ref')

    participating_organisation_role = CommaSeparatedCharFilter(
        lookup_type='in',
        name='participating_organisations__role__code')

    reporting_organisation = CommaSeparatedCharFilter(
        lookup_type='in',
        name='reporting_organisations__ref')

    xml_source_ref = CommaSeparatedCharFilter(
        lookup_type='in',
        name='xml_source_ref')

    activity_status = CommaSeparatedCharFilter(
        lookup_type='in',
        name='activity_status',
        distinct=True)

    hierarchy = CommaSeparatedCharFilter(
        lookup_type='in',
        name='hierarchy')

    related_activity_id = CommaSeparatedCharFilter(
        lookup_type='in',
        name='relatedactivity__ref_activity__id', distinct=True)

    related_activity_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='relatedactivity__type__code',
        distinct=True)

    related_activity_recipient_country = CommaSeparatedCharFilter(
        lookup_type='in',
        name='relatedactivity__ref_activity__recipient_country',
        distinct=True)

    related_activity_recipient_region = CommaSeparatedCharFilter(
        lookup_type='in',
        name='relatedactivity__ref_activity__recipient_region',
        distinct=True)

    related_activity_sector = CommaSeparatedCharFilter(
        lookup_type='in',
        name='relatedactivity__ref_activity__sector',
        distinct=True)

    related_activity_sector_category = CommaSeparatedCharFilter(
        lookup_type='in',
        name='relatedactivity__ref_activity__sector__category',
        distinct=True)

    budget_period_start = DateFilter(
        lookup_type='gte',
        name='budget__period_start')

    budget_period_end = DateFilter(
        lookup_type='lte',
        name='budget__period_end')

    budget_currency = CommaSeparatedCharFilter(
        lookup_type='in',
        name='budget__currency__code')

    transaction_provider_activity = CommaSeparatedCharFilter(
        lookup_type='in',
        name='transaction__provider_organisation__provider_activity_ref',
        distinct=True)

    transaction_date_year = NumberFilter(
        lookup_type='year',
        name='transaction__transaction_date',
        distinct=True)

    class Meta:
        model = Activity
        together_exclusive = [('budget_period_start', 'budget_period_end')]

class BudgetFilter(FilterSet):

    budget_period_start = DateFilter(
        lookup_type='gte',
        name='period_start')

    budget_period_end = DateFilter(
        lookup_type='lte',
        name='period_end')

    class Meta:
        model = Budget

class RelatedActivityFilter(FilterSet):

    related_activity_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='type__code')

    class Meta:
        model = RelatedActivity

class RelatedOrderingFilter(OrderingFilter):
    """
    Extends OrderingFilter to support ordering by fields in related models
    using the Django ORM __ notation

    Also provides support for mapping of fields,
    in remove_invalid_fields a mapping is maintained
    to make 'user-friendly' names possible
    """

    def is_valid_field(self, model, field):
        """
        Return true if the field exists within the model (or in the related
        model specified using the Django ORM __ notation)
        """
        components = field.split('__', 1)
        try:
            field, parent_model, direct, m2m = model._meta.get_field_by_name(components[0])

            if isinstance(field, OneToOneRel):
                return self.is_valid_field(field.related_model, components[1])

            # reverse relation
            if isinstance(field, ForeignObjectRel):
                return self.is_valid_field(field.model, components[1])

            # foreign key
            if field.rel and len(components) == 2:
                return self.is_valid_field(field.rel.to, components[1])
            return True
        except FieldDoesNotExist:
            return False

    def remove_invalid_fields(self, queryset, ordering, view):

        mapped_fields = {
            'title': 'title__narratives__content',
            'activity_budget_value': 'activity_aggregation__budget_value',
            'activity_plus_child_budget_value': 'activity_plus_child_aggregation__budget_value',
            'planned_start_date': 'planned_start',
            'actual_start_date': 'actual_start',
            'planned_end_date': 'planned_end',
            'actual_end_date': 'actual_end',
            'start_date': 'start_date',
            'end_date': 'end_date',
        }

        for i, term in enumerate(ordering):
            if term.lstrip('-') in mapped_fields:
                ordering[i] = ordering[i].replace(term.lstrip('-'), mapped_fields[term.lstrip('-')])

        return [term for term in ordering
                if self.is_valid_field(queryset.model, term.lstrip('-'))]
<|endoftext|>


<commit_before># Copyright (c) 2022, DjaoDjin inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""
PEP 386-compliant version number for the extended_templates django app.
"""

__version__ = '0.3.0'
<commit_msg>bumps version number to 0.3.1-dev<commit_after># Copyright (c) 2022, DjaoDjin inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""
PEP 386-compliant version number for the extended_templates django app.
"""

__version__ = '0.3.1-dev'
<|endoftext|>


<commit_before>import os, sys
import re
import click
from textx.lang import get_language
from textx.exceptions import TextXError
from txtools import copy_scaffolding, render_scaffolding

@click.command()
@click.option('-t', '--project-type', default='app',
              type=click.Choice(['lang', 'gen', 'app']))
@click.option('-l', '--language',
              help='Language for generator projects.')
@click.argument('project_name')
def startproject(project_type, language, project_name):
    """
    Generates scaffolding for textX projects.
    """
    if project_type == 'gen':
        if not language:
            click.echo('Language must be specified for generator'
                       ' projects. See -l switch.')
            sys.exit(1)
        else:
            try:
                get_language(language)
            except TextXError as e:
                click.echo(e)
                click.echo('Try to install it with "pip install textx-lang-{}"'
                           .format(language))
                click.echo('Use "textx list-langs" to see all registered '
                           'languages.')
                sys.exit(1)

    if not _check_project_name(project_name):
        click.echo('Project name may contain letters, digits and dashes.')
        sys.exit(1)

    if project_type == 'lang':
        package_name = _project_name_to_package_name(project_name)
        full_project_name = 'textx-{}-{}'.format(project_type, project_name)
    elif project_type == 'gen':
        package_name = "{}_{}".format(
            language, _project_name_to_package_name(project_name))
        full_project_name = 'textx-{}-{}-{}'.format(project_type,
                                                    language,
                                                    project_name)
    else:
        package_name = _project_name_to_package_name(project_name)
        full_project_name = project_name

    project_type_full = {
        'lang': 'language',
        'gen': 'generator'
    }.get(project_type, project_type)

    click.echo('Generating {} project "{}" in folder "{}".'
               .format(project_type_full, project_name, full_project_name))

    context = {
        'project_name': project_name,
        'full_project_name': full_project_name,
        'package_name': package_name,
        'project_type': project_type,
        'language': language
    }

    _generate_project(context)

    click.echo('Done.')

def _generate_project(context):
    """
    Generates project scaffolding.
    """

    current_dir = os.curdir
    project_folder = os.path.join(current_dir, context['full_project_name'])
    project_type = context['project_type']
    package_name = context['package_name']

    os.makedirs(project_folder)

    # Copy generic part defined in "all" folder
    copy_scaffolding("all", project_folder, package_name)

    # Render generic templates "all" folder
    render_scaffolding("all", project_folder, package_name, context)

    # Copy project type specific
    copy_scaffolding(project_type, project_folder, package_name)

    # Render project type specific
    render_scaffolding(project_type, project_folder, package_name, context)

def _check_project_name(project_name):
    return re.match('^[a-zA-Z][a-zA-Z0-9-]*[a-zA-Z0-9]$', project_name)

def _project_name_to_package_name(project_name):
    """
    Package name contains only lowercase letters and digits.
    """
    return project_name.lower().replace('-', '')
<commit_msg>Handling exception in startproject<commit_after>import os
import sys
import re
import click
from textx.lang import get_language
from textx.exceptions import TextXError
from txtools import copy_scaffolding, render_scaffolding
from textx.exceptions import TextXToolsException

@click.command()
@click.option('-t', '--project-type', default='app',
              type=click.Choice(['lang', 'gen', 'app']))
@click.option('-l', '--language',
              help='Language for generator projects.')
@click.argument('project_name')
def startproject(project_type, language, project_name):
    """
    Generates scaffolding for textX projects.
    """
    if project_type == 'gen':
        if not language:
            click.echo('Language must be specified for generator'
                       ' projects. See -l switch.')
            sys.exit(1)
        else:
            try:
                get_language(language)
            except TextXError as e:
                click.echo(e)
                click.echo('Try to install it with "pip install textx-lang-{}"'
                           .format(language))
                click.echo('Use "textx list-langs" to see all registered '
                           'languages.')
                sys.exit(1)

    if not _check_project_name(project_name):
        click.echo('Project name may contain letters, digits and dashes.')
        sys.exit(1)

    if project_type == 'lang':
        package_name = _project_name_to_package_name(project_name)
        full_project_name = 'textx-{}-{}'.format(project_type, project_name)
    elif project_type == 'gen':
        package_name = "{}_{}".format(
            language, _project_name_to_package_name(project_name))
        full_project_name = 'textx-{}-{}-{}'.format(project_type,
                                                    language,
                                                    project_name)
    else:
        package_name = _project_name_to_package_name(project_name)
        full_project_name = project_name

    project_type_full = {
        'lang': 'language',
        'gen': 'generator'
    }.get(project_type, project_type)

    click.echo('Generating {} project "{}" in folder "{}".'
               .format(project_type_full, project_name, full_project_name))

    context = {
        'project_name': project_name,
        'full_project_name': full_project_name,
        'package_name': package_name,
        'project_type': project_type,
        'language': language
    }

    try:
        _generate_project(context)
    except TextXToolsException as e:
        click.echo(e)
        sys.exit(1)

    click.echo('Done.')

def _generate_project(context):
    """
    Generates project scaffolding.
    """

    current_dir = os.curdir
    project_folder = os.path.join(current_dir, context['full_project_name'])
    project_type = context['project_type']
    package_name = context['package_name']

    try:
        os.makedirs(project_folder)
    except FileExistsError:
        raise TextXToolsException('Error: Project folder "{}" already exists.'
                                  .format(project_folder))

    # Copy generic part defined in "all" folder
    copy_scaffolding("all", project_folder, package_name)

    # Render generic templates "all" folder
    render_scaffolding("all", project_folder, package_name, context)

    # Copy project type specific
    copy_scaffolding(project_type, project_folder, package_name)

    # Render project type specific
    render_scaffolding(project_type, project_folder, package_name, context)

def _check_project_name(project_name):
    return re.match('^[a-zA-Z][a-zA-Z0-9-]*[a-zA-Z0-9]$', project_name)

def _project_name_to_package_name(project_name):
    """
    Package name contains only lowercase letters and digits.
    """
    return project_name.lower().replace('-', '')
<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys
from PyQt5.QtCore import Qt, QBasicTimer
from PyQt5.QtWidgets import QMainWindow, QWidget, QPushButton, QApplication, QVBoxLayout, QHBoxLayout
from PyQt5.QtGui import QPainter, QColor, QBrush, QPen

class Board(QWidget):

    def __init__(self, width=9, height=9, offset=0, size=10):
        super().__init__()
        self.width = width
        self.height = height
        self.offset = offset
        self.size   = size
        self.cells  = []
        self.pressEventEnabled = True

        for y in range(self.height):
            for x in range(self.width):
                self.cells.append(0)

        widgetSize = self.offset*2 + self.width*self.size
        self.setMinimumSize(widgetSize, widgetSize)

    def paintEvent(self, e):
        qp = QPainter()
        qp.begin(self)
        self.drawBoard(qp)
        qp.end()

    def drawBoard(self, qp):

        cellIndex = 0
        for y in range(self.height):
            for x in range(self.width):
                qp.setPen(QPen(Qt.gray, 1, Qt.SolidLine))
                if(self.cells[cellIndex] == 0):
                    qp.setBrush(QColor(255,250,250))
                else:
                    qp.setBrush(QColor(0,0,0))

                qp.drawRect(
                        self.offset + x*self.size,
                        self.offset + y*self.size,
                        self.size, self.size)

                cellIndex += 1

    def clearCells(self):
        for i in range(len(self.cells)):
            self.cells[i] = 0
        self.update()

    def setCell(self, x, y, status):
        self.cells[(self.width*y) + x] = status
        self.update()

    def getCell(self, x, y):
        if x < 0 or y < 0 or x > self.width-1 or y > self.width-1:
            return 0
        else:
            return self.cells[(self.width*y) + x]

    def setCells(self, cells):
        self.cells=cells
        self.update()

    def invertCell(self, x, y):
        if self.cells[(self.width*y) + x] == 0:
            self.cells[(self.width*y) + x] = 1
        elif self.cells[(self.width*y) + x] == 1:
            self.cells[(self.width*y) + x] = 0

        self.update()

    def drawRectangles(self, qp):
        pen = QPen(Qt.gray, 1, Qt.SolidLine)
        qp.setPen(pen)
        qp.drawRect(1,1,10,10)

        col = QColor(0,0,0)
        col.setNamedColor('#d4d4d4')
        qp.setPen(col)

        qp.setBrush(QColor(200, 0,0))
        qp.drawRect(10,15,90,60)

        qp.setBrush(QColor(25, 0, 90, 200))
        qp.drawRect(130,15,90,60)

        qp.setBrush(QColor(25, 80, 90, 200))
        qp.drawRect(250,15,90,60)

    def mousePressEvent(self, e):
        if(self.pressEventEnabled):
            x = (e.x() - self.offset)/self.size
            y = (e.y() - self.offset)/self.size

            if x<0 or y<0 or x>self.width or y>self.height:
                return

            self.invertCell(int(x), int(y))

    def enablePressEvent(self):
        self.pressEventEnabled = True

    def disablePressEvent(self):
        self.pressEventEnabled = False

    def nextGeneration(self):
        newCells = []
        for y in range(self.height):
            for x in range(self.width):
                neighbours = 0
                neighbours += self.getCell(x-1, y-1)
                neighbours += self.getCell(x, y-1)
                neighbours += self.getCell(x+1, y-1)
                neighbours += self.getCell(x-1, y)
                neighbours += self.getCell(x+1, y)
                neighbours += self.getCell(x-1, y+1)
                neighbours += self.getCell(x, y+1)
                neighbours += self.getCell(x+1, y+1)

                if self.getCell(x, y) == 1:
                    if neighbours < 2:
                        newCells.append(0)
                    elif neighbours == 2 or neighbours == 3:
                        newCells.append(1)
                    else:
                        newCells.append(0)
                else:
                    if neighbours == 3:
                        newCells.append(1)
                    else:
                        newCells.append(0)

        self.setCells(newCells)

class GuiMain(QWidget):

    Speed = 300

    def __init__(self, width=9, height=9, offset=0, size=10):
        super().__init__()

        self.timer = QBasicTimer()

        self.board = Board(width, height, offset, size)

        self.stepForwardBtn = QPushButton('>')
        self.stepForwardBtn.clicked.connect(self.stepForward)

        self.moveForwardBtn = QPushButton('>>')
        self.moveForwardBtn.clicked.connect(self.moveForward)

        self.pauseBtn = QPushButton('Pause')
        self.pauseBtn.clicked.connect(self.pause)
        self.pauseBtn.setEnabled(False)

        self.clearBtn = QPushButton('Clear')
        self.clearBtn.clicked.connect(self.clear)

        hbox = QHBoxLayout()
        hbox.addStretch(1)
        hbox.addWidget(self.stepForwardBtn)
        hbox.addWidget(self.moveForwardBtn)
        hbox.addWidget(self.pauseBtn)
        hbox.addWidget(self.clearBtn)

        vbox = QVBoxLayout()
        vbox.addWidget(self.board)
        vbox.addStretch(1)
        vbox.addLayout(hbox)

        self.setWindowTitle('Game of Life')

        self.setLayout(vbox)
        self.show()

    def stepForward(self):
        self.board.nextGeneration()

    def moveForward(self):
        self.stepForwardBtn.setEnabled(False)
        self.pauseBtn.setEnabled(True)
        self.clearBtn.setEnabled(False)
        self.moveForwardBtn.setEnabled(False)
        self.board.disablePressEvent()
        self.timer.start(GuiMain.Speed, self)

    def clear(self):
        self.board.clearCells()

    def pause(self):
        self.stepForwardBtn.setEnabled(True)
        self.pauseBtn.setEnabled(False)
        self.clearBtn.setEnabled(True)
        self.moveForwardBtn.setEnabled(True)
        self.board.enablePressEvent()
        self.timer.stop()

    def timerEvent(self, event):
        if event.timerId() == self.timer.timerId():
            self.stepForward()
        else:
            super(GuiMain, self).timerEvent(event)

if __name__=='__main__':
    app = QApplication(sys.argv)
    myGui = GuiMain(50, 50, 10, 10)
    sys.exit(app.exec_())

<commit_msg>Delete unused code<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys
from PyQt5.QtCore import Qt, QBasicTimer
from PyQt5.QtWidgets import QMainWindow, QWidget, QPushButton, QApplication, QVBoxLayout, QHBoxLayout
from PyQt5.QtGui import QPainter, QColor, QBrush, QPen

class Board(QWidget):

    def __init__(self, width=9, height=9, offset=0, size=10):
        super().__init__()
        self.width = width
        self.height = height
        self.offset = offset
        self.size   = size
        self.cells  = []
        self.pressEventEnabled = True

        for y in range(self.height):
            for x in range(self.width):
                self.cells.append(0)

        widgetSize = self.offset*2 + self.width*self.size
        self.setMinimumSize(widgetSize, widgetSize)

    def paintEvent(self, e):
        qp = QPainter()
        qp.begin(self)
        self.drawBoard(qp)
        qp.end()

    def drawBoard(self, qp):

        cellIndex = 0
        for y in range(self.height):
            for x in range(self.width):
                qp.setPen(QPen(Qt.gray, 1, Qt.SolidLine))
                if(self.cells[cellIndex] == 0):
                    qp.setBrush(QColor(255,250,250))
                else:
                    qp.setBrush(QColor(0,0,0))

                qp.drawRect(
                        self.offset + x*self.size,
                        self.offset + y*self.size,
                        self.size, self.size)

                cellIndex += 1

    def clearCells(self):
        for i in range(len(self.cells)):
            self.cells[i] = 0
        self.update()

    def setCell(self, x, y, status):
        self.cells[(self.width*y) + x] = status
        self.update()

    def getCell(self, x, y):
        if x < 0 or y < 0 or x > self.width-1 or y > self.width-1:
            return 0
        else:
            return self.cells[(self.width*y) + x]

    def setCells(self, cells):
        self.cells=cells
        self.update()

    def invertCell(self, x, y):
        if self.cells[(self.width*y) + x] == 0:
            self.cells[(self.width*y) + x] = 1
        elif self.cells[(self.width*y) + x] == 1:
            self.cells[(self.width*y) + x] = 0

        self.update()

    def mousePressEvent(self, e):
        if(self.pressEventEnabled):
            x = (e.x() - self.offset)/self.size
            y = (e.y() - self.offset)/self.size

            if x<0 or y<0 or x>self.width or y>self.height:
                return

            self.invertCell(int(x), int(y))

    def enablePressEvent(self):
        self.pressEventEnabled = True

    def disablePressEvent(self):
        self.pressEventEnabled = False

    def nextGeneration(self):
        newCells = []
        for y in range(self.height):
            for x in range(self.width):
                neighbours = 0
                neighbours += self.getCell(x-1, y-1)
                neighbours += self.getCell(x, y-1)
                neighbours += self.getCell(x+1, y-1)
                neighbours += self.getCell(x-1, y)
                neighbours += self.getCell(x+1, y)
                neighbours += self.getCell(x-1, y+1)
                neighbours += self.getCell(x, y+1)
                neighbours += self.getCell(x+1, y+1)

                if self.getCell(x, y) == 1:
                    if neighbours < 2:
                        newCells.append(0)
                    elif neighbours == 2 or neighbours == 3:
                        newCells.append(1)
                    else:
                        newCells.append(0)
                else:
                    if neighbours == 3:
                        newCells.append(1)
                    else:
                        newCells.append(0)

        self.setCells(newCells)

class GuiMain(QWidget):

    Speed = 300

    def __init__(self, width=9, height=9, offset=0, size=10):
        super().__init__()

        self.timer = QBasicTimer()

        self.board = Board(width, height, offset, size)

        self.stepForwardBtn = QPushButton('>')
        self.stepForwardBtn.clicked.connect(self.stepForward)

        self.moveForwardBtn = QPushButton('>>')
        self.moveForwardBtn.clicked.connect(self.moveForward)

        self.pauseBtn = QPushButton('Pause')
        self.pauseBtn.clicked.connect(self.pause)
        self.pauseBtn.setEnabled(False)

        self.clearBtn = QPushButton('Clear')
        self.clearBtn.clicked.connect(self.clear)

        hbox = QHBoxLayout()
        hbox.addStretch(1)
        hbox.addWidget(self.stepForwardBtn)
        hbox.addWidget(self.moveForwardBtn)
        hbox.addWidget(self.pauseBtn)
        hbox.addWidget(self.clearBtn)

        vbox = QVBoxLayout()
        vbox.addWidget(self.board)
        vbox.addStretch(1)
        vbox.addLayout(hbox)

        self.setWindowTitle('Game of Life')

        self.setLayout(vbox)
        self.show()

    def stepForward(self):
        self.board.nextGeneration()

    def moveForward(self):
        self.stepForwardBtn.setEnabled(False)
        self.pauseBtn.setEnabled(True)
        self.clearBtn.setEnabled(False)
        self.moveForwardBtn.setEnabled(False)
        self.board.disablePressEvent()
        self.timer.start(GuiMain.Speed, self)

    def clear(self):
        self.board.clearCells()

    def pause(self):
        self.stepForwardBtn.setEnabled(True)
        self.pauseBtn.setEnabled(False)
        self.clearBtn.setEnabled(True)
        self.moveForwardBtn.setEnabled(True)
        self.board.enablePressEvent()
        self.timer.stop()

    def timerEvent(self, event):
        if event.timerId() == self.timer.timerId():
            self.stepForward()
        else:
            super(GuiMain, self).timerEvent(event)

if __name__=='__main__':
    app = QApplication(sys.argv)
    myGui = GuiMain(50, 50, 10, 10)
    sys.exit(app.exec_())

<|endoftext|>


<commit_before># Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the LICENSE file in
# the root directory of this source tree. An additional grant of patent rights
# can be found in the PATENTS file in the same directory.

import math

import numpy as np
import torch

from typing import Dict, List, Tuple

from . import FairseqDataset, data_utils

from fairseq.data import Dictionary
from fairseq.data.block_pair_dataset import BlockPairDataset
from fairseq.data.token_block_dataset import TokenBlockDataset
from fairseq.data.concat_dataset import ConcatDataset

class MaskedLMDataset(FairseqDataset):
    """
    A wrapper Dataset for masked language modelling. The dataset
    wraps around TokenBlockDataset or BlockedPairDataset and creates a batch
    where the input blocks are masked according to the specified masking
    probability. Additionally the batch can also contain sentence level targets
    if this is specified.

    Args:
        dataset: Dataset which generates blocks of data. Only BlockPairDataset
            and TokenBlockDataset are supported.
        sizes: Sentence lengths
        vocab: Dictionary with the vocabulary and special tokens.
        pad_idx: Id of padding token in dictionary
        mask_idx: Id of mask token in dictionary
        classif_token_idx: Id of classification token in dictionary. This is the
            token associated with the sentence embedding (Eg: CLS for BERT)
        sep_token_idx: Id of separator token in dictionary
            (Eg: SEP in BERT)
        seed: Seed for random number generator for reproducibility.
        shuffle: Shuffle the elements before batching.
        has_pairs: Specifies whether the underlying dataset
            generates a pair of blocks along with a sentence_target or not.
            Setting it to True assumes that the underlying dataset generates a
            label for the pair of sentences which is surfaced as
            sentence_target. The default value assumes a single block with no
            sentence target.
        segment_id: An optional segment id for filling in the segment labels
            when we are in the single block setting (Eg: XLM). Default is 0.
        masking_ratio: specifies what percentage of the blocks should be masked.
        masking_prob: specifies the probability of a given token being
            replaced with the "MASK" token.
        random_token_prob: specifies the probability of a given token being
            replaced by a random token from the vocabulary.
    """

    def __init__(
            self,
            dataset: FairseqDataset,
            sizes: np.ndarray,
            vocab: Dictionary,
            pad_idx: int,
            mask_idx: int,
            classif_token_idx: int,
            sep_token_idx: int,
            seed: int = 1,
            shuffle: bool = True,
            has_pairs: bool = True,
            segment_id: int = 0,
            masking_ratio: float = 0.15,
            masking_prob: float = 0.8,
            random_token_prob: float = 0.1
    ):
        # Make sure the input datasets are the ones supported
        assert (
            isinstance(dataset, TokenBlockDataset) or
            isinstance(dataset, BlockPairDataset) or
            isinstance(dataset, ConcatDataset)
        ), "MaskedLMDataset only wraps TokenBlockDataset or BlockPairDataset or " \
           "ConcatDataset"

        self.dataset = dataset
        self.sizes = np.array(sizes)
        self.vocab = vocab
        self.pad_idx = pad_idx
        self.mask_idx = mask_idx
        self.classif_token_idx = classif_token_idx
        self.sep_token_idx = sep_token_idx
        self.shuffle = shuffle
        self.seed = seed
        self.has_pairs = has_pairs
        self.segment_id = segment_id
        self.masking_ratio = masking_ratio
        self.masking_prob = masking_prob
        self.random_token_prob = random_token_prob

        # If we have only one block then sizes needs to be updated to include
        # the classification token
        if not has_pairs:
            self.sizes = self.sizes + 1

    def __getitem__(
            self,
            index: int
    ):
        # if has_pairs, then expect 2 blocks and a sentence target
        if self.has_pairs:
            (block_one, block_two, sentence_target) = self.dataset[index]
        else:
            block_one = self.dataset[index]

        return {
            "id": index,
            "block_one": block_one,
            "block_two": block_two if self.has_pairs else None,
            "sentence_target": sentence_target if self.has_pairs else None,
        }

    def __len__(self):
        return len(self.dataset)

    def _mask_block(
            self,
            sentence: np.ndarray,
            mask_idx: int,
            pad_idx: int,
            dictionary_token_range: Tuple,
    ):
        """
        Mask tokens for Masked Language Model training
        Samples mask_ratio tokens that will be predicted by LM.

        Note:This function may not be efficient enough since we had multiple
        conversions between np and torch, we can replace them with torch
        operators later.

        Args:
            sentence: 1d tensor to be masked
            mask_idx: index to use for masking the sentence
            pad_idx: index to use for masking the target for tokens we aren't
                predicting
            dictionary_token_range: range of indices in dictionary which can
                be used for random word replacement
                (e.g. without special characters)
        Return:
            masked_sent: masked sentence
            target: target with words which we are not predicting replaced
                by pad_idx
        """
        masked_sent = np.copy(sentence)
        sent_length = len(sentence)
        mask_num = math.ceil(sent_length * self.masking_ratio)
        mask = np.random.choice(sent_length, mask_num)
        target = np.copy(sentence)

        for i in range(sent_length):
            if i in mask:
                rand = np.random.random()

                # replace with mask if probability is less than masking_prob
                # (Eg: 0.8)
                if rand < self.masking_prob:
                    masked_sent[i] = mask_idx

                # replace with random token if probability is less than
                # masking_prob + random_token_prob (Eg: 0.9)
                elif rand < (self.masking_ratio + self.random_token_prob):
                    # sample random token from dictionary
                    masked_sent[i] = (
                        np.random.randint(
                            dictionary_token_range[0], dictionary_token_range[1]
                        )
                    )
            else:
                target[i] = pad_idx

        return masked_sent, target

    def _collate(
            self,
            samples: List[Dict],
            pad_idx: int,
            eos_idx: int
    ):
        """
        Does the heavy lifting for creating a batch from the input list of
        examples. The logic is as follows:
            1. Mask the input blocks. In case has_pair is True then we have 2
               blocks to mask.
            2. Prepend the first masked block tensor with the special token
               used as sentence embedding. Eg: CLS in BERT. This happens
               irrespective of the value of has_pair.
            3. If has_pair is True, then append the first masked block with the
               special separator token (eg: SEP for BERT) and compute segment
               label accordingly. In this case, also append the second masked
               block with this special separator token and compute its segment
               label.
            4. For the targets tensor, prepend and append with padding index
               accordingly.
            5. Concatenate all tensors.
        """
        if len(samples) == 0:
            return {}
        # To ensure determinism, we reset the state of the PRNG after every
        # batch based on the seed and the first id of the batch. This ensures
        # that across epochs we get the same mask for the same example. This
        # is needed for reproducibility and is how BERT does masking
        # TODO: Can we add deteminism without this constraint?
        with data_utils.numpy_seed(self.seed + samples[0]["id"]):
            for s in samples:

                # token range is needed for replacing with random token during
                # masking
                token_range = (self.vocab.nspecial, len(self.vocab))

                # mask according to specified probabilities.
                masked_blk_one, masked_tgt_one = self._mask_block(
                    s["block_one"], self.mask_idx, self.pad_idx, token_range,
                )

                tokens = np.concatenate([
                    [self.classif_token_idx], masked_blk_one
                ])
                targets = np.concatenate([[self.pad_idx], masked_tgt_one])
                segments = np.ones(len(tokens)) * self.segment_id

                # if has_pairs is True then we need to add the SEP token to both
                # the blocks after masking and re-compute segments based on the new
                # lengths.
                if self.has_pairs:
                    tokens_one = np.concatenate([tokens, [self.sep_token_idx]])
                    targets_one = np.concatenate([targets, [self.pad_idx]])

                    masked_blk_two, masked_tgt_two = self._mask_block(
                        s["block_two"], self.mask_idx, self.pad_idx, token_range)
                    tokens_two = np.concatenate(
                        [masked_blk_two, [self.sep_token_idx]])
                    targets_two = np.concatenate([masked_tgt_two, [self.pad_idx]])

                    # block + 1 sep + 1 special (CLS)
                    segments_one = np.zeros(len(tokens_one))
                    # block + 1 sep
                    segments_two = np.ones(len(tokens_two))

                    tokens = np.concatenate([tokens_one, tokens_two])
                    targets = np.concatenate([targets_one, targets_two])
                    segments = np.concatenate([segments_one, segments_two])

                s["source"] = torch.LongTensor(tokens)
                s["segment_labels"] = torch.LongTensor(segments)
                s["lm_target"] = torch.LongTensor(targets)

        def merge(key):
            return data_utils.collate_tokens(
                [s[key] for s in samples], pad_idx, eos_idx, left_pad=False
            )
        return {
            "id": torch.LongTensor([s["id"] for s in samples]),
            "ntokens": sum(len(s["source"]) for s in samples),
            "net_input": {
                "src_tokens": merge("source"),
                "segment_labels": merge("segment_labels"),
            },
            "lm_target": merge("lm_target"),
            "sentence_target": torch.LongTensor(
                [s["sentence_target"] for s in samples]
            ) if self.has_pairs else None,
            "nsentences": len(samples),
        }

    def collater(
            self,
            samples: List[Dict]
    ):
        """Merge a list of samples to form a mini-batch.

        Args:
            samples (List[dict]): samples to collate

        Returns:
            dict: a mini-batch of data
        """
        return self._collate(samples, self.vocab.pad(), self.vocab.eos())

    def num_tokens(
            self,
            index: int
    ):
        """
        Return the number of tokens in a sample. This value is used to
        enforce max-tokens during batching.
        """
        return self.sizes[index]

    def size(
            self,
            index: int
    ):
        """
        Return an example's size as a float or tuple. This value is used when
        filtering a dataset with max-positions.
        """
        return self.sizes[index]

    def ordered_indices(self):
        """
        Return an ordered list of indices. Batches will be constructed based
        on this order.
        """
        if self.shuffle:
            return np.random.permutation(len(self))
        else:
            order = [np.arange(len(self))]
            order.append(self.sizes)
            return np.lexsort(order)

    @property
    def supports_prefetch(self):
        return getattr(self.dataset, "supports_prefetch", False)

    def prefetch(self, indices):
        self.dataset.prefetch(indices)
<commit_msg>fix bug for masking (#752)<commit_after># Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the LICENSE file in
# the root directory of this source tree. An additional grant of patent rights
# can be found in the PATENTS file in the same directory.

import math

import numpy as np
import torch

from typing import Dict, List, Tuple

from . import FairseqDataset, data_utils

from fairseq.data import Dictionary
from fairseq.data.block_pair_dataset import BlockPairDataset
from fairseq.data.token_block_dataset import TokenBlockDataset
from fairseq.data.concat_dataset import ConcatDataset

class MaskedLMDataset(FairseqDataset):
    """
    A wrapper Dataset for masked language modelling. The dataset
    wraps around TokenBlockDataset or BlockedPairDataset and creates a batch
    where the input blocks are masked according to the specified masking
    probability. Additionally the batch can also contain sentence level targets
    if this is specified.

    Args:
        dataset: Dataset which generates blocks of data. Only BlockPairDataset
            and TokenBlockDataset are supported.
        sizes: Sentence lengths
        vocab: Dictionary with the vocabulary and special tokens.
        pad_idx: Id of padding token in dictionary
        mask_idx: Id of mask token in dictionary
        classif_token_idx: Id of classification token in dictionary. This is the
            token associated with the sentence embedding (Eg: CLS for BERT)
        sep_token_idx: Id of separator token in dictionary
            (Eg: SEP in BERT)
        seed: Seed for random number generator for reproducibility.
        shuffle: Shuffle the elements before batching.
        has_pairs: Specifies whether the underlying dataset
            generates a pair of blocks along with a sentence_target or not.
            Setting it to True assumes that the underlying dataset generates a
            label for the pair of sentences which is surfaced as
            sentence_target. The default value assumes a single block with no
            sentence target.
        segment_id: An optional segment id for filling in the segment labels
            when we are in the single block setting (Eg: XLM). Default is 0.
        masking_ratio: specifies what percentage of the blocks should be masked.
        masking_prob: specifies the probability of a given token being
            replaced with the "MASK" token.
        random_token_prob: specifies the probability of a given token being
            replaced by a random token from the vocabulary.
    """

    def __init__(
            self,
            dataset: FairseqDataset,
            sizes: np.ndarray,
            vocab: Dictionary,
            pad_idx: int,
            mask_idx: int,
            classif_token_idx: int,
            sep_token_idx: int,
            seed: int = 1,
            shuffle: bool = True,
            has_pairs: bool = True,
            segment_id: int = 0,
            masking_ratio: float = 0.15,
            masking_prob: float = 0.8,
            random_token_prob: float = 0.1
    ):
        # Make sure the input datasets are the ones supported
        assert (
            isinstance(dataset, TokenBlockDataset) or
            isinstance(dataset, BlockPairDataset) or
            isinstance(dataset, ConcatDataset)
        ), "MaskedLMDataset only wraps TokenBlockDataset or BlockPairDataset or " \
           "ConcatDataset"

        self.dataset = dataset
        self.sizes = np.array(sizes)
        self.vocab = vocab
        self.pad_idx = pad_idx
        self.mask_idx = mask_idx
        self.classif_token_idx = classif_token_idx
        self.sep_token_idx = sep_token_idx
        self.shuffle = shuffle
        self.seed = seed
        self.has_pairs = has_pairs
        self.segment_id = segment_id
        self.masking_ratio = masking_ratio
        self.masking_prob = masking_prob
        self.random_token_prob = random_token_prob

        # If we have only one block then sizes needs to be updated to include
        # the classification token
        if not has_pairs:
            self.sizes = self.sizes + 1

    def __getitem__(
            self,
            index: int
    ):
        # if has_pairs, then expect 2 blocks and a sentence target
        if self.has_pairs:
            (block_one, block_two, sentence_target) = self.dataset[index]
        else:
            block_one = self.dataset[index]

        return {
            "id": index,
            "block_one": block_one,
            "block_two": block_two if self.has_pairs else None,
            "sentence_target": sentence_target if self.has_pairs else None,
        }

    def __len__(self):
        return len(self.dataset)

    def _mask_block(
            self,
            sentence: np.ndarray,
            mask_idx: int,
            pad_idx: int,
            dictionary_token_range: Tuple,
    ):
        """
        Mask tokens for Masked Language Model training
        Samples mask_ratio tokens that will be predicted by LM.

        Note:This function may not be efficient enough since we had multiple
        conversions between np and torch, we can replace them with torch
        operators later.

        Args:
            sentence: 1d tensor to be masked
            mask_idx: index to use for masking the sentence
            pad_idx: index to use for masking the target for tokens we aren't
                predicting
            dictionary_token_range: range of indices in dictionary which can
                be used for random word replacement
                (e.g. without special characters)
        Return:
            masked_sent: masked sentence
            target: target with words which we are not predicting replaced
                by pad_idx
        """
        masked_sent = np.copy(sentence)
        sent_length = len(sentence)
        mask_num = math.ceil(sent_length * self.masking_ratio)
        mask = np.random.choice(sent_length, mask_num, replace=False)
        target = np.copy(sentence)

        for i in range(sent_length):
            if i in mask:
                rand = np.random.random()

                # replace with mask if probability is less than masking_prob
                # (Eg: 0.8)
                if rand < self.masking_prob:
                    masked_sent[i] = mask_idx

                # replace with random token if probability is less than
                # masking_prob + random_token_prob (Eg: 0.9)
                elif rand < (self.masking_ratio + self.random_token_prob):
                    # sample random token from dictionary
                    masked_sent[i] = (
                        np.random.randint(
                            dictionary_token_range[0], dictionary_token_range[1]
                        )
                    )
            else:
                target[i] = pad_idx

        return masked_sent, target

    def _collate(
            self,
            samples: List[Dict],
            pad_idx: int,
            eos_idx: int
    ):
        """
        Does the heavy lifting for creating a batch from the input list of
        examples. The logic is as follows:
            1. Mask the input blocks. In case has_pair is True then we have 2
               blocks to mask.
            2. Prepend the first masked block tensor with the special token
               used as sentence embedding. Eg: CLS in BERT. This happens
               irrespective of the value of has_pair.
            3. If has_pair is True, then append the first masked block with the
               special separator token (eg: SEP for BERT) and compute segment
               label accordingly. In this case, also append the second masked
               block with this special separator token and compute its segment
               label.
            4. For the targets tensor, prepend and append with padding index
               accordingly.
            5. Concatenate all tensors.
        """
        if len(samples) == 0:
            return {}
        # To ensure determinism, we reset the state of the PRNG after every
        # batch based on the seed and the first id of the batch. This ensures
        # that across epochs we get the same mask for the same example. This
        # is needed for reproducibility and is how BERT does masking
        # TODO: Can we add deteminism without this constraint?
        with data_utils.numpy_seed(self.seed + samples[0]["id"]):
            for s in samples:

                # token range is needed for replacing with random token during
                # masking
                token_range = (self.vocab.nspecial, len(self.vocab))

                # mask according to specified probabilities.
                masked_blk_one, masked_tgt_one = self._mask_block(
                    s["block_one"], self.mask_idx, self.pad_idx, token_range,
                )

                tokens = np.concatenate([
                    [self.classif_token_idx], masked_blk_one
                ])
                targets = np.concatenate([[self.pad_idx], masked_tgt_one])
                segments = np.ones(len(tokens)) * self.segment_id

                # if has_pairs is True then we need to add the SEP token to both
                # the blocks after masking and re-compute segments based on the new
                # lengths.
                if self.has_pairs:
                    tokens_one = np.concatenate([tokens, [self.sep_token_idx]])
                    targets_one = np.concatenate([targets, [self.pad_idx]])

                    masked_blk_two, masked_tgt_two = self._mask_block(
                        s["block_two"], self.mask_idx, self.pad_idx, token_range)
                    tokens_two = np.concatenate(
                        [masked_blk_two, [self.sep_token_idx]])
                    targets_two = np.concatenate([masked_tgt_two, [self.pad_idx]])

                    # block + 1 sep + 1 special (CLS)
                    segments_one = np.zeros(len(tokens_one))
                    # block + 1 sep
                    segments_two = np.ones(len(tokens_two))

                    tokens = np.concatenate([tokens_one, tokens_two])
                    targets = np.concatenate([targets_one, targets_two])
                    segments = np.concatenate([segments_one, segments_two])

                s["source"] = torch.LongTensor(tokens)
                s["segment_labels"] = torch.LongTensor(segments)
                s["lm_target"] = torch.LongTensor(targets)

        def merge(key):
            return data_utils.collate_tokens(
                [s[key] for s in samples], pad_idx, eos_idx, left_pad=False
            )
        return {
            "id": torch.LongTensor([s["id"] for s in samples]),
            "ntokens": sum(len(s["source"]) for s in samples),
            "net_input": {
                "src_tokens": merge("source"),
                "segment_labels": merge("segment_labels"),
            },
            "lm_target": merge("lm_target"),
            "sentence_target": torch.LongTensor(
                [s["sentence_target"] for s in samples]
            ) if self.has_pairs else None,
            "nsentences": len(samples),
        }

    def collater(
            self,
            samples: List[Dict]
    ):
        """Merge a list of samples to form a mini-batch.

        Args:
            samples (List[dict]): samples to collate

        Returns:
            dict: a mini-batch of data
        """
        return self._collate(samples, self.vocab.pad(), self.vocab.eos())

    def num_tokens(
            self,
            index: int
    ):
        """
        Return the number of tokens in a sample. This value is used to
        enforce max-tokens during batching.
        """
        return self.sizes[index]

    def size(
            self,
            index: int
    ):
        """
        Return an example's size as a float or tuple. This value is used when
        filtering a dataset with max-positions.
        """
        return self.sizes[index]

    def ordered_indices(self):
        """
        Return an ordered list of indices. Batches will be constructed based
        on this order.
        """
        if self.shuffle:
            return np.random.permutation(len(self))
        else:
            order = [np.arange(len(self))]
            order.append(self.sizes)
            return np.lexsort(order)

    @property
    def supports_prefetch(self):
        return getattr(self.dataset, "supports_prefetch", False)

    def prefetch(self, indices):
        self.dataset.prefetch(indices)
<|endoftext|>


<commit_before># encoding: utf-8
import re
import logging
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils.text import truncate_words
from django.contrib.contenttypes import generic
from django.contrib.auth.models import User

from django.core.exceptions import ValidationError
from django.contrib.contenttypes.models import ContentType
from tagging.models import Tag
from annotatetext.models import Annotation
from knesset.events.models import Event

COMMITTEE_PROTOCOL_PAGINATE_BY = 400

logger = logging.getLogger("open-knesset.committees.models")

class Committee(models.Model):
    name = models.CharField(max_length=256)
    members = models.ManyToManyField('mks.Member', related_name='committees')
    chairpersons = models.ManyToManyField('mks.Member', related_name='chaired_committees')
    replacements = models.ManyToManyField('mks.Member', related_name='replacing_in_committees')
    events = generic.GenericRelation(Event, content_type_field="which_type",
       object_id_field="which_pk")
    agenda_topics = models.ManyToManyField('topics.Topic', related_name='committees', through="AgendaTopic")

    def __unicode__(self):
        return "%s" % self.name

    @models.permalink
    def get_absolute_url(self):
        return ('committee-detail', [str(self.id)])

    @property
    def annotations(self):
        protocol_part_tn = ProtocolPart._meta.db_table
        meeting_tn = CommitteeMeeting._meta.db_table
        committee_tn = Committee._meta.db_table
        annotation_tn = Annotation._meta.db_table
        protocol_part_ct = ContentType.objects.get_for_model(ProtocolPart)
        ret = Annotation.objects.filter(content_type=protocol_part_ct)
        return ret.extra(tables = [protocol_part_tn,
                    meeting_tn, committee_tn],
                    where = [ "%s.object_id=%s.id" % (annotation_tn, protocol_part_tn),
                              "%s.meeting_id=%s.id" % (protocol_part_tn, meeting_tn),
                              "%s.committee_id=%%s" % meeting_tn],
                    params = [ self.id ]).distinct()

    def members_by_presence(self):
        members = []
        for m in (self.members.all()|
                  self.chairpersons.all()|
                  self.replacements.all()).distinct():
            m.meetings_count = \
                100 * m.committee_meetings.filter(committee=self).count() \
                / self.meetings.count()
            members.append(m)
        members.sort(key=lambda x:x.meetings_count, reverse=True)
        return members

    def recent_meetings(self):
        return self.meetings.all().order_by('-date')[:10]

not_header = re.compile(r'(^אני )|((אלה|אלו|יבוא|מאלה|ייאמר|אומר|אומרת|נאמר|כך|הבאים|הבאות):$)|(\(.\))|(\(\d+\))|(\d\.)'.decode('utf8'))
def legitimate_header(line):
    """Retunrs true if 'line' looks like something should should be a protocol part header"""
    if not(line.endswith(':')) or len(line)>50 or not_header.search(line):
        return False
    return True

class CommitteeMeeting(models.Model):
    committee = models.ForeignKey(Committee, related_name='meetings')
    # TODO: do we really need a date string? can't we just format date?
    date_string = models.CharField(max_length=256)
    date = models.DateField()
    mks_attended = models.ManyToManyField('mks.Member', related_name='committee_meetings')
    votes_mentioned = models.ManyToManyField('laws.Vote', related_name='committee_meetings', blank=True)
    protocol_text = models.TextField(null=True,blank=True)
    topics = models.TextField(null=True,blank=True)
    src_url  = models.URLField(verify_exists=False, max_length=1024,null=True,blank=True)

    class Meta:
        ordering = ('-date',)
        verbose_name = _('Committee Meeting')
        verbose_name_plural = _('Committee Meetings')

    def title (self):
        return truncate_words (self.topics, 12)

    def __unicode__(self):
        return (u"%s - %s" % (self.committee.name,
                                self.title())).replace(" ", u"\u00A0")

    @models.permalink
    def get_absolute_url(self):
        return ('committee-meeting', [str(self.id)])

    def _get_tags(self):
        tags = Tag.objects.get_for_object(self)
        return tags

    def _set_tags(self, tag_list):
        Tag.objects.update_tags(self, tag_list)

    tags = property(_get_tags, _set_tags)

    def save(self, **kwargs):
        super(CommitteeMeeting, self).save(**kwargs)

    def create_protocol_parts(self, delete_existing=False):
        """ Create protocol parts from this instance's protocol_text
            Optionally, delete existing parts.
            If the meeting already has parts, and you don't ask to
            delete them, a ValidationError will be thrown, because
            it doesn't make sense to create the parts again.
        """
        if delete_existing:
            ppct = ContentType.objects.get_for_model(ProtocolPart)
            annotations = Annotation.objects.filter(content_type=ppct, object_id__in=self.parts.all)
            logger.debug('deleting %d annotations, because I was asked to delete the relevant protocol parts on cm.id=%d' % (annotations.count(), self.id))
            annotations.delete()
            self.parts.all().delete()
        else:
            if self.parts.count():
                raise ValidationError('CommitteeMeeting already has parts. delete them if you want to run create_protocol_parts again.')

        if not self.protocol_text: # sometimes there are empty protocols
            return # then we don't need to do anything here.

        # break the protocol to its parts
        # first, fix places where the colon is in the begining of next line
        # (move it to the end of the correct line)
        protocol_text = []
        for line in re.sub("[ ]+"," ", self.protocol_text).split('\n'):
            if line.startswith(':'):
                protocol_text[-1] += ':'
                protocol_text.append(line[1:])
            else:
                protocol_text.append(line)

        i = 1
        section = []
        header = ''

        # now create the sections
        for line in protocol_text:
            if legitimate_header(line):
                if section:
                    ProtocolPart(meeting=self, order=i,
                        header=header, body='\n'.join(section)).save()
                i += 1
                header = line[:-1]
                section = []
            else:
                section.append (line)

        # don't forget the last section
        ProtocolPart(meeting=self, order=i,
            header=header, body='\n'.join(section)).save()

class ProtocolPartManager(models.Manager):
    def list(self):
        return self.order_by("order")

class ProtocolPart(models.Model):
    meeting = models.ForeignKey(CommitteeMeeting, related_name='parts')
    order = models.IntegerField()
    header = models.TextField(blank=True)
    body = models.TextField(blank=True)
    speaker = models.ForeignKey('persons.Person', blank=True, null=True, related_name='protocol_parts')
    objects = ProtocolPartManager()

    annotatable = True

    def get_absolute_url(self):
        if self.order == 1:
            return self.meeting.get_absolute_url()
        else:
            page_num = 1 + (self.order-1)/COMMITTEE_PROTOCOL_PAGINATE_BY
            if page_num==1: # this is on first page
                return "%s#speech-%d-%d" % (self.meeting.get_absolute_url(),
                                            self.meeting.id, self.order)
            else:
                return "%s?page=%d#speech-%d-%d" % (self.meeting.get_absolute_url(),
                                                    page_num,
                                                    self.meeting.id, self.order)

    def __unicode__(self):
        return "%s %s: %s" % (self.meeting.committee.name, self.header,
                              self.header)

AGENDA_TOPIC_STATUS = (
    (1, 'published'),
    (2, 'flagged'),
    (3, 'rejacted'),
    (4, 'accepted'),
    (5, 'requested'),
)

class AgendaTopic(models.Model):
    '''
        AgendaTopic is used to hold the latest event about a topic and a committee

        Fields:
            created - the time a topic was first connected to a committee
            modified - last time the status or the message was updated
            editor - the user that entered the data
            status - the current status
            message - a message explaining the status change
    '''

    topic = models.ForeignKey('topics.Topic')
    committee = models.ForeignKey(Committee)
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)
    editor = models.ForeignKey(User)
    status = models.IntegerField(choices = AGENDA_TOPIC_STATUS)
    message = models.TextField()

from listeners import *
<commit_msg>adding defaults<commit_after># encoding: utf-8
import re
import logging
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils.text import truncate_words
from django.contrib.contenttypes import generic
from django.contrib.auth.models import User

from django.core.exceptions import ValidationError
from django.contrib.contenttypes.models import ContentType
from tagging.models import Tag
from annotatetext.models import Annotation
from knesset.events.models import Event

COMMITTEE_PROTOCOL_PAGINATE_BY = 400

logger = logging.getLogger("open-knesset.committees.models")

class Committee(models.Model):
    name = models.CharField(max_length=256)
    members = models.ManyToManyField('mks.Member', related_name='committees')
    chairpersons = models.ManyToManyField('mks.Member', related_name='chaired_committees')
    replacements = models.ManyToManyField('mks.Member', related_name='replacing_in_committees')
    events = generic.GenericRelation(Event, content_type_field="which_type",
       object_id_field="which_pk")
    agenda_topics = models.ManyToManyField('topics.Topic', related_name='committees', through="AgendaTopic")

    def __unicode__(self):
        return "%s" % self.name

    @models.permalink
    def get_absolute_url(self):
        return ('committee-detail', [str(self.id)])

    @property
    def annotations(self):
        protocol_part_tn = ProtocolPart._meta.db_table
        meeting_tn = CommitteeMeeting._meta.db_table
        committee_tn = Committee._meta.db_table
        annotation_tn = Annotation._meta.db_table
        protocol_part_ct = ContentType.objects.get_for_model(ProtocolPart)
        ret = Annotation.objects.filter(content_type=protocol_part_ct)
        return ret.extra(tables = [protocol_part_tn,
                    meeting_tn, committee_tn],
                    where = [ "%s.object_id=%s.id" % (annotation_tn, protocol_part_tn),
                              "%s.meeting_id=%s.id" % (protocol_part_tn, meeting_tn),
                              "%s.committee_id=%%s" % meeting_tn],
                    params = [ self.id ]).distinct()

    def members_by_presence(self):
        members = []
        for m in (self.members.all()|
                  self.chairpersons.all()|
                  self.replacements.all()).distinct():
            m.meetings_count = \
                100 * m.committee_meetings.filter(committee=self).count() \
                / self.meetings.count()
            members.append(m)
        members.sort(key=lambda x:x.meetings_count, reverse=True)
        return members

    def recent_meetings(self):
        return self.meetings.all().order_by('-date')[:10]

not_header = re.compile(r'(^אני )|((אלה|אלו|יבוא|מאלה|ייאמר|אומר|אומרת|נאמר|כך|הבאים|הבאות):$)|(\(.\))|(\(\d+\))|(\d\.)'.decode('utf8'))
def legitimate_header(line):
    """Retunrs true if 'line' looks like something should should be a protocol part header"""
    if not(line.endswith(':')) or len(line)>50 or not_header.search(line):
        return False
    return True

class CommitteeMeeting(models.Model):
    committee = models.ForeignKey(Committee, related_name='meetings')
    # TODO: do we really need a date string? can't we just format date?
    date_string = models.CharField(max_length=256)
    date = models.DateField()
    mks_attended = models.ManyToManyField('mks.Member', related_name='committee_meetings')
    votes_mentioned = models.ManyToManyField('laws.Vote', related_name='committee_meetings', blank=True)
    protocol_text = models.TextField(null=True,blank=True)
    topics = models.TextField(null=True,blank=True)
    src_url  = models.URLField(verify_exists=False, max_length=1024,null=True,blank=True)

    class Meta:
        ordering = ('-date',)
        verbose_name = _('Committee Meeting')
        verbose_name_plural = _('Committee Meetings')

    def title (self):
        return truncate_words (self.topics, 12)

    def __unicode__(self):
        return (u"%s - %s" % (self.committee.name,
                                self.title())).replace(" ", u"\u00A0")

    @models.permalink
    def get_absolute_url(self):
        return ('committee-meeting', [str(self.id)])

    def _get_tags(self):
        tags = Tag.objects.get_for_object(self)
        return tags

    def _set_tags(self, tag_list):
        Tag.objects.update_tags(self, tag_list)

    tags = property(_get_tags, _set_tags)

    def save(self, **kwargs):
        super(CommitteeMeeting, self).save(**kwargs)

    def create_protocol_parts(self, delete_existing=False):
        """ Create protocol parts from this instance's protocol_text
            Optionally, delete existing parts.
            If the meeting already has parts, and you don't ask to
            delete them, a ValidationError will be thrown, because
            it doesn't make sense to create the parts again.
        """
        if delete_existing:
            ppct = ContentType.objects.get_for_model(ProtocolPart)
            annotations = Annotation.objects.filter(content_type=ppct, object_id__in=self.parts.all)
            logger.debug('deleting %d annotations, because I was asked to delete the relevant protocol parts on cm.id=%d' % (annotations.count(), self.id))
            annotations.delete()
            self.parts.all().delete()
        else:
            if self.parts.count():
                raise ValidationError('CommitteeMeeting already has parts. delete them if you want to run create_protocol_parts again.')

        if not self.protocol_text: # sometimes there are empty protocols
            return # then we don't need to do anything here.

        # break the protocol to its parts
        # first, fix places where the colon is in the begining of next line
        # (move it to the end of the correct line)
        protocol_text = []
        for line in re.sub("[ ]+"," ", self.protocol_text).split('\n'):
            if line.startswith(':'):
                protocol_text[-1] += ':'
                protocol_text.append(line[1:])
            else:
                protocol_text.append(line)

        i = 1
        section = []
        header = ''

        # now create the sections
        for line in protocol_text:
            if legitimate_header(line):
                if section:
                    ProtocolPart(meeting=self, order=i,
                        header=header, body='\n'.join(section)).save()
                i += 1
                header = line[:-1]
                section = []
            else:
                section.append (line)

        # don't forget the last section
        ProtocolPart(meeting=self, order=i,
            header=header, body='\n'.join(section)).save()

class ProtocolPartManager(models.Manager):
    def list(self):
        return self.order_by("order")

class ProtocolPart(models.Model):
    meeting = models.ForeignKey(CommitteeMeeting, related_name='parts')
    order = models.IntegerField()
    header = models.TextField(blank=True)
    body = models.TextField(blank=True)
    speaker = models.ForeignKey('persons.Person', blank=True, null=True, related_name='protocol_parts')
    objects = ProtocolPartManager()

    annotatable = True

    def get_absolute_url(self):
        if self.order == 1:
            return self.meeting.get_absolute_url()
        else:
            page_num = 1 + (self.order-1)/COMMITTEE_PROTOCOL_PAGINATE_BY
            if page_num==1: # this is on first page
                return "%s#speech-%d-%d" % (self.meeting.get_absolute_url(),
                                            self.meeting.id, self.order)
            else:
                return "%s?page=%d#speech-%d-%d" % (self.meeting.get_absolute_url(),
                                                    page_num,
                                                    self.meeting.id, self.order)

    def __unicode__(self):
        return "%s %s: %s" % (self.meeting.committee.name, self.header,
                              self.header)

AGENDA_TOPIC_STATUS = (
    (1, 'published'),
    (2, 'flagged'),
    (3, 'rejacted'),
    (4, 'accepted'),
    (5, 'requested'),
)

class AgendaTopic(models.Model):
    '''
        AgendaTopic is used to hold the latest event about a topic and a committee

        Fields:
            created - the time a topic was first connected to a committee
            modified - last time the status or the message was updated
            editor - the user that entered the data
            status - the current status
            message - a message explaining the status change
    '''

    topic = models.ForeignKey('topics.Topic')
    committee = models.ForeignKey(Committee)
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)
    editor = models.ForeignKey(User)
    status = models.IntegerField(choices = AGENDA_TOPIC_STATUS, default=1)
    message = models.TextField(default="")

from listeners import *
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
This script downloads the latest KeyForge card data from keyforgegame.com and processes
it to turn the highly-structured data there into a flat list of card names to descriptions
formatted to send down the chat.
"""

import common
common.FRAMEWORK_ONLY = True
import sys
import json
import re
import dateutil.parser
import sqlalchemy

from common import utils
import common.postgres
from common.card import clean_text, CARD_GAME_KEYFORGE
from common.http import request

URL = 'https://www.keyforgegame.com/api/decks/?page={page}&page_size=25&links=cards'

EXPANSIONS = {
	341: {
		'code': 'KF02',
		'name': 'Call of the Archons',
		'count': 370,
		'releaseDate': '2018-11-15',
	},
}

engine, metadata = common.postgres.get_engine_and_metadata()

def main():
	print("Downloading card data...")
	carddata, houses = fetch_card_data()

	print("Processing...")
	cards = metadata.tables["cards"]
	card_collector = metadata.tables["card_collector"]
	with engine.begin() as conn, conn.begin() as trans:
		conn.execute(cards.delete().where(cards.c.game == CARD_GAME_KEYFORGE))
		for setid, cardset in sorted(carddata.items()):
			expansion = EXPANSIONS[setid]
			# Allow only importing individual sets for faster testing
			if len(sys.argv) > 1 and expansion['code'] not in sys.argv[1:]:
				continue

			release_date = dateutil.parser.parse(expansion.get('releaseDate', '1970-01-01')).date()
			for filteredname, cardname, description, collectors, hidden in process_set(expansion, cardset, houses):
				# Check if there's already a row for this card in the DB
				# (keep the one with the latest release date - it's more likely to have the accurate text)
				rows = conn.execute(sqlalchemy.select([cards.c.id, cards.c.lastprinted])
					.where(cards.c.filteredname == filteredname)
					.where(cards.c.game == CARD_GAME_KEYFORGE)).fetchall()
				if not rows:
					cardid, = conn.execute(cards.insert().returning(cards.c.id),
						game=CARD_GAME_KEYFORGE,
						filteredname=filteredname,
						name=cardname,
						text=description,
						lastprinted=release_date,
						hidden=hidden,
					).first()
				elif rows[0][1] < release_date:
					cardid = rows[0][0]
					conn.execute(cards.update().where(cards.c.id == cardid),
						name=cardname,
						text=description,
						lastprinted=release_date,
						hidden=hidden,
					)
				else:
					cardid = rows[0][0]

				for csetid, collector in collectors:
					rows = conn.execute(sqlalchemy.select([card_collector.c.cardid])
						.where((card_collector.c.setid == csetid) & (card_collector.c.collector == collector))).fetchall()
					if not rows:
						conn.execute(card_collector.insert(),
							setid=csetid,
							collector=collector,
							cardid=cardid,
						)
					elif rows[0][0] != cardid:
						rows2 = conn.execute(sqlalchemy.select([cards.c.name]).where(cards.c.id == rows[0][0])).fetchall()
						print("Different names for set %s collector number %s: \"%s\" and \"%s\"" % (csetid, collector, cardname, rows2[0][0]))

def fetch_card_data():
	# KeyForge does not appear to have an API for directly downloading card data
	# however, it _does_ have an API for downloading _decks_, and then that has
	# an option for downloading the data for the cards that appear in those decks.
	# So we can just keep downloading until we see all the cards. Though this does
	# require hard-coding how many cards we expect to see.
	complete = False
	cards = {}
	houses = {}
	pagenum = 0
	lastcount = 0
	noopcount = 0
	while True:
		pagenum += 1
		page = request(URL.format(page=pagenum))
		page = json.loads(page)
		for card in page['_linked']['cards']:
			# Mavericks are copies of existing cards in a different house
			# Don't need them for the DB, and they're rare enough that it takes a lot
			# of pageloads to find them all
			if not card['is_maverick']:
				if card['expansion'] not in cards:
					cards[card['expansion']] = {}
				cards[card['expansion']][card['id']] = card
		for house in page['_linked']['houses']:
			houses[house['id']] = house

		# count our cards to see if we've gotten them all
		if all(len(cards[k]) >= v['count'] for k,v in EXPANSIONS.items()):
			break

		# as a safety catch, if we go a bunch of pages without seeing any new cards
		# then bail out, in case our expected count is incorrect
		count = sum(len(i) for i in cards.values())
		if count == lastcount:
			noopcount += 1
			if noopcount >= 10:
				break
		else:
			noopcount = 0
		lastcount = count
	return {k: list(v.values()) for k,v in cards.items()}, houses

re_check = re.compile(r"^[a-z0-9_]+$")
re_newlines = re.compile(r"[\r\n\x0b]+")
re_multiplespaces = re.compile(r"\s{2,}")
re_remindertext = re.compile(r"( *)\([^()]*\)( *)")
re_minuses = re.compile(r"(?:^|(?<=[\s/]))[-\u2013](?=[\dXY])")
def process_card(card, expansion, houses, include_reminder=False):
	# sanitise card name
	filtered = clean_text(card.get('internalname', card["card_title"]))
	if not re_check.match(filtered):
		print("Still some junk left in name %s (%s)" % (card.get('internalname', card["card_title"]), json.dumps(filtered)))
		print(json.dumps(card))
		sys.exit(1)

	def build_description():
		yield card['card_title']
		yield ' ['
		if card['house'] in houses:
			yield houses[card['house']]['name']
		else:
			yield card['house']
		if card.get('amber'):
			yield '\u2014'
			yield str(card['amber'])
			yield '<A>'
		yield '] | '
		yield card['card_type']
		if card.get('traits'):
			yield ' \u2014 '
			yield card['traits']
		if card.get('power') or card.get('armor') or card['card_type'] == "Creature":
			yield ' ['
			yield str(card['power'])
			if card.get('armor'):
				yield '/'
				yield str(card['armor'])
			yield ']'
		if card.get('card_text'):
			yield ' | '
			yield process_text(card['card_text'], include_reminder)

	desc = ''.join(build_description())
	desc = re_multiplespaces.sub(' ', desc).strip()
	desc = utils.trim_length(desc)

	numbers = card['card_number'] if card.get('card_number') else []
	if not isinstance(numbers, list):
		numbers = [numbers]
	numbers = [(expansion['code'].lower(), str(i)) for i in numbers]
	hidden = 'internalname' in card

	return filtered, card['card_title'], desc, numbers, hidden

def process_text(text, include_reminder):
	text = re_minuses.sub('\u2212', text) # replace hyphens with real minus signs
	if not include_reminder:
		text = re_remindertext.sub(lambda match: ' ' if match.group(1) and match.group(2) else '', text)
	text = re_newlines.sub(' / ', text.strip())
	return text

SPECIAL_SETS = {}
def special_set(setid):
	def decorator(func):
		SPECIAL_SETS[setid] = func
		return func
	return decorator

def process_set(expansion, cards, houses):
	handler = SPECIAL_SETS.get(expansion['code'], process_set_general)
	yield from handler(expansion, cards, houses)

def process_set_general(expansion, cards, houses):
	for card in cards:
		# Turn on include_reminder by default for now, since KF is newer than MTG
		# Can't assume watchers know all the basic abilities
		# Also, most KF card text is shorter than some of the lengthier MTG cards,
		# we can afford leaving in the reminder text
		yield process_card(card, expansion, houses, include_reminder=True)

if __name__ == '__main__':
	main()
<commit_msg>Add handling for new expansion for Keyforge<commit_after>#!/usr/bin/env python3
"""
This script downloads the latest KeyForge card data from keyforgegame.com and processes
it to turn the highly-structured data there into a flat list of card names to descriptions
formatted to send down the chat.
"""

import common
common.FRAMEWORK_ONLY = True
import sys
import json
import re
import os
import math
import dateutil.parser
import sqlalchemy

from common import utils
import common.postgres
from common.card import clean_text, CARD_GAME_KEYFORGE
from common.http import request

PAGE_SIZE = 25
URL = 'https://www.keyforgegame.com/api/decks/?page={{page}}&page_size={page_size}&links=cards'.format(page_size=PAGE_SIZE)

EXPANSIONS = {
	341: {
		'code': 'KF02',
		'name': 'Call of the Archons',
		'releaseDate': '2018-11-15',
	},
	435: {
		'code': 'KF04',
		'name': 'Age of Ascension',
		'releaseDate': '2019-05-30',
	}
}

engine, metadata = common.postgres.get_engine_and_metadata()

def main():
	print("Downloading card data...")
	carddata, houses = fetch_card_data()

	print("Processing...")
	cards = metadata.tables["cards"]
	card_collector = metadata.tables["card_collector"]
	with engine.begin() as conn, conn.begin() as trans:
		conn.execute(cards.delete().where(cards.c.game == CARD_GAME_KEYFORGE))
		for setid, cardset in sorted(carddata.items()):
			expansion = EXPANSIONS[setid]
			# Allow only importing individual sets for faster testing
			if len(sys.argv) > 1 and expansion['code'] not in sys.argv[1:]:
				continue

			release_date = dateutil.parser.parse(expansion.get('releaseDate', '1970-01-01')).date()
			for filteredname, cardname, description, collectors, hidden in process_set(expansion, cardset, houses):
				# Check if there's already a row for this card in the DB
				# (keep the one with the latest release date - it's more likely to have the accurate text)
				rows = conn.execute(sqlalchemy.select([cards.c.id, cards.c.lastprinted])
					.where(cards.c.filteredname == filteredname)
					.where(cards.c.game == CARD_GAME_KEYFORGE)).fetchall()
				if not rows:
					cardid, = conn.execute(cards.insert().returning(cards.c.id),
						game=CARD_GAME_KEYFORGE,
						filteredname=filteredname,
						name=cardname,
						text=description,
						lastprinted=release_date,
						hidden=hidden,
					).first()
				elif rows[0][1] < release_date:
					cardid = rows[0][0]
					conn.execute(cards.update().where(cards.c.id == cardid),
						name=cardname,
						text=description,
						lastprinted=release_date,
						hidden=hidden,
					)
				else:
					cardid = rows[0][0]

				for csetid, collector in collectors:
					rows = conn.execute(sqlalchemy.select([card_collector.c.cardid])
						.where((card_collector.c.setid == csetid) & (card_collector.c.collector == collector))).fetchall()
					if not rows:
						conn.execute(card_collector.insert(),
							setid=csetid,
							collector=collector,
							cardid=cardid,
						)
					elif rows[0][0] != cardid:
						rows2 = conn.execute(sqlalchemy.select([cards.c.name]).where(cards.c.id == rows[0][0])).fetchall()
						print("Different names for set %s collector number %s: \"%s\" and \"%s\"" % (csetid, collector, cardname, rows2[0][0]))

def getpage(page):
	fn = ".kfcache/{}.json".format(page)
	if os.path.exists(fn):
		with open(fn, 'r') as fp:
			return fp.read()
	else:
		dat = request(URL.format(page=page))
		with open(fn, 'w') as fp:
			fp.write(dat)
		return dat

def fetch_card_data():
	# KeyForge does not appear to have an API for directly downloading card data
	# however, it _does_ have an API for downloading _decks_, and then that has
	# an option for downloading the data for the cards that appear in those decks.
	# So we can just keep downloading until we see all the cards. Though this does
	# require hard-coding how many cards we expect to see.
	cards = {}
	houses = {}
	pagenum = 1
	pagetotal = 1
	pagestep = 1.0
	while pagenum <= pagetotal:
		#print("%d/%d (+%.02f)" % (pagenum, pagetotal, pagestep))
		page = getpage(pagenum)
		page = json.loads(page)

		pagetotal = math.ceil(page['count'] / PAGE_SIZE)

		foundnew = False
		for card in page['_linked']['cards']:
			# Mavericks are copies of existing cards in a different house
			# Don't need them for the DB, and they're rare enough that it takes a lot
			# of pageloads to find them all
			if card['is_maverick']:
				continue
			if card['expansion'] not in cards:
				cards[card['expansion']] = {}
			if card['id'] not in cards[card['expansion']]:
				foundnew = True
			cards[card['expansion']][card['id']] = card
		for house in page['_linked']['houses']:
			houses[house['id']] = house

		# The decks are grouped by expansion, so there'll be a bunch of new cards
		# at the start, then a long run of duplicates, then a bunch of new cards at
		# the next expansion, etc. So load every page while we're still seeing new
		# cards, but then jump forward to try to find the next section.
		if foundnew:
			pagestep = 1.0
		else:
			pagestep *= 1.2
		pagenum += math.floor(pagestep)

	#print({k:len(v) for k,v in cards.items()})

	return {k: list(v.values()) for k,v in cards.items()}, houses

re_check = re.compile(r"^[a-z0-9_]+$")
re_newlines = re.compile(r"[\r\n\x0b]+")
re_multiplespaces = re.compile(r"\s{2,}")
re_remindertext = re.compile(r"( *)\([^()]*\)( *)")
re_minuses = re.compile(r"(?:^|(?<=[\s/]))[-\u2013](?=[\dXY])")
def process_card(card, expansion, houses, include_reminder=False):
	# sanitise card name
	filtered = clean_text(card.get('internalname', card["card_title"]))
	if not re_check.match(filtered):
		print("Still some junk left in name %s (%s)" % (card.get('internalname', card["card_title"]), json.dumps(filtered)))
		print(json.dumps(card))
		sys.exit(1)

	def build_description():
		yield card['card_title']
		yield ' ['
		if card['house'] in houses:
			yield houses[card['house']]['name']
		else:
			yield card['house']
		if card.get('amber'):
			yield '\u2014'
			yield str(card['amber'])
			yield '<A>'
		yield '] | '
		yield card['card_type']
		if card.get('traits'):
			yield ' \u2014 '
			yield card['traits']
		if card.get('power') or card.get('armor') or card['card_type'] == "Creature":
			yield ' ['
			yield str(card['power'])
			if card.get('armor'):
				yield '/'
				yield str(card['armor'])
			yield ']'
		if card.get('card_text'):
			yield ' | '
			yield process_text(card['card_text'], include_reminder)

	desc = ''.join(build_description())
	desc = re_multiplespaces.sub(' ', desc).strip()
	desc = utils.trim_length(desc)

	numbers = card['card_number'] if card.get('card_number') else []
	if not isinstance(numbers, list):
		numbers = [numbers]
	numbers = [(expansion['code'].lower(), str(i)) for i in numbers]
	hidden = 'internalname' in card

	return filtered, card['card_title'], desc, numbers, hidden

def process_text(text, include_reminder):
	text = re_minuses.sub('\u2212', text) # replace hyphens with real minus signs
	if not include_reminder:
		text = re_remindertext.sub(lambda match: ' ' if match.group(1) and match.group(2) else '', text)
	text = re_newlines.sub(' / ', text.strip())
	return text

SPECIAL_SETS = {}
def special_set(setid):
	def decorator(func):
		SPECIAL_SETS[setid] = func
		return func
	return decorator

def process_set(expansion, cards, houses):
	handler = SPECIAL_SETS.get(expansion['code'], process_set_general)
	yield from handler(expansion, cards, houses)

def process_set_general(expansion, cards, houses):
	for card in cards:
		# Turn on include_reminder by default for now, since KF is newer than MTG
		# Can't assume watchers know all the basic abilities
		# Also, most KF card text is shorter than some of the lengthier MTG cards,
		# we can afford leaving in the reminder text
		yield process_card(card, expansion, houses, include_reminder=True)

if __name__ == '__main__':
	main()
<|endoftext|>