<commit_before>#!/usr/bin/env python3

r"""
Panflute filter to parse table in fenced YAML code blocks.
Currently only CSV table is supported.

7 metadata keys are recognized:

-   caption: the caption of the table. If omitted, no caption will be inserted.
-   alignment: a string of characters among L,R,C,D, case-insensitive,
        corresponds to Left-aligned, Right-aligned,
        Center-aligned, Default-aligned respectively.
    e.g. LCRD for a table with 4 columns
    default: DDD...
-   width: a list of relative width corresponding to the width of each columns.
    default: auto calculate from the length of each line in table cells.
-   table-width: the relative width of the table (e.g. relative to \linewidth).
    default: 1.0
-   header: If it has a header row. default: true
-   markdown: If CSV table cell contains markdown syntax. default: False
-   include: the path to an CSV file.
    If non-empty, override the CSV in the CodeBlock.
    default: None

When the metadata keys is invalid, the default will be used instead.
Note that width and table-width accept fractions as well.

e.g.

```table
---
caption: '*Awesome* **Markdown** Table'
alignment: RC
table-width: 2/3
markdown: True
---
First row,defaulted to be header row,can be disabled
1,cell can contain **markdown**,"It can be aribrary block element:

- following standard markdown syntax
- like this"
2,"Any markdown syntax, e.g.",$$E = mc^2$$
```
"""

import csv
import fractions
import io
import panflute

# begin helper functions
def to_bool(to_be_bool, default=True):
    """
    Do nothing if to_be_bool is boolean,
    return `False` if it is "false" or "no" (case-insensitive),
    otherwise return default.
    """
    if not isinstance(to_be_bool, bool):
        try:
            lowered_bool = to_be_bool.lower()
            if lowered_bool in ("false", "no"):
                to_be_bool = False
            elif lowered_bool in ("true", "yes"):
                to_be_bool = True
            else:
                raise ValueError
        except (ValueError, TypeError, AttributeError):
            to_be_bool = default
            panflute.debug("""pantable: invalid boolean. \
Should be true/false/yes/no, case-insensitive. Default is used.""")
    return to_be_bool

def get_width(options, number_of_columns):
    """
    get width: set to `None` when

    1. not given
    2. not a list
    3. length not equal to the number of columns
    4. negative entries
    """
    try:
        width = options['width']
        if len(width) != number_of_columns:
            raise ValueError
        custom_float = lambda x: float(fractions.Fraction(x))
        width = [custom_float(x) for x in options['width']]
        if not all(i >= 0 for i in width):
            raise ValueError
    except KeyError:
        width = None
    except (ValueError, TypeError):
        width = None
        panflute.debug("pantable: invalid width")
    return width

def get_table_width(options):
    """
    `table-width` set to `1.0` if invalid
    """
    try:
        table_width = float(fractions.Fraction((options.get('table-width'))))
        if table_width <= 0:
            raise ValueError
    except KeyError:
        table_width = 1.0
    except (ValueError, TypeError):
        table_width = 1.0
        panflute.debug("pantable: invalid table-width")
    return table_width
# end helper functions

def auto_width(table_width, number_of_columns, table_list):
    """
    `width` is auto-calculated if not given in YAML
    It also returns None when table is empty.
    """
    # calculate width
    # The +3 match the way pandoc handle width, see jgm/pandoc commit 0dfceda
    width_abs = [3 + max(
        [max(
            [len(line) for line in row[i].split("\n")]
        ) for row in table_list]
    ) for i in range(number_of_columns)]
    try:
        width_tot = sum(width_abs)
        # when all are 3 means all are empty, see comment above
        if width_tot == 3 * number_of_columns:
            raise ValueError
        width = [
            width_abs[i] / width_tot * table_width
            for i in range(number_of_columns)
        ]
    except ValueError:
        panflute.debug("pantable: table is empty")
        width = None
    return width

def parse_alignment(alignment_string, number_of_columns):
    """
    `alignment` string is parsed into pandoc format (AlignDefault, etc.)
    """
    # initialize
    alignment_string = str(alignment_string)
    number_of_alignments = len(alignment_string)
    # truncate and debug if too long
    try:
        if number_of_alignments > number_of_columns:
            raise ValueError
    except ValueError:
        alignment_string = alignment_string[:number_of_columns]
        panflute.debug("pantable: alignment string is too long")
    # parsing
    alignment = [("AlignLeft" if i.lower() == "l"
                  else "AlignCenter" if i.lower() == "c"
                  else "AlignRight" if i.lower() == "r"
                  else "AlignDefault" if i.lower() == "d"
                  else None) for i in alignment_string]
    # debug if invalid; set to default
    try:
        if None in alignment:
            raise ValueError
    except ValueError:
        alignment = [(i if i is not None else "AlignDefault")
                     for i in alignment]
        panflute.debug(
            "pantable: alignment: invalid character found, default is used instead.")
    # fill up with default if too short
    if number_of_columns > number_of_alignments:
        alignment += ["AlignDefault" for __ in range(
            number_of_columns - len(alignment))]
    return alignment

def read_data(include, data):
    """
    read csv and return the table in list
    """
    if include is None:
        with io.StringIO(data) as file:
            raw_table_list = list(csv.reader(file))
    else:
        try:
            with open(str(include)) as file:
                raw_table_list = list(csv.reader(file))
        except FileNotFoundError:
            raw_table_list = None
            panflute.debug('{} {}'.format("pantable: file not found from the path", include))
    return raw_table_list

def regularize_table_list(raw_table_list):
    """
    When the length of rows are uneven, make it as long as the longest row.
    """
    length_of_rows = [len(row) for row in raw_table_list]
    number_of_columns = max(length_of_rows)
    try:
        if all(i == number_of_columns for i in length_of_rows):
            table_list = raw_table_list
        else:
            raise ValueError
    except ValueError:
        table_list = [
            row + ['' for __ in range(number_of_columns - len(row))] for row in raw_table_list]
        panflute.debug(
            "pantable: table rows are of irregular length. Empty cells appended.")
    return (table_list, number_of_columns)

def parse_table_list(markdown, table_list):
    """
    read table in list and return panflute table format
    """
    if markdown:
        table_body = [panflute.TableRow(*[
            panflute.TableCell(*panflute.convert_text(x))
            for x in row
        ]) for row in table_list]
    else:
        table_body = [panflute.TableRow(*[
            panflute.TableCell(panflute.Plain(panflute.Str(x)))
            for x in row
        ]) for row in table_list]
    return table_body

def convert2table(options, data, **__):
    """
    provided to panflute.yaml_filter to parse its content as pandoc table.
    """
    # prepare table in list from data/include
    raw_table_list = read_data(options.get('include', None), data)
    # delete element if table is empty (by returning [])
    # element unchanged if include is invalid (by returning None)
    try:
        if not raw_table_list or raw_table_list is None:
            raise ValueError
    except ValueError:
        panflute.debug("pantable: table is empty or include is invalid")
        return raw_table_list
    # regularize table: all rows should have same length
    table_list, number_of_columns = regularize_table_list(raw_table_list)

    # Initialize the `options` output from `panflute.yaml_filter`
    # parse width
    width = get_width(options, number_of_columns)
    # auto-width when width is not specified
    if width is None:
        width = auto_width(get_table_width(
            options), number_of_columns, table_list)
    # delete element if table is empty (by returning [])
    # width remains None only when table is empty
    try:
        if width is None:
            raise ValueError
    except ValueError:
        panflute.debug("pantable: table is empty")
        return []
    # parse alignment
    alignment = parse_alignment(options.get(
        'alignment', None), number_of_columns)
    header = to_bool(options.get('header', True), True)
    markdown = to_bool(options.get('markdown', False), False)

    # get caption: parsed as markdown into panflute AST if non-empty.
    caption = panflute.convert_text(str(options['caption']))[
        0].content if 'caption' in options else None
    # parse list to panflute table
    table_body = parse_table_list(markdown, table_list)
    # extract header row
    header_row = table_body.pop(0) if (
        len(table_body) > 1 and header
    ) else None
    return panflute.Table(
        *table_body,
        caption=caption,
        alignment=alignment,
        width=width,
        header=header_row
    )

def main(_=None):
    """
    Fenced code block with class table will be parsed using
    panflute.yaml_filter with the fuction convert2table above.
    """
    return panflute.run_filter(
        panflute.yaml_filter,
        tag='table',
        function=convert2table,
        strict_yaml=True
    )

if __name__ == '__main__':
    main()
<commit_msg>pantable.py: auto_width: minor simplification in list comprehension<commit_after>#!/usr/bin/env python3

r"""
Panflute filter to parse table in fenced YAML code blocks.
Currently only CSV table is supported.

7 metadata keys are recognized:

-   caption: the caption of the table. If omitted, no caption will be inserted.
-   alignment: a string of characters among L,R,C,D, case-insensitive,
        corresponds to Left-aligned, Right-aligned,
        Center-aligned, Default-aligned respectively.
    e.g. LCRD for a table with 4 columns
    default: DDD...
-   width: a list of relative width corresponding to the width of each columns.
    default: auto calculate from the length of each line in table cells.
-   table-width: the relative width of the table (e.g. relative to \linewidth).
    default: 1.0
-   header: If it has a header row. default: true
-   markdown: If CSV table cell contains markdown syntax. default: False
-   include: the path to an CSV file.
    If non-empty, override the CSV in the CodeBlock.
    default: None

When the metadata keys is invalid, the default will be used instead.
Note that width and table-width accept fractions as well.

e.g.

```table
---
caption: '*Awesome* **Markdown** Table'
alignment: RC
table-width: 2/3
markdown: True
---
First row,defaulted to be header row,can be disabled
1,cell can contain **markdown**,"It can be aribrary block element:

- following standard markdown syntax
- like this"
2,"Any markdown syntax, e.g.",$$E = mc^2$$
```
"""

import csv
import fractions
import io
import panflute

# begin helper functions
def to_bool(to_be_bool, default=True):
    """
    Do nothing if to_be_bool is boolean,
    return `False` if it is "false" or "no" (case-insensitive),
    otherwise return default.
    """
    if not isinstance(to_be_bool, bool):
        try:
            lowered_bool = to_be_bool.lower()
            if lowered_bool in ("false", "no"):
                to_be_bool = False
            elif lowered_bool in ("true", "yes"):
                to_be_bool = True
            else:
                raise ValueError
        except (ValueError, TypeError, AttributeError):
            to_be_bool = default
            panflute.debug("""pantable: invalid boolean. \
Should be true/false/yes/no, case-insensitive. Default is used.""")
    return to_be_bool

def get_width(options, number_of_columns):
    """
    get width: set to `None` when

    1. not given
    2. not a list
    3. length not equal to the number of columns
    4. negative entries
    """
    try:
        width = options['width']
        if len(width) != number_of_columns:
            raise ValueError
        custom_float = lambda x: float(fractions.Fraction(x))
        width = [custom_float(x) for x in options['width']]
        if not all(i >= 0 for i in width):
            raise ValueError
    except KeyError:
        width = None
    except (ValueError, TypeError):
        width = None
        panflute.debug("pantable: invalid width")
    return width

def get_table_width(options):
    """
    `table-width` set to `1.0` if invalid
    """
    try:
        table_width = float(fractions.Fraction((options.get('table-width'))))
        if table_width <= 0:
            raise ValueError
    except KeyError:
        table_width = 1.0
    except (ValueError, TypeError):
        table_width = 1.0
        panflute.debug("pantable: invalid table-width")
    return table_width
# end helper functions

def auto_width(table_width, number_of_columns, table_list):
    """
    `width` is auto-calculated if not given in YAML
    It also returns None when table is empty.
    """
    # calculate width
    # The +3 match the way pandoc handle width, see jgm/pandoc commit 0dfceda
    width_abs = [3 + max(
        [max(
            [len(line) for line in row[i].split("\n")]
        ) for row in table_list]
    ) for i in range(number_of_columns)]
    try:
        width_tot = sum(width_abs)
        # when all are 3 means all are empty, see comment above
        if width_tot == 3 * number_of_columns:
            raise ValueError
        width = [
            each_width / width_tot * table_width
            for each_width in width_abs
        ]
    except ValueError:
        panflute.debug("pantable: table is empty")
        width = None
    return width

def parse_alignment(alignment_string, number_of_columns):
    """
    `alignment` string is parsed into pandoc format (AlignDefault, etc.)
    """
    # initialize
    alignment_string = str(alignment_string)
    number_of_alignments = len(alignment_string)
    # truncate and debug if too long
    try:
        if number_of_alignments > number_of_columns:
            raise ValueError
    except ValueError:
        alignment_string = alignment_string[:number_of_columns]
        panflute.debug("pantable: alignment string is too long")
    # parsing
    alignment = [("AlignLeft" if i.lower() == "l"
                  else "AlignCenter" if i.lower() == "c"
                  else "AlignRight" if i.lower() == "r"
                  else "AlignDefault" if i.lower() == "d"
                  else None) for i in alignment_string]
    # debug if invalid; set to default
    try:
        if None in alignment:
            raise ValueError
    except ValueError:
        alignment = [(i if i is not None else "AlignDefault")
                     for i in alignment]
        panflute.debug(
            "pantable: alignment: invalid character found, default is used instead.")
    # fill up with default if too short
    if number_of_columns > number_of_alignments:
        alignment += ["AlignDefault" for __ in range(
            number_of_columns - len(alignment))]
    return alignment

def read_data(include, data):
    """
    read csv and return the table in list
    """
    if include is None:
        with io.StringIO(data) as file:
            raw_table_list = list(csv.reader(file))
    else:
        try:
            with open(str(include)) as file:
                raw_table_list = list(csv.reader(file))
        except FileNotFoundError:
            raw_table_list = None
            panflute.debug('{} {}'.format("pantable: file not found from the path", include))
    return raw_table_list

def regularize_table_list(raw_table_list):
    """
    When the length of rows are uneven, make it as long as the longest row.
    """
    length_of_rows = [len(row) for row in raw_table_list]
    number_of_columns = max(length_of_rows)
    try:
        if all(i == number_of_columns for i in length_of_rows):
            table_list = raw_table_list
        else:
            raise ValueError
    except ValueError:
        table_list = [
            row + ['' for __ in range(number_of_columns - len(row))] for row in raw_table_list]
        panflute.debug(
            "pantable: table rows are of irregular length. Empty cells appended.")
    return (table_list, number_of_columns)

def parse_table_list(markdown, table_list):
    """
    read table in list and return panflute table format
    """
    if markdown:
        table_body = [panflute.TableRow(*[
            panflute.TableCell(*panflute.convert_text(x))
            for x in row
        ]) for row in table_list]
    else:
        table_body = [panflute.TableRow(*[
            panflute.TableCell(panflute.Plain(panflute.Str(x)))
            for x in row
        ]) for row in table_list]
    return table_body

def convert2table(options, data, **__):
    """
    provided to panflute.yaml_filter to parse its content as pandoc table.
    """
    # prepare table in list from data/include
    raw_table_list = read_data(options.get('include', None), data)
    # delete element if table is empty (by returning [])
    # element unchanged if include is invalid (by returning None)
    try:
        if not raw_table_list or raw_table_list is None:
            raise ValueError
    except ValueError:
        panflute.debug("pantable: table is empty or include is invalid")
        return raw_table_list
    # regularize table: all rows should have same length
    table_list, number_of_columns = regularize_table_list(raw_table_list)

    # Initialize the `options` output from `panflute.yaml_filter`
    # parse width
    width = get_width(options, number_of_columns)
    # auto-width when width is not specified
    if width is None:
        width = auto_width(get_table_width(
            options), number_of_columns, table_list)
    # delete element if table is empty (by returning [])
    # width remains None only when table is empty
    try:
        if width is None:
            raise ValueError
    except ValueError:
        panflute.debug("pantable: table is empty")
        return []
    # parse alignment
    alignment = parse_alignment(options.get(
        'alignment', None), number_of_columns)
    header = to_bool(options.get('header', True), True)
    markdown = to_bool(options.get('markdown', False), False)

    # get caption: parsed as markdown into panflute AST if non-empty.
    caption = panflute.convert_text(str(options['caption']))[
        0].content if 'caption' in options else None
    # parse list to panflute table
    table_body = parse_table_list(markdown, table_list)
    # extract header row
    header_row = table_body.pop(0) if (
        len(table_body) > 1 and header
    ) else None
    return panflute.Table(
        *table_body,
        caption=caption,
        alignment=alignment,
        width=width,
        header=header_row
    )

def main(_=None):
    """
    Fenced code block with class table will be parsed using
    panflute.yaml_filter with the fuction convert2table above.
    """
    return panflute.run_filter(
        panflute.yaml_filter,
        tag='table',
        function=convert2table,
        strict_yaml=True
    )

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import json

from braces.views import LoginRequiredMixin, _access
from django.conf import settings
from django.http import HttpResponse, Http404
from django.utils.decorators import method_decorator
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import TemplateView, DetailView
from django.core.exceptions import PermissionDenied

from rest_framework import viewsets, status
from rest_framework.response import Response
from rest_framework.permissions import IsAuthenticated

from core.models import Course, CourseStudent, Class, CertificateTemplate, CourseCertification
from core.views import ClassViewSet
from core.permissions import IsProfessorCoordinatorOrAdminPermissionOrReadOnly
from core.serializers import CertificateTemplateImageSerializer, CourseCertificationSerializer

from accounts.models import TimtecUser
from accounts.views import GroupViewSet, GroupAdminViewSet

from paralapraca.serializers import (AnswerNotificationSerializer,
    UnreadNotificationSerializer, UserInDetailSerializer,
    UsersByClassSerializer, ContractSerializer, ContractGroupSerializer)
from paralapraca.models import (AnswerNotification, UnreadNotification,
                                Contract, CertificateData)
from paralapraca.serializers import ContractGroupAdminSerializer, ContractClassSerializer, \
    CertificateDataSerializer, CertificateImageDataSerializer

from discussion.models import Comment, CommentLike, Topic, TopicLike
from rest_pandas import PandasViewSet
from rest_pandas.renderers import PandasCSVRenderer, PandasJSONRenderer
import pandas as pd

from administration.views import AdminMixin

ROCKET_CHAT = {
    'address': 'http://chat.paralapraca.org.br',
    'service': 'paralapraca'
}

class ChatScreenView(TemplateView):
    template_name = 'chat.html'

    def get_context_data(self, **kwargs):
        context = super(ChatScreenView, self).get_context_data(**kwargs)
        context['rocketchat'] = getattr(settings, 'ROCKET_CHAT', ROCKET_CHAT)
        return context

class RocketchatIframeAuthView(TemplateView):
    template_name = 'rocketchat.html'

    @method_decorator(csrf_exempt)
    def dispatch(self, request, *args, **kwargs):
        return super(RocketchatIframeAuthView, self).dispatch(request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(RocketchatIframeAuthView, self).get_context_data(**kwargs)
        context['rocketchat'] = getattr(settings, 'ROCKET_CHAT', ROCKET_CHAT)
        return context

    def get(self, request, *argz, **kwargz):
        response = super(RocketchatIframeAuthView, self).get(request, *argz, **kwargz)
        return response

    def post(self, request, *argz, **kwargz):
        origin = request.META.get('HTTP_ORIGIN', '')
        token = request.COOKIES.get('rc_token', '')

        response = HttpResponse(json.dumps({
            'token': token
        }))

        response['Content-Type'] = 'application/json'
        response['Access-Control-Allow-Origin'] = origin
        response['Access-Control-Allow-Credentials'] = 'true'
        response['Access-Control-Allow-Methods'] = 'GET,PUT,POST,DELETE'
        response["Access-Control-Allow-Headers"] = "Origin, X-Requested-With, Content-Type, Accept"

        return response

class ContractViewSet(viewsets.ModelViewSet):
    queryset = Contract.objects.all()
    serializer_class = ContractSerializer

class AnswerNotificationViewSet(viewsets.ModelViewSet):
    """
    """

    queryset = AnswerNotification.objects.all()
    serializer_class = AnswerNotificationSerializer
    permission_classes = [IsAuthenticated]
    lookup_field = 'topic'

    def get_queryset(self):
        queryset = super(AnswerNotificationViewSet, self).get_queryset()
        queryset = queryset.filter(user=self.request.user)

        limit_to = self.request.query_params.get('limit_to', None)
        if limit_to:
            queryset = queryset[:int(limit_to)]
        return queryset

    def update(self, request, **kwargs):
        # Find the corresponding AnswerNotification for this call and mark it as read
        try:
            notification = AnswerNotification.objects.get(topic=kwargs.get('topic'), user=request.user)
            notification.is_read = True
            notification.save(skip_date=True)
            return Response(self.get_serializer(notification).data, status=status.HTTP_200_OK)
        except AnswerNotification.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)
            pass

class UnreadNotificationViewSet(viewsets.ModelViewSet):

    queryset = UnreadNotification.objects.all()
    serializer_class = UnreadNotificationSerializer
    permission_classes = [IsAuthenticated]

    def update(self, request, pk=None):
        # Since the frontend is sending a PUT request to this view, the counter must be reset
        unread = UnreadNotification.objects.get(user=self.request.user)
        unread.counter = 0
        unread.save()
        return Response(self.get_serializer(unread).data)

    def get_queryset(self):
        # If the user is new, a new UnreadNotification must be created
        queryset = UnreadNotification.objects.filter(user=self.request.user)
        if queryset.count() == 0:
            UnreadNotification.objects.create(user=self.request.user)
            queryset = UnreadNotification.objects.filter(user=self.request.user)
        return queryset

class SummaryViewSet(viewsets.ViewSet):

    permission_classes = [IsAuthenticated]

    def list(self, request):
        courses = Course.objects.all()
        stats = []
        for course in courses:
            course_stats = {
                'name': course.name,
                'user_count': course.coursestudent_set.count(),
                'user_finished_course_count': 0
            }
            classes = Class.objects.filter(course=course)
            classes_stats = []
            for cclass in classes:
                cclass_stats = {
                    'name': cclass.name,
                    'user_count': cclass.get_students.count(),
                    'certificate_count': 0,
                    'user_finished': 0
                }

                course_students = cclass.get_students.all()
                certified = course_students.filter(certificate__type='certificate')
                cclass_stats['certificate_count'] += certified.count()

                for cs in course_students:
                    # if cs.course_finished:
                    #    cclass_stats['user_finished'] += 1
                    if cs.can_emmit_receipt():
                        course_stats['user_finished_course_count'] += 1
                        cclass_stats['user_finished'] += 1

                classes_stats.append(cclass_stats)

            course_stats['classes'] = classes_stats
            stats.append(course_stats)

        response = Response({
            'user_count': TimtecUser.objects.count(),
            'total_number_of_topics': Topic.objects.count(),
            'total_number_of_comments': Comment.objects.count(),
            'total_number_of_likes': TopicLike.objects.count() + CommentLike.objects.count(),
            'statistics_per_course': stats})
        return response

class UsersByGroupViewSet(PandasViewSet):

    permission_classes = [IsAuthenticated]
    renderer_classes = [PandasCSVRenderer, PandasJSONRenderer]
    queryset = TimtecUser.objects.all()

    def list(self, request, format=None):
        groups = request.query_params.get('group', None)
        if groups is not None:
            serializer = UserInDetailSerializer(self.queryset.filter(groups__name__in=groups.split(',')), many=True)
        else:
            serializer = UserInDetailSerializer(self.queryset, many=True)
        # in order to get the data in the wanted column form, I'll need to make some transformations
        return Response(self.transform_data(serializer.data))

    def transform_data(self, data):
        response = []
        for user in data:
            for course in user.get('courses'):
                user.update({
                    u'{} - progresso'.format(course['course_name']): course['percent_progress'],
                    u'{} - nome da turma'.format(course['course_name']): course['class_name'],
                    u'{} - concluiu'.format(course['course_name']): course['course_finished'],
                    u'{} - possui certificado'.format(course['course_name']): course['has_certificate']
                })
            user.pop('courses', None)
            response.append(user)
        return pd.DataFrame.from_dict(response)

class UsersByClassViewSet(PandasViewSet):

    permission_classes = [IsAuthenticated]
    queryset = CourseStudent.objects.all()

    def list(self, request, format=None):
        try:
            ids = request.query_params.get('id', None).split(',')
            ids = [int(i) for i in ids]
        except AttributeError:
            ids = []

        classes = Class.objects.all().filter(pk__in=ids)
        students = [cls.students.all() for cls in classes]
        students = [s for cls in students for s in cls]
        courses = [cls['course_id'] for cls in classes.values()]

        queryset = self.queryset
        if len(ids) > 0:
            queryset = self.queryset \
                .filter(user__in=students, course__id__in=courses)

        serializer = UsersByClassSerializer(queryset, many=True)

        return Response(pd.DataFrame
                        .from_dict(self.transform_data(serializer.data))
                        .set_index('cpf'))

    def transform_data(self, data):
        for coursestudent in data:
            for lesson in coursestudent.get(u'percent_progress_by_lesson', None):
                coursestudent.update({
                    u'Lição {} - Progresso'.format(lesson['name']): lesson['progress']
                })
                for activity in lesson.get(u'activities', None):
                    coursestudent.update({
                        u'Lição {} - Atividade {} realizada'.format(lesson['name'], activity['name']): activity['done']
                    })
            coursestudent.pop('percent_progress_by_lesson', None)
        return data

class ContractGroupViewSet(GroupViewSet):
    """
    Override group viewset add contracts.
    """
    def get_serializer_class(self):
        return ContractGroupSerializer

class ContractGroupAdminViewSet(GroupAdminViewSet):
    """
    Override group viewset add contracts.
    """
    def get_serializer_class(self):
        return ContractGroupAdminSerializer

class ContractClassViewSet(ClassViewSet):
    """
    Override group viewset add contracts.
    """

    def get_queryset(self):
        queryset = super(ContractClassViewSet, self).get_queryset()

        contract_id = self.request.query_params.get('contract')
        if contract_id:
            try:
                contracts = [Contract.objects.get(pk=contract_id), ]
            except Contract.DoesNotExist:
                return  Class.objects.none()

            queryset.filter(contract__in=contracts)

        return queryset

    def get_serializer_class(self):
        return ContractClassSerializer

class CertificateDataAdminView(AdminMixin, TemplateView, _access.AccessMixin):
    raise_exception = True

    def dispatch(self, request, *args, **kwargs):

        response = super(CertificateDataAdminView, self).dispatch(
            request, *args, **kwargs)

        if not (request.user.is_superuser or self.object.get_professor_role(request.user) == 'coordinator'):
            if self.raise_exception:  # *and* if an exception was desired
                raise PermissionDenied  # return a forbidden response.

        return response

class CertificateDataMixin(viewsets.ModelViewSet):
     def get_queryset(self):
        queryset = CertificateData.objects.all()
        course = self.request.query_params.get('course', None)
        if course:
            queryset = queryset.filter(certificate_template__course=course)

        contract = self.request.query_params.get('contract', None)
        if contract:
            queryset = queryset.filter(contract=contract)

        return queryset

class CertificateDataViewSet(LoginRequiredMixin, CertificateDataMixin, viewsets.ModelViewSet):
    model = CertificateData
    serializer_class = CertificateDataSerializer

class CertificateImageDataViewSet(CertificateDataMixin, viewsets.ModelViewSet):
    queryset = CertificateData.objects.all()
    model = CertificateData
    permission_classes = (IsProfessorCoordinatorOrAdminPermissionOrReadOnly, )

    def post(self, request, **kwargs):
        obj = self.get_object()
        errors = []

        ct_serializer = CertificateTemplateImageSerializer(
            obj.certificate_template, request.FILES)
        if ct_serializer.is_valid():
            ct_serializer.save()
        else:
            errors += ct_serializer.errors

        serializer = CertificateImageDataSerializer(obj, request.FILES)
        if serializer.is_valid():
            serializer.save()
        else:
            errors += serializer.errors

        if len(errors) > 0:
            return Response(serializer.errors,
                            status=status.HTTP_400_BAD_REQUEST)
        else:
            s = CertificateDataSerializer(obj)
            return Response(s.data, status=status.HTTP_200_OK)

class CourseCertificationDetailView(DetailView):
    model = CourseCertification
    template_name = 'certificate.html'
    slug_field = "link_hash"
    serializer_class = CourseCertificationSerializer

    def render_to_response(self, context, **response_kwargs):
        from django.core.urlresolvers import resolve

        certificate = context.get('object')
        contract = certificate.course_student\
            .get_current_class().contract.first()

        if not certificate.course_student.can_emmit_receipt():
            raise Http404

        if certificate:
            context['cert_template'] = CertificateData.objects\
                .get(certificate_template__course=certificate.course_student.course,
                     type=certificate.type, contract=contract)
            # Interpolate data into text string
            # {Nome} : certificate.student (name)
            # {CPF} : certificate.student.cpf)
            # {MODULO} : certificate.course.name)
            # {CONTRATO} : context['cert_template'].contract.name)
            # {NUM_UNIDADES} : str(certificate.course_total_units))
            # {TURMA} : certificate.course_student.get_current_class().name)

            context['cert_template'].text = context['cert_template'].text\
                .replace('{NOME}', certificate.student.get_full_name())\
                .replace('{CPF}', certificate.student.cpf)\
                .replace('{MODULO}', certificate.course.name)\
                .replace('{CONTRATO}', context['cert_template'].contract.name) \
                .replace('{NUM_UNIDADES}', str(certificate.course_total_units))\
                .replace('{TURMA}', certificate.course_student.get_current_class().name)\

        url_name = resolve(self.request.path_info).url_name

        if url_name == 'certificate-download':
            from selenium import webdriver
            from signal import SIGTERM
            from time import gmtime, strftime
            from timtec.settings import MEDIA_ROOT, CERTIFICATE_SIZE, PHANTOMJS_PATH
            from PIL import Image
            import os

            today = strftime("%d%b%Y", gmtime())

            width, height = CERTIFICATE_SIZE
            url = self.request.build_absolute_uri().split('download')[0] + 'print/'
            png_path = os.path.join(MEDIA_ROOT, certificate.link_hash + '.png')
            pdf_filename = certificate.link_hash + today + '.pdf'
            pdf_path = os.path.join(MEDIA_ROOT, pdf_filename)

            driver = webdriver.PhantomJS(executable_path=PHANTOMJS_PATH)
            driver.set_window_size(width, height)
            driver.get(url)
            driver.save_screenshot(filename=png_path)

            driver.service.process.send_signal(SIGTERM)
            driver.quit()
            Image.open(png_path).convert("RGB").save(pdf_path, format='PDF', quality=100, dpi=(300, 300))

            response = HttpResponse(content_type='application/pdf')
            response['Content-Disposition'] = 'attachment; filename=%s' % pdf_filename

            certi = open(pdf_path)
            response.write(certi.read())
            certi.close()

            return response
        else:
            return super(CourseCertificationDetailView, self)\
                .render_to_response(context, **response_kwargs)
<commit_msg>certificate settings: add support to clear images<commit_after># -*- coding: utf-8 -*-
import json

from braces.views import LoginRequiredMixin, _access
from django.conf import settings
from django.http import HttpResponse, Http404
from django.utils.decorators import method_decorator
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import TemplateView, DetailView
from django.core.exceptions import PermissionDenied

from rest_framework import viewsets, status
from rest_framework.response import Response
from rest_framework.permissions import IsAuthenticated

from core.models import Course, CourseStudent, Class, CertificateTemplate, CourseCertification
from core.views import ClassViewSet
from core.permissions import IsProfessorCoordinatorOrAdminPermissionOrReadOnly
from core.serializers import CertificateTemplateImageSerializer, CourseCertificationSerializer

from accounts.models import TimtecUser
from accounts.views import GroupViewSet, GroupAdminViewSet

from paralapraca.serializers import (AnswerNotificationSerializer,
    UnreadNotificationSerializer, UserInDetailSerializer,
    UsersByClassSerializer, ContractSerializer, ContractGroupSerializer)
from paralapraca.models import (AnswerNotification, UnreadNotification,
                                Contract, CertificateData)
from paralapraca.serializers import ContractGroupAdminSerializer, ContractClassSerializer, \
    CertificateDataSerializer, CertificateImageDataSerializer

from discussion.models import Comment, CommentLike, Topic, TopicLike
from rest_pandas import PandasViewSet
from rest_pandas.renderers import PandasCSVRenderer, PandasJSONRenderer
import pandas as pd

from administration.views import AdminMixin

ROCKET_CHAT = {
    'address': 'http://chat.paralapraca.org.br',
    'service': 'paralapraca'
}

class ChatScreenView(TemplateView):
    template_name = 'chat.html'

    def get_context_data(self, **kwargs):
        context = super(ChatScreenView, self).get_context_data(**kwargs)
        context['rocketchat'] = getattr(settings, 'ROCKET_CHAT', ROCKET_CHAT)
        return context

class RocketchatIframeAuthView(TemplateView):
    template_name = 'rocketchat.html'

    @method_decorator(csrf_exempt)
    def dispatch(self, request, *args, **kwargs):
        return super(RocketchatIframeAuthView, self).dispatch(request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(RocketchatIframeAuthView, self).get_context_data(**kwargs)
        context['rocketchat'] = getattr(settings, 'ROCKET_CHAT', ROCKET_CHAT)
        return context

    def get(self, request, *argz, **kwargz):
        response = super(RocketchatIframeAuthView, self).get(request, *argz, **kwargz)
        return response

    def post(self, request, *argz, **kwargz):
        origin = request.META.get('HTTP_ORIGIN', '')
        token = request.COOKIES.get('rc_token', '')

        response = HttpResponse(json.dumps({
            'token': token
        }))

        response['Content-Type'] = 'application/json'
        response['Access-Control-Allow-Origin'] = origin
        response['Access-Control-Allow-Credentials'] = 'true'
        response['Access-Control-Allow-Methods'] = 'GET,PUT,POST,DELETE'
        response["Access-Control-Allow-Headers"] = "Origin, X-Requested-With, Content-Type, Accept"

        return response

class ContractViewSet(viewsets.ModelViewSet):
    queryset = Contract.objects.all()
    serializer_class = ContractSerializer

class AnswerNotificationViewSet(viewsets.ModelViewSet):
    """
    """

    queryset = AnswerNotification.objects.all()
    serializer_class = AnswerNotificationSerializer
    permission_classes = [IsAuthenticated]
    lookup_field = 'topic'

    def get_queryset(self):
        queryset = super(AnswerNotificationViewSet, self).get_queryset()
        queryset = queryset.filter(user=self.request.user)

        limit_to = self.request.query_params.get('limit_to', None)
        if limit_to:
            queryset = queryset[:int(limit_to)]
        return queryset

    def update(self, request, **kwargs):
        # Find the corresponding AnswerNotification for this call and mark it as read
        try:
            notification = AnswerNotification.objects.get(topic=kwargs.get('topic'), user=request.user)
            notification.is_read = True
            notification.save(skip_date=True)
            return Response(self.get_serializer(notification).data, status=status.HTTP_200_OK)
        except AnswerNotification.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)
            pass

class UnreadNotificationViewSet(viewsets.ModelViewSet):

    queryset = UnreadNotification.objects.all()
    serializer_class = UnreadNotificationSerializer
    permission_classes = [IsAuthenticated]

    def update(self, request, pk=None):
        # Since the frontend is sending a PUT request to this view, the counter must be reset
        unread = UnreadNotification.objects.get(user=self.request.user)
        unread.counter = 0
        unread.save()
        return Response(self.get_serializer(unread).data)

    def get_queryset(self):
        # If the user is new, a new UnreadNotification must be created
        queryset = UnreadNotification.objects.filter(user=self.request.user)
        if queryset.count() == 0:
            UnreadNotification.objects.create(user=self.request.user)
            queryset = UnreadNotification.objects.filter(user=self.request.user)
        return queryset

class SummaryViewSet(viewsets.ViewSet):

    permission_classes = [IsAuthenticated]

    def list(self, request):
        courses = Course.objects.all()
        stats = []
        for course in courses:
            course_stats = {
                'name': course.name,
                'user_count': course.coursestudent_set.count(),
                'user_finished_course_count': 0
            }
            classes = Class.objects.filter(course=course)
            classes_stats = []
            for cclass in classes:
                cclass_stats = {
                    'name': cclass.name,
                    'user_count': cclass.get_students.count(),
                    'certificate_count': 0,
                    'user_finished': 0
                }

                course_students = cclass.get_students.all()
                certified = course_students.filter(certificate__type='certificate')
                cclass_stats['certificate_count'] += certified.count()

                for cs in course_students:
                    # if cs.course_finished:
                    #    cclass_stats['user_finished'] += 1
                    if cs.can_emmit_receipt():
                        course_stats['user_finished_course_count'] += 1
                        cclass_stats['user_finished'] += 1

                classes_stats.append(cclass_stats)

            course_stats['classes'] = classes_stats
            stats.append(course_stats)

        response = Response({
            'user_count': TimtecUser.objects.count(),
            'total_number_of_topics': Topic.objects.count(),
            'total_number_of_comments': Comment.objects.count(),
            'total_number_of_likes': TopicLike.objects.count() + CommentLike.objects.count(),
            'statistics_per_course': stats})
        return response

class UsersByGroupViewSet(PandasViewSet):

    permission_classes = [IsAuthenticated]
    renderer_classes = [PandasCSVRenderer, PandasJSONRenderer]
    queryset = TimtecUser.objects.all()

    def list(self, request, format=None):
        groups = request.query_params.get('group', None)
        if groups is not None:
            serializer = UserInDetailSerializer(self.queryset.filter(groups__name__in=groups.split(',')), many=True)
        else:
            serializer = UserInDetailSerializer(self.queryset, many=True)
        # in order to get the data in the wanted column form, I'll need to make some transformations
        return Response(self.transform_data(serializer.data))

    def transform_data(self, data):
        response = []
        for user in data:
            for course in user.get('courses'):
                user.update({
                    u'{} - progresso'.format(course['course_name']): course['percent_progress'],
                    u'{} - nome da turma'.format(course['course_name']): course['class_name'],
                    u'{} - concluiu'.format(course['course_name']): course['course_finished'],
                    u'{} - possui certificado'.format(course['course_name']): course['has_certificate']
                })
            user.pop('courses', None)
            response.append(user)
        return pd.DataFrame.from_dict(response)

class UsersByClassViewSet(PandasViewSet):

    permission_classes = [IsAuthenticated]
    queryset = CourseStudent.objects.all()

    def list(self, request, format=None):
        try:
            ids = request.query_params.get('id', None).split(',')
            ids = [int(i) for i in ids]
        except AttributeError:
            ids = []

        classes = Class.objects.all().filter(pk__in=ids)
        students = [cls.students.all() for cls in classes]
        students = [s for cls in students for s in cls]
        courses = [cls['course_id'] for cls in classes.values()]

        queryset = self.queryset
        if len(ids) > 0:
            queryset = self.queryset \
                .filter(user__in=students, course__id__in=courses)

        serializer = UsersByClassSerializer(queryset, many=True)

        return Response(pd.DataFrame
                        .from_dict(self.transform_data(serializer.data))
                        .set_index('cpf'))

    def transform_data(self, data):
        for coursestudent in data:
            for lesson in coursestudent.get(u'percent_progress_by_lesson', None):
                coursestudent.update({
                    u'Lição {} - Progresso'.format(lesson['name']): lesson['progress']
                })
                for activity in lesson.get(u'activities', None):
                    coursestudent.update({
                        u'Lição {} - Atividade {} realizada'.format(lesson['name'], activity['name']): activity['done']
                    })
            coursestudent.pop('percent_progress_by_lesson', None)
        return data

class ContractGroupViewSet(GroupViewSet):
    """
    Override group viewset add contracts.
    """
    def get_serializer_class(self):
        return ContractGroupSerializer

class ContractGroupAdminViewSet(GroupAdminViewSet):
    """
    Override group viewset add contracts.
    """
    def get_serializer_class(self):
        return ContractGroupAdminSerializer

class ContractClassViewSet(ClassViewSet):
    """
    Override group viewset add contracts.
    """

    def get_queryset(self):
        queryset = super(ContractClassViewSet, self).get_queryset()

        contract_id = self.request.query_params.get('contract')
        if contract_id:
            try:
                contracts = [Contract.objects.get(pk=contract_id), ]
            except Contract.DoesNotExist:
                return  Class.objects.none()

            queryset.filter(contract__in=contracts)

        return queryset

    def get_serializer_class(self):
        return ContractClassSerializer

class CertificateDataAdminView(AdminMixin, TemplateView, _access.AccessMixin):
    raise_exception = True

    def dispatch(self, request, *args, **kwargs):

        response = super(CertificateDataAdminView, self).dispatch(
            request, *args, **kwargs)

        if not (request.user.is_superuser or self.object.get_professor_role(request.user) == 'coordinator'):
            if self.raise_exception:  # *and* if an exception was desired
                raise PermissionDenied  # return a forbidden response.

        return response

class CertificateDataMixin(viewsets.ModelViewSet):
     def get_queryset(self):
        queryset = CertificateData.objects.all()
        course = self.request.query_params.get('course', None)
        if course:
            queryset = queryset.filter(certificate_template__course=course)

        contract = self.request.query_params.get('contract', None)
        if contract:
            queryset = queryset.filter(contract=contract)

        return queryset

class CertificateDataViewSet(LoginRequiredMixin, CertificateDataMixin, viewsets.ModelViewSet):
    model = CertificateData
    serializer_class = CertificateDataSerializer

class CertificateImageDataViewSet(CertificateDataMixin, viewsets.ModelViewSet):
    queryset = CertificateData.objects.all()
    model = CertificateData
    permission_classes = (IsProfessorCoordinatorOrAdminPermissionOrReadOnly, )

    def post(self, request, **kwargs):
        obj = self.get_object()
        errors = []

        clear_logos = (
            ('base_logo', request.data.get('base_logo_clear', None)),
            ('signature', request.data.get('signature_clear', None)),
            ('cert_logo', request.data.get('cert_logo_clear', None)),
        )
        for cl in clear_logos:
            if(cl[1]):
                setattr(obj.certificate_template, cl[0], None)

        ct_serializer = CertificateTemplateImageSerializer(
            obj.certificate_template, request.FILES)
        if ct_serializer.is_valid():
            ct_serializer.save()
        else:
            errors += ct_serializer.errors

        cl = ('site_logo', request.data.get('site_logo_clear', None))
        if(cl[1]):
            setattr(obj, cl[0], None)
        serializer = CertificateImageDataSerializer(obj, request.FILES)
        if serializer.is_valid():
            serializer.save()
        else:
            errors += serializer.errors

        if len(errors) > 0:
            return Response(serializer.errors,
                            status=status.HTTP_400_BAD_REQUEST)
        else:
            s = CertificateDataSerializer(obj)
            return Response(s.data, status=status.HTTP_200_OK)

class CourseCertificationDetailView(DetailView):
    model = CourseCertification
    template_name = 'certificate.html'
    slug_field = "link_hash"
    serializer_class = CourseCertificationSerializer

    def render_to_response(self, context, **response_kwargs):
        from django.core.urlresolvers import resolve

        certificate = context.get('object')
        contract = certificate.course_student\
            .get_current_class().contract.first()

        if not certificate.course_student.can_emmit_receipt():
            raise Http404

        if certificate:
            context['cert_template'] = CertificateData.objects\
                .get(certificate_template__course=certificate.course_student.course,
                     type=certificate.type, contract=contract)
            # Interpolate data into text string
            # {Nome} : certificate.student (name)
            # {CPF} : certificate.student.cpf)
            # {MODULO} : certificate.course.name)
            # {CONTRATO} : context['cert_template'].contract.name)
            # {NUM_UNIDADES} : str(certificate.course_total_units))
            # {TURMA} : certificate.course_student.get_current_class().name)

            context['cert_template'].text = context['cert_template'].text\
                .replace('{NOME}', certificate.student.get_full_name())\
                .replace('{CPF}', certificate.student.cpf)\
                .replace('{MODULO}', certificate.course.name)\
                .replace('{CONTRATO}', context['cert_template'].contract.name) \
                .replace('{NUM_UNIDADES}', str(certificate.course_total_units))\
                .replace('{TURMA}', certificate.course_student.get_current_class().name)\

        url_name = resolve(self.request.path_info).url_name

        if url_name == 'certificate-download':
            from selenium import webdriver
            from signal import SIGTERM
            from time import gmtime, strftime
            from timtec.settings import MEDIA_ROOT, CERTIFICATE_SIZE, PHANTOMJS_PATH
            from PIL import Image
            import os

            today = strftime("%d%b%Y", gmtime())

            width, height = CERTIFICATE_SIZE
            url = self.request.build_absolute_uri().split('download')[0] + 'print/'
            png_path = os.path.join(MEDIA_ROOT, certificate.link_hash + '.png')
            pdf_filename = certificate.link_hash + today + '.pdf'
            pdf_path = os.path.join(MEDIA_ROOT, pdf_filename)

            driver = webdriver.PhantomJS(executable_path=PHANTOMJS_PATH)
            driver.set_window_size(width, height)
            driver.get(url)
            driver.save_screenshot(filename=png_path)

            driver.service.process.send_signal(SIGTERM)
            driver.quit()
            Image.open(png_path).convert("RGB").save(pdf_path, format='PDF', quality=100, dpi=(300, 300))

            response = HttpResponse(content_type='application/pdf')
            response['Content-Disposition'] = 'attachment; filename=%s' % pdf_filename

            certi = open(pdf_path)
            response.write(certi.read())
            certi.close()

            return response
        else:
            return super(CourseCertificationDetailView, self)\
                .render_to_response(context, **response_kwargs)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from functools import wraps
from inspect import signature

import numpy

SYMBOLS = {
    'μ': 'mu',
    'σ': 'sigma',
    'α': 'alpha',
    'β': 'beta',
    'γ': 'gamma',
    'λ': 'lamda',
    'θ': 'theta',
}

def greco_deco(func):
    """ Decorator to let you use greek characters for fxn kwargs."""
    @wraps(func)
    def wrapper(*args, **kwargs):
        sig = signature(func)
        kwargs = {SYMBOLS.get(k, k): v for k, v in kwargs.items()}
        bound = sig.bind(*args, **kwargs)
        return func(**bound.arguments)
    return wrapper

def seed(func):
    """ Decorator to seed the RNG before any function. """
    @wraps(func)
    def wrapper(*args, **kwargs):
        numpy.random.seed(0)
        return func(*args, **kwargs)
    return wrapper

def _get_loc_scale_keys(fit=False):
    if fit:
        return 'floc', 'fscale'
    else:
        return 'loc', 'scale'

def _remove_nones(**kwargs):
    """
    Removes any kwargs whose values are `None`.
    """

    final = kwargs.copy()
    for k in kwargs:
        if kwargs[k] is None:
            final.pop(k)
    return final
<commit_msg>simplify _remove_nones function<commit_after># -*- coding: utf-8 -*-
from functools import wraps
from inspect import signature

import numpy

SYMBOLS = {
    'μ': 'mu',
    'σ': 'sigma',
    'α': 'alpha',
    'β': 'beta',
    'γ': 'gamma',
    'λ': 'lamda',
    'θ': 'theta',
}

def greco_deco(func):
    """ Decorator to let you use greek characters for fxn kwargs."""
    @wraps(func)
    def wrapper(*args, **kwargs):
        sig = signature(func)
        kwargs = {SYMBOLS.get(k, k): v for k, v in kwargs.items()}
        bound = sig.bind(*args, **kwargs)
        return func(**bound.arguments)
    return wrapper

def seed(func):
    """ Decorator to seed the RNG before any function. """
    @wraps(func)
    def wrapper(*args, **kwargs):
        numpy.random.seed(0)
        return func(*args, **kwargs)
    return wrapper

def _get_loc_scale_keys(fit=False):
    if fit:
        return 'floc', 'fscale'
    else:
        return 'loc', 'scale'

def _remove_nones(**kwargs):
    """
    Removes any kwargs whose values are `None`.
    """
    return {k: v for k, v in kwargs.items() if v is not None}

<|endoftext|>


<commit_before>import xml.etree.ElementTree as ET
import re
import math
from pprint import pprint
import sys
import codecs

def computeDataRegions(thisPage):
	tokens = thisPage.findall('.//TOKEN')

	reVoterId = re.compile('[A-Z].*[0-9]{6,}')
	reElector = re.compile("Elector's") # Words seem to be getting split in the PDF
	reRelative = re.compile("(Father|Husband|Mother)'s")
	reHouse = re.compile("House")
	rePhoto = re.compile("Photo")
	reAge = re.compile("Age")
	reSex = re.compile("Sex")
	reSerial = re.compile("[0-9]+")

	fields = [ reElector, reRelative, reHouse, rePhoto, reAge, reSex, reSerial, reVoterId ]
	fieldIds = range(len(fields))

	boxValues = [None]*len(fields)
	boxNodes = [None]*len(fields)

	valSerial = None
	valVoter = None

	rects = []

	pageTotal = 0

	# Demarcate the page into boxes that contain voter info
	for tok in tokens:
		content = tok.text
		# If the current text matches a pattern, then we'll
		# keep the text.  If a similar pattern repeats multiple
		# times, only the last one will remain. This case happens
		# with the numbers (serial number, age, maybe house no).
		# Luckily the serial number comes last in the list, but this
		# is a possible gotcha in this algorithm
		for check in zip(fieldIds,fields):
			obj = check[1].match(content)
			if obj:
				boxValues[check[0]] = obj.group()
				boxNodes[check[0]] = tok
				break
		idx = None
		try:
			idx = boxValues.index(None)
		except ValueError:
			pass
		if (idx >= 6) or (idx is None):
			xVal = []
			yVal = []
			for nv in boxNodes:
				if nv is None:
					continue
				x = float(nv.attrib['x'])
				y = float(nv.attrib['y'])
				w = float(nv.attrib['width'])
				h = float(nv.attrib['height'])
				xVal.append(x)
				xVal.append(x+w)
				yVal.append(y)
				yVal.append(y+h)

			boundingRect = [min(xVal), min(yVal), max(xVal), max(yVal)]
			rectDim = [ boundingRect[2]-boundingRect[0], boundingRect[3]-boundingRect[1]]
			rects.append(boundingRect)
			boxValues = [None]*len(fields)
			boxNodes = [None]*len(fields)
	return rects

def extractVoterInfo(textRect, textNodes, pageNo):

	v_tolerance = 1.0
	def cmpBoxFields(a, b):
		y1 = float(a.attrib['y'])
		y2 = float(b.attrib['y'])
		if math.fabs(y1-y2) > v_tolerance:
			if y1 < y2:
				return -1
			elif y1 > y2:
				return 1
		x1 = float(a.attrib['x'])
		x2 = float(b.attrib['x'])
		if x1 < x2:
			return -1
		elif x1 > x2:
			return 1
		return 0

	textNodes.sort(cmp=cmpBoxFields)

#	print '-------'
#	print textNodes[0].text,
#	prevTok = textNodes[0]
#	for tok in textNodes[1:]:
#		if float(tok.attrib['y'])>(float(prevTok.attrib['y'])+v_tolerance):
#			print
#		print tok.text,
#		prevTok = tok
#	print
#	print '-------'

	reVoterId = re.compile('[A-Z].*[0-9]{6,}')
	reElector = re.compile("Elector's") # Words seem to be getting split in the PDF
	reRelative = re.compile("(Father|Husband)'s")
	reHouse = re.compile("House")
	rePhoto = re.compile("Photo")
	reAge = re.compile("Age")
	reSex = re.compile("Sex")
	reSerial = re.compile("[0-9]+")

	info = {}

	info['page'] = pageNo

	# First item in the list needs to be the serial number
	ob = reSerial.match(textNodes[0].text)
	if ob:
		info["serial"] = ob.group()
		textNodes.pop(0)
	else:
		# If the first item is not a serial number, then
		# try to find the EPIC number and roll back by one
		#
		# Inability to find the serial number is mostly to do
		# with data that creeps into the box by virtue of
		# landing up inside the box...
		#
		idx = 1
		while True:
			#print 'considering :',textNodes[idx].text
			ob = reVoterId.match(textNodes[idx].text)
			if ob:
				#print 'matched'
				idx = idx - 1
				break
			idx = idx + 1
			if idx == len(textNodes):
				print '!!! ERROR - reached end of list'
				return None
		ob = reSerial.match(textNodes[idx].text)
		if ob:
			info["serial"] = ob.group()
			#print info["serial"]
			for i in range(idx+1):
				textNodes.pop(0)
		else:
			#print textNodes[0].text
			# possibly raise an error
			#raise RuntimeError, "First item in rect needs to be serial"
			print '!!! ERROR - did not find serial no'
			return None

	# Next item is the EPIC number. This may be missed in
	# some nodes!
	info["epic"] = None
	ob = reVoterId.match(textNodes[0].text)
	if ob:
		info["epic"] = ob.group()
		textNodes.pop(0)

	# Filter out certain keywords that will not make it into the data
	textNodes = filter(lambda x: x.text not in ['Name',':','No.', 'Photo','Not', 'Available' ], textNodes)

	appendTo = None
	info["name"] = ""
	info["relative"] = ""
	info["residence"] = ""
	info["age"] = ""
	info["sex"] = ""

	appendTo = None
	for node in textNodes:
		content = node.text
		nodeChanged = False
		for tryMatch in zip(['name','relative','residence','age','sex'], [reElector, reRelative, reHouse, reAge, reSex]):
			if tryMatch[1].match(content):
				appendTo = tryMatch[0]
				nodeChanged = True
				break
		if (not nodeChanged) and (appendTo is not None):
			info[appendTo] =( '%s %s'%(info[appendTo], content)).strip()
	#print info
	return info

def getVoterInfo(thisPage, rects, pageNo):
	def pointInRect(x, y, r):
		eps = 0.1
		if x<(r[0]-eps):
			return False
		if y<(r[1]-eps):
			return False
		if x>(r[2]-eps):
			return False
		if y>(r[3]-eps):
			return False
		return True

	tokens = thisPage.findall('.//TOKEN')

	voterInfo = []
	for thisRect in rects:
		# Figure out all the text nodes that belong to
		# this rect
		thisRectNodes = []
		for tok in tokens:
			x = float(tok.attrib['x'])
			y = float(tok.attrib['y'])
			w = float(tok.attrib['width'])
			h = float(tok.attrib['height'])
			if pointInRect(x,y, thisRect) or pointInRect(x+w, y+h, thisRect):
				thisRectNodes.append(tok)

		#
		info = extractVoterInfo(thisRect, thisRectNodes,pageNo)
		if info is not None:
			voterInfo.append(info)
	return voterInfo

doc = ET.parse('indented-vl-eng.xml')
root = doc.getroot()
pages = root.findall('PAGE')

voterInfo = []
for pageInfo in zip(range(len(pages)),pages):
	rects = computeDataRegions(pageInfo[1])
	#print 'Info about %d voters is in page %d'%(len(rects),pageInfo[0]+1)
	vInfo = getVoterInfo(pageInfo[1], rects, pageInfo[0]+1)
	if len(vInfo)>0:
		voterInfo.extend(vInfo)

fname = 'voterlist.csv'
print 'Writing data for %d voters to %s'%(len(voterInfo), fname)

f= codecs.open(fname,'w','utf-8')
for vInfo in voterInfo:
	print >>f,'%s,%s,%s,%s,%s,%s,%s,%s'%(vInfo['page'],vInfo["serial"],vInfo["epic"],vInfo["name"],vInfo["relative"],vInfo['residence'],vInfo["sex"],vInfo["age"])

f.close()
<commit_msg>Add Mother, show relation in CSV. Fix a couple of issues.<commit_after>import xml.etree.ElementTree as ET
import re
import math
from pprint import pprint
import sys
import codecs

def computeDataRegions(thisPage):
	tokens = thisPage.findall('.//TOKEN')

	reVoterId = re.compile('[A-Z].*[0-9]{6,}')
	reElector = re.compile("Elector's") # Words seem to be getting split in the PDF
	reRelative = re.compile("(Father|Husband|Mother)'s")
	reHouse = re.compile("House")
	rePhoto = re.compile("Photo")
	reAge = re.compile("Age")
	reSex = re.compile("Sex")
	reSerial = re.compile("[0-9]+")

	fields = [ reElector, reRelative, reHouse, rePhoto, reAge, reSex, reSerial, reVoterId ]
	fieldIds = range(len(fields))

	boxValues = [None]*len(fields)
	boxNodes = [None]*len(fields)

	valSerial = None
	valVoter = None

	rects = []

	pageTotal = 0

	# Demarcate the page into boxes that contain voter info
	for tok in tokens:
		content = tok.text
		# If the current text matches a pattern, then we'll
		# keep the text.  If a similar pattern repeats multiple
		# times, only the last one will remain. This case happens
		# with the numbers (serial number, age, maybe house no).
		# Luckily the serial number comes last in the list, but this
		# is a possible gotcha in this algorithm
		for check in zip(fieldIds,fields):
			obj = check[1].match(content)
			if obj:
				boxValues[check[0]] = obj.group()
				boxNodes[check[0]] = tok
				break
		idx = None
		try:
			idx = boxValues.index(None)
		except ValueError:
			pass
		if (idx >= 6) or (idx is None):
			xVal = []
			yVal = []
			for nv in boxNodes:
				if nv is None:
					continue
				x = float(nv.attrib['x'])
				y = float(nv.attrib['y'])
				w = float(nv.attrib['width'])
				h = float(nv.attrib['height'])
				xVal.append(x)
				xVal.append(x+w)
				yVal.append(y)
				yVal.append(y+h)

			boundingRect = [min(xVal), min(yVal), max(xVal), max(yVal)]
			rectDim = [ boundingRect[2]-boundingRect[0], boundingRect[3]-boundingRect[1]]
			rects.append(boundingRect)
			boxValues = [None]*len(fields)
			boxNodes = [None]*len(fields)
	return rects

def extractVoterInfo(textRect, textNodes, pageNo):

	v_tolerance = 1.0
	def cmpBoxFields(a, b):
		y1 = float(a.attrib['y'])
		y2 = float(b.attrib['y'])
		if math.fabs(y1-y2) > v_tolerance:
			if y1 < y2:
				return -1
			elif y1 > y2:
				return 1
		x1 = float(a.attrib['x'])
		x2 = float(b.attrib['x'])
		if x1 < x2:
			return -1
		elif x1 > x2:
			return 1
		return 0

	textNodes.sort(cmp=cmpBoxFields)

#	print '-------'
#	print textNodes[0].text,
#	prevTok = textNodes[0]
#	for tok in textNodes[1:]:
#		if float(tok.attrib['y'])>(float(prevTok.attrib['y'])+v_tolerance):
#			print
#		print tok.text,
#		prevTok = tok
#	print
#	print '-------'

	reVoterId = re.compile('[A-Z].*[0-9]{6,}')
	reElector = re.compile("Elector's") # Words seem to be getting split in the PDF
	reRelative = re.compile("(Father|Husband|Mother)'s")
	reHouse = re.compile("House")
	rePhoto = re.compile("Photo")
	reAge = re.compile("Age")
	reSex = re.compile("Sex")
	reSerial = re.compile("[0-9]+")

	info = {}

	info['page'] = pageNo

	# First item in the list needs to be the serial number
	ob = reSerial.match(textNodes[0].text)
	if ob:
		info["serial"] = ob.group()
		textNodes.pop(0)
	else:
		# If the first item is not a serial number, then
		# try to find the EPIC number and roll back by one
		#
		# Inability to find the serial number is mostly to do
		# with data that creeps into the box by virtue of
		# landing up inside the box...
		#
		idx = 1
		while True:
			#print 'considering :',textNodes[idx].text
			ob = reVoterId.match(textNodes[idx].text)
			if ob:
				#print 'matched'
				idx = idx - 1
				break
			idx = idx + 1
			if idx == len(textNodes):
				print '!!! ERROR - reached end of list'
				return None
		ob = reSerial.match(textNodes[idx].text)
		if ob:
			info["serial"] = ob.group()
			#print info["serial"]
			for i in range(idx+1):
				textNodes.pop(0)
		else:
			#print textNodes[0].text
			# possibly raise an error
			#raise RuntimeError, "First item in rect needs to be serial"
			print '!!! ERROR - did not find serial no'
			return None

	# Next item is the EPIC number. This may be missed in
	# some nodes!
	info["epic"] = None
	ob = reVoterId.match(textNodes[0].text)
	if ob:
		info["epic"] = ob.group()
		textNodes.pop(0)

	# Filter out certain keywords that will not make it into the data
	textNodes = filter(lambda x: x.text not in ['Name',':','No.', 'Photo','Not', 'Available' ], textNodes)

	appendTo = None
	info["name"] = ""
	info["relative"] = ""
	info["residence"] = ""
	info["age"] = ""
	info["sex"] = ""

	appendTo = None
	for node in textNodes:
		content = node.text
		nodeChanged = False
		for tryMatch in zip(['name','relative','residence','age','sex'], [reElector, reRelative, reHouse, reAge, reSex]):
			ob = tryMatch[1].match(content)
			if ob:
				appendTo = tryMatch[0]
				nodeChanged = True
				if tryMatch[0] == 'relative':
					info["relation"] = ob.groups()[0]
				break
		if (not nodeChanged) and (appendTo is not None):
			info[appendTo] =( '%s %s'%(info[appendTo], content)).strip()
	#print info
	return info

def getVoterInfo(thisPage, rects, pageNo):
	def pointInRect(x, y, r):
		eps = 0.1
		if x<(r[0]-eps):
			return False
		if y<(r[1]-eps):
			return False
		if x>(r[2]-eps):
			return False
		if y>(r[3]-eps):
			return False
		return True

	tokens = thisPage.findall('.//TOKEN')

	voterInfo = []
	for thisRect in rects:
		# Figure out all the text nodes that belong to
		# this rect
		thisRectNodes = []
		for tok in tokens:
			x = float(tok.attrib['x'])
			y = float(tok.attrib['y'])
			w = float(tok.attrib['width'])
			h = float(tok.attrib['height'])
			if pointInRect(x,y, thisRect) or pointInRect(x+w, y+h, thisRect):
				thisRectNodes.append(tok)

		#
		info = extractVoterInfo(thisRect, thisRectNodes,pageNo)
		if info is not None:
			voterInfo.append(info)
	return voterInfo

doc = ET.parse('indented-vl-eng.xml')
root = doc.getroot()
pages = root.findall('PAGE')

voterInfo = []
for pageInfo in zip(range(len(pages)),pages):
	rects = computeDataRegions(pageInfo[1])
	#print 'Info about %d voters is in page %d'%(len(rects),pageInfo[0]+1)
	vInfo = getVoterInfo(pageInfo[1], rects, pageInfo[0]+1)
	if len(vInfo)>0:
		voterInfo.extend(vInfo)

fname = 'voterlist.csv'
print 'Writing data for %d voters to %s'%(len(voterInfo), fname)

f= codecs.open(fname,'w','utf-8')
for vInfo in voterInfo:
	#pprint(vInfo)
	print >>f,'%s,%s,%s,%s,%s,%s,%s,%s,%s'%(vInfo['page'],vInfo["serial"],vInfo["epic"],vInfo["name"],vInfo['relation'],vInfo["relative"],vInfo['residence'],vInfo["sex"],vInfo["age"])

f.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Apr 22 23:20:13 2017

@author: colm
"""

from bs4 import BeautifulSoup
import requests
import time
import json
from configparser import ConfigParser

def get_google_api_key(config_file = '/home/colm/alchemy-keys/google-api-keys.ini'):
  parser = ConfigParser()
  retval = parser.read(config_file)
  if len(retval) != 1:
    print('Error: could not find '+config_file)
    raise FileNotFoundError

  cx = parser['google_api_keys']['cx']
  key = parser['google_api_keys']['key']

  return cx, key

def update_logo_from_google(charity, cx, key):

  #print('Fetching logo for '+charity)
  r = requests.get('https://www.googleapis.com/customsearch/v1?q='+charity+' charity logo&searchType=image&cx='+cx+'&key='+key+'M&num=1')
  try:
    link = r.json()['items'][0]['link']
  except KeyError:
    link = ''

    time.sleep(1) # don't overload API with requests. Need to also bear in mind restriction of 100 reqs a day

  return link

def repair_charities(charity_dict, cx, api_key):

  repair_list = []

  for charity, info in charity_dict.items():
    if info['logo'] == '':
      repair_list.append(charity)

  if len(repair_list) > 0:
    print('Repairing '+str(len(repair_list))+' charities')
    for charity in repair_list:
      charity_dict[charity]['logo'] = update_logo_from_google(charity, cx, api_key)

  return charity_dict

# refresh list of charities
# load latest list
# search for more
#     - by category if possible
# if new ones are detected, get logos for them
# perform check for missing logos - get logos for them

def refresh_charities():

  try:
    with open('charity_info.json', 'r') as f:
      old_list = json.load(f)
  except FileNotFoundError:
    old_list = {}

  cx, api_key = get_google_api_key()

  r = requests.get('https://api.likecharity.com/charities/')

  soup = BeautifulSoup(r.text, "lxml")

  categories_soup = soup.findAll("div", { "class" : "filter--categories" })
  categories_soup = categories_soup[0].findAll("option")
  categories = []
  for category in categories_soup:
    cat_name = category.findAll(text=True)[0]
    if 'Choose Category' != cat_name:
      categories.append(cat_name)

  print(categories)
  charity_dict = {}

  for category in categories:

    print('Working on '+category)

    r = requests.post('https://api.likecharity.com/charities/', data = {'category_name':category})

    soup = BeautifulSoup(r.text, "lxml")

    charities = soup.findAll("h3", { "class" : "charity" })
    charities_key_value = soup.findAll("div", { "class" : "keywords" })

    # get the info for the charities

    for charity, charity_key_value in zip(charities, charities_key_value):
      payload = {}
      payload['donation_list'] = {}
      payload['category'] = category
      payload['logo'] = ''
      payload['number'] = '50300'
      for entry in charity_key_value.findAll(text=True):
        key = entry.split(' - ')
        if len(key) != 2:
          print('Bad key value splitting')
          raise(Exception)
        payload['donation_list'][key[0]] = key[1]

      charity_name = ''.join(charity.findAll(text=True))
      charity_dict[charity_name] =  payload

  new_charities = set(charity_dict.keys()) - set(old_list.keys())

  if len(new_charities) > 0:
    print('New charities detected. Pulling logos from google images.')
    for charity in new_charities:
      charity_dict[charity]['logo'] = update_logo_from_google(charity, cx, api_key)

  charity_dict = repair_charities(charity_dict, cx, api_key)

  with open('charity_info.json', 'w') as f:
    json.dump(charity_dict, f)

  return categories, charity_dict

if __name__ == "__main__":
  categories, charity_dict = refresh_charities()

<commit_msg>Better logging.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Apr 22 23:20:13 2017

@author: colm
"""

from bs4 import BeautifulSoup
import requests
import time
import json
from configparser import ConfigParser

def get_google_api_key(config_file = '/home/colm/alchemy-keys/google-api-keys.ini'):
  parser = ConfigParser()
  retval = parser.read(config_file)
  if len(retval) != 1:
    print('Error: could not find '+config_file)
    raise FileNotFoundError

  cx = parser['google_api_keys']['cx']
  key = parser['google_api_keys']['key']

  return cx, key

def update_logo_from_google(charity, cx, key):

  #print('Fetching logo for '+charity)
  r = requests.get('https://www.googleapis.com/customsearch/v1?q='+charity+' charity logo&searchType=image&cx='+cx+'&key='+key+'M&num=1')
  try:
    link = r.json()['items'][0]['link']
  except KeyError:
    link = ''

    time.sleep(1) # don't overload API with requests. Need to also bear in mind restriction of 100 reqs a day

  return link

def repair_charities(charity_dict, cx, api_key):

  repair_list = []

  for charity, info in charity_dict.items():
    if info['logo'] == '':
      repair_list.append(charity)

  if len(repair_list) > 0:
    print('Repairing '+str(len(repair_list))+' charities')
    for charity in repair_list:
      charity_dict[charity]['logo'] = update_logo_from_google(charity, cx, api_key)

  return charity_dict

# refresh list of charities
# load latest list
# search for more
#     - by category if possible
# if new ones are detected, get logos for them
# perform check for missing logos - get logos for them

def refresh_charities():

  try:
    with open('charity_info.json', 'r') as f:
      old_list = json.load(f)
      print(str(len(old_list)) + 'charities loaded from stored json.')
  except FileNotFoundError:
    old_list = {}
    print('No stored json found, will start from scratch.')

  cx, api_key = get_google_api_key()

  r = requests.get('https://api.likecharity.com/charities/')

  soup = BeautifulSoup(r.text, "lxml")

  categories_soup = soup.findAll("div", { "class" : "filter--categories" })
  categories_soup = categories_soup[0].findAll("option")
  categories = []
  for category in categories_soup:
    cat_name = category.findAll(text=True)[0]
    if 'Choose Category' != cat_name:
      categories.append(cat_name)

  print(categories)
  charity_dict = {}

  for category in categories:

    print('Working on '+category)

    r = requests.post('https://api.likecharity.com/charities/', data = {'category_name':category})

    soup = BeautifulSoup(r.text, "lxml")

    charities = soup.findAll("h3", { "class" : "charity" })
    charities_key_value = soup.findAll("div", { "class" : "keywords" })

    # get the info for the charities

    for charity, charity_key_value in zip(charities, charities_key_value):
      payload = {}
      payload['donation_list'] = {}
      payload['category'] = category
      payload['logo'] = ''
      payload['number'] = '50300'
      for entry in charity_key_value.findAll(text=True):
        key = entry.split(' - ')
        if len(key) != 2:
          print('Bad key value splitting')
          raise(Exception)
        payload['donation_list'][key[0]] = key[1]

      charity_name = ''.join(charity.findAll(text=True))
      charity_dict[charity_name] =  payload

  new_charities = set(charity_dict.keys()) - set(old_list.keys())

  if len(new_charities) > 0:
    print(str(len(new_charities))+' new charities detected. Pulling logos from google images.')
    for charity in new_charities:
      charity_dict[charity]['logo'] = update_logo_from_google(charity, cx, api_key)

  charity_dict = repair_charities(charity_dict, cx, api_key)

  with open('charity_info.json', 'w') as f:
    json.dump(charity_dict, f)

  return categories, charity_dict

if __name__ == "__main__":
  categories, charity_dict = refresh_charities()

<|endoftext|>


<commit_before>from nltk.stem import SnowballStemmer
from nltk.stem.api import StemmerI
import nltk
import json

class ParticleStemmer(SnowballStemmer):

	def __init__(self, language="english", ignore_stopwords=False, suffix_rule_list={}):
		super().__init__(language=language, ignore_stopwords=ignore_stopwords)
		if language == "english":
			self.stemmer._EnglishStemmer__special_words.update({
				"experiment":"experiment",
				"experimented":"experiment",
				"experimenting":"experiment",
				"experiments":"experiment",
				'organization': 'organiz',
				"organization's": 'organiz',
				'organizational': 'organiz',
				'organizationally': 'organiz',
				'organizations': 'organiz',
				'organize': 'organiz',
				'organized': 'organiz',
				'organizer': 'organiz',
				'organizers': 'organiz',
				'organizes': 'organiz',
				'organizing': 'organiz',
				'science': 'scient',
				'sciences': 'scient',
				'scientific': 'scient',
				'scientifically': 'scient',
				'scientist': 'scient',
				'scientistic': 'scient',
				'scientists': 'scient',
				})

			from partstem.word_list import word_list
			self.word_list = word_list
			self.word_list += nltk.corpus.words.words()

			self.stem = self.__stem
			self.suffix_rule_list = {
				'ant': {"with": ['ation'], "exception": []},
				'eti': {"with": ['ant', ''], "exception": []},
				'or': {"with": ['ion'], "exception": []},
				'um': {"with": ['a'], "exception": ["medium"]},
				'a': {"with": ['um'], "exception": ["media"]},
				'ri': {"with": [' -ried'], "exception": []},
				'er': {"with": ['y'], "exception": []},
				'al': {"with": ['us'], "exception": ["animal"]},
				'us': {"with": ['al'], "exception": []},
				'ifi': {"with": ['e'], "exception": []},
				'e': {"with": ['ification'], "exception": []},
				'ion': {"with": ['e'], "exception": []},
				'i': {"with": ['e', 'us'], "exception": []},
				'si': {"with": ['sis'], "exception": []},
				's': {"with": ['sis'], "exception": []},
				't': {"with": ['sis'], "exception": []},
				'z': {"with": ['sis'], "exception": []},
				"ier": {"with": ["ying", ""], "exception": []},
				"abl": {"with": ["es", ""], "exception": ["stabl", "capabl"]},
				"th": {"with": [""], "exception": []},
				"atori": {"with": ["ation"], "exception": []},
				"ori": {"with": ["ion"], "exception": []},
				"ous": {"with": ["y", "", "e", "on", "ity"], "exception": []},
				"ic": {"with": ["", "e"], "exception": ["sonic"]},
				"iti": {"with": ["est+ification"], "exception": []},
				"iz": {"with": ["ize", "izate"], "exception": []},
				"at": {"with": ["atic"], "exception": []},
				'if': {"with": ["ity+est", "e"], "exception": []}
			}
			self.suffix_rule_list.update(suffix_rule_list)
			self.suffix_list = sorted(list(self.suffix_rule_list.keys()), key=lambda x: -len(x))

	def __stem(self, word, return_snowball=False):
		if not word.startswith("improv"):
			remove_suffix = {"isate":"izate", "isated":"izated", "isating":"izating", "isates":"izates"}
			for key in remove_suffix.keys():
				if word.endswith(remove_suffix[key]):
					word = word[:-len(key)] + remove_suffix[key]
					break

			remove_suffix = {"ise":"ize", "ised":"ized", "ising":"izing", "ises":"izes"}
			for key in remove_suffix.keys():
				if word.endswith(remove_suffix[key]):
					new_word = word[:-len(key)] + remove_suffix[key]
					if new_word in self.word_list:
						word = new_word
						break

		word = self.stemmer.stem(word)
		stem_word = word
		num = 0
		if word not in list(self.stemmer._EnglishStemmer__special_words.keys()) + list(self.stemmer._EnglishStemmer__special_words.values()):
			while num < len(self.suffix_list):
				if stem_word.endswith(self.suffix_list[num]) and stem_word not in self.suffix_rule_list[self.suffix_list[num]]["exception"]:
					without_suffix = stem_word[:-len(self.suffix_list[num])]
					if len(without_suffix) == 0:
						num += 1
						continue
					for el in self.suffix_rule_list[self.suffix_list[num]]["with"]:

						el = el.replace("+", " ")
						el = el.replace("-", " -") if "-" in el and " -" not in el else el
						el = el.split(" ")
						key = True
						for el1 in el:
							if not ((without_suffix + el1 in self.word_list and not el1.startswith("-")) or (without_suffix + el1.replace("-", "") not in self.word_list and el1.startswith("-"))):
								key = False
								break
						if key:
							stem_word = without_suffix
							break
					break
				num += 1

			return (stem_word, word) if return_snowball else stem_word
			<commit_msg>new rules<commit_after>from nltk.stem import SnowballStemmer
from nltk.stem.api import StemmerI
import nltk
import json

class ParticleStemmer(SnowballStemmer):

	def __init__(self, language="english", ignore_stopwords=False, suffix_rule_list={}):
		super().__init__(language=language, ignore_stopwords=ignore_stopwords)
		if language == "english":
			self.stemmer._EnglishStemmer__special_words.update({
				"experiment":"experiment",
				"experimented":"experiment",
				"experimenting":"experiment",
				"experiments":"experiment",
				'organization': 'organiz',
				"organization's": 'organiz',
				'organizational': 'organiz',
				'organizationally': 'organiz',
				'organizations': 'organiz',
				'organize': 'organiz',
				'organized': 'organiz',
				'organizer': 'organiz',
				'organizers': 'organiz',
				'organizes': 'organiz',
				'organizing': 'organiz',
				'science': 'scient',
				'sciences': 'scient',
				'scientific': 'scient',
				'scientifically': 'scient',
				'scientist': 'scient',
				'scientistic': 'scient',
				'scientists': 'scient',
				})

			from partstem.word_list import word_list
			self.word_list = word_list
			self.word_list += nltk.corpus.words.words()

			self.stem = self.__stem
			self.suffix_rule_list = {
				'ant': {"with": ['ation'], "exception": []},
				'eti': {"with": ['ant', ''], "exception": []},
				'or': {"with": ['ion'], "exception": []},
				'um': {"with": ['a'], "exception": ["medium"]},
				'a': {"with": ['um'], "exception": ["media"]},
				'ri': {"with": [' -ried'], "exception": []},
				'er': {"with": ['y'], "exception": []},
				'al': {"with": ['us'], "exception": ["animal"]},
				'us': {"with": ['al'], "exception": []},
				'ifi': {"with": ['e'], "exception": []},
				'e': {"with": ['ification'], "exception": []},
				'ion': {"with": ['e'], "exception": []},
				'i': {"with": ['e', 'us'], "exception": []},
				'si': {"with": ['sis'], "exception": []},
				's': {"with": ['sis'], "exception": []},
				't': {"with": ['sis'], "exception": []},
				'z': {"with": ['sis'], "exception": []},
				"ier": {"with": ["ying", ""], "exception": []},
				"abl": {"with": ["es", ""], "exception": ["stabl", "capabl"]},
				"th": {"with": [""], "exception": []},
				"atori": {"with": ["ation"], "exception": []},
				"ori": {"with": ["ion"], "exception": []},
				"ous": {"with": ["y", "", "e", "on", "ity"], "exception": []},
				"ic": {"with": ["", "e"], "exception": ["sonic"]},
				"iti": {"with": ["est+ification"], "exception": []},
				"iz": {"with": ["ize", "izate"], "exception": []},
				"at": {"with": ["atic"], "exception": []},
				'if': {"with": ["ity+est", "e"], "exception": []}
			}
			self.suffix_rule_list.update(suffix_rule_list)
			self.suffix_list = sorted(list(self.suffix_rule_list.keys()), key=lambda x: -len(x))

	def __stem(self, word, return_snowball=False):
		if not word.startswith("improv"):
			remove_suffix = {"isate":"izate", "isated":"izated", "isating":"izating", "isates":"izates"}
			for key in remove_suffix.keys():
				if word.endswith(remove_suffix[key]):
					word = word[:-len(key)] + remove_suffix[key]
					break

			remove_suffix = {"ise":"ize", "ised":"ized", "ising":"izing", "ises":"izes"}
			for key in remove_suffix.keys():
				if word.endswith(remove_suffix[key]):
					new_word = word[:-len(key)] + remove_suffix[key]
					if new_word in self.word_list:
						word = new_word
						break

		word = self.stemmer.stem(word)
		stem_word = word
		num = 0
		if word not in list(self.stemmer._EnglishStemmer__special_words.keys()) + list(self.stemmer._EnglishStemmer__special_words.values()):
			while num < len(self.suffix_list):
				if stem_word.endswith(self.suffix_list[num]) and stem_word not in self.suffix_rule_list[self.suffix_list[num]]["exception"]:
					without_suffix = stem_word[:-len(self.suffix_list[num])]
					if len(without_suffix) == 0:
						num += 1
						continue
					for el in self.suffix_rule_list[self.suffix_list[num]]["with"]:

						el = el.replace("+", " ")
						el = el.replace("-", " -") if "-" in el and " -" not in el else el
						el = el.split(" ")
						key = True
						for el1 in el:
							if not ((without_suffix + el1 in self.word_list and not el1.startswith("-")) or (without_suffix + el1.replace("-", "") not in self.word_list and el1.startswith("-"))):
								key = False
								break
						if key:
							stem_word = without_suffix
							break
					break
				num += 1

		return (stem_word, word) if return_snowball else stem_word
			<|endoftext|>