<commit_before># pylint: disable=mixed-indentation
"""
Extract data from a table present in a form
"""

import json
from copy import deepcopy
from common.utils.logging_handler import Logger

class TableExtractor:
  """
  Extract data from a table present in the form
  """
  def __init__(self, json_path):
    self.json_path = json_path
    # master_dict --> page_num > tables > table_num > table data
    self.master_dict = {}

    with open(json_path, encoding="utf-8") as f_obj:
      self.data = json.load(f_obj)
    self.table_attributes()

  def table_attributes(self):
    """
    This function obtains information regarding all the tables.
    For ex. total tables, table header info, table row wise data
		in dataframe format
    """

    if "pages" in self.data.keys():
      # Iterate over pages
      for pg_num, page in enumerate(self.data["pages"]):

        page_data = {}
        if "tables" in page.keys():

          # Iterate over tables
          for table_num, table in enumerate(page["tables"]):

            # extract header(columns)
            if "bodyRows" in table and "headerRows" in table:
              for _, hrow in enumerate(table["headerRows"]):
                header_row = [
                  TableExtractor.get_text(
                    cell["layout"], self.data) for cell in hrow["cells"]
                ]
                columns = []
                for val, conf, cord in header_row:
                  if val is None:
                    columns.append(val, conf, cord)
                  else:
                    columns.append((" ".join(val.split()), conf, cord))
                table_data = {"headers": columns}
                col_data = {}
                try:
                  for row_num, row in enumerate(table["bodyRows"]):
                    row_data = [
                        TableExtractor.get_text(
                          cell["layout"], self.data) for cell in row["cells"]
                    ]
                    for i_col in range(len(header_row)):
                      entity_val, conf, coordinates = row_data[i_col]
                      col_data[i_col] = {
                        "value": entity_val,
                        "extraction_confidence": conf,
                        "value_coordinates": coordinates,
                        "manual_extraction": False,
                        "corrected_value": None
                      }
                    table_data[row_num] = {"rows": deepcopy(col_data)}

                except ValueError as e:
                  Logger.error(e)
                  return "Table Empty !!!"

              page_data[table_num] = table_data
              page_data["height"] = page["dimension"]["height"]
              page_data["width"] = page["dimension"]["width"]
          self.master_dict[pg_num] = page_data
    else:
      Logger.error("no data found in table")
      return None

  @staticmethod
  def get_text(el, data):
    """Convert text offset indexes into text snippets."""
    text = ""

    # Span over the textSegments
    if "textAnchor" in el.keys():
      if "textSegments" in el["textAnchor"].keys():
        for segment in el["textAnchor"]["textSegments"]:
          # Check for startIndex. If not present = 0
          if "startIndex" in segment.keys():
            start_index = segment["startIndex"]
          else:
            start_index = 0
          # Check for endIndex. If not present = 0
          if "endIndex" in segment.keys():
            end_index = segment["endIndex"]
          else:
            end_index = 0
          text += data["text"][int(start_index) : int(end_index)]
          cell_conf = el["confidence"]
          cell_coordinates = el["boundingPoly"]["normalizedVertices"]
          coordinates = []
          for bb_cord in cell_coordinates:
            coordinates.append(deepcopy(bb_cord['x']))
            coordinates.append(deepcopy(bb_cord['y']))

    if text in ("", None):
      text = cell_conf = coordinates = None
    return (text, cell_conf, coordinates)

  @staticmethod
  def compare_lists(master_list, sub_list):
    """Compare two list and return the avg match percentage

    Args:
        list1 (list): list with items
        list2 (list): list with items
    """
    x = lambda x: x in master_list

    matched = list(filter(x, sub_list))
    return len(matched)/len(master_list)

  @staticmethod
  def get_table_using_header(page, inp_header):
    """uses the page info to extract the table

    Args:
        page (dict): dict that contains a table info
        inp_header (list): list of column names to
				 match with the header of a table
    """

    for pg_num in page:
      for table_num in page[pg_num]:
        table_dict = page[pg_num][table_num]
        table_header = [val[0] for val in table_dict["headers"]]
        if TableExtractor.compare_lists(table_header, inp_header) >= 0.70:
          return table_dict, table_header
        else:
          continue
    Logger.error("Input headers does not match up to 70% with any table.")
    return None

  def get_entities(self, table_entities):
    """
    Extract data from table based on user specific inputs for a table.

    Args:
      table_entities (list): user specified table parameters

    Returns:
      out(list): extracted entities
    """

    if table_entities["isheader"]:
      inp_header = table_entities["headers"]
      if isinstance(table_entities["table_num"], int):
        table_num = table_entities["table_num"]
      else:
        table_num = 0
      if isinstance(table_entities["page_num"], int):
        page_num = table_entities["page_num"]
      else:
        page_num = 0

      try:
        if table_num > 0 and page_num > 0:
          table_dict = self.master_dict[page_num][table_num]
          columns = [val[0] for val in table_dict["headers"]]
          if TableExtractor.compare_lists(columns, inp_header) < 0.70:
            return "Table does not match with the headers provided"
        # if no table and page info provided.Iterate over all the pages to find
        # the table based on header
        elif page_num == 0 and table_num == 0:
          table_data = TableExtractor.get_table_using_header(
            self.master_dict, inp_header)
          if table_data:
            table_dict, columns = table_data
          else:
            return None

        elif page_num > 0 and table_num == 0:
          if page_num not in self.master_dict:
            return "page not found"
          page_dict = self.master_dict[page_num]
          table_dict, columns = TableExtractor.get_table_using_header(
            page_dict, inp_header)
        else:
          return "Operation cannot be performed. Check your config"
      except ValueError as e:
        Logger.error("Check your config")
        return None
      out = []

      for user_inp in table_entities["entity_extraction"]:
        try:
          entity_data = {}
          suffix, row = user_inp["entity_suffix"], user_inp["row_no"]
          col = user_inp["col"]
          row_dict = table_dict[row]["rows"]
          if suffix in (None, ""):
            suffix = ""
          entity_name = f"{columns[col]} {suffix}"
          entity_data = row_dict[col]
          entity_data["entity"] = entity_name
          entity_data["key_coordinates"] = table_dict["headers"][col][2]

          out.append(deepcopy(entity_data))
        except ValueError as e:
          Logger.error(e)
          continue
      return out
    else:
      Logger.error("No header present in the table. Table not extracted.")
      return None

<commit_msg>linting<commit_after># pylint: disable=mixed-indentation
"""
Extract data from a table present in a form
"""

import json
from copy import deepcopy
from common.utils.logging_handler import Logger

class TableExtractor:
  """
  Extract data from a table present in the form
  """
  def __init__(self, json_path):
    self.json_path = json_path
    # master_dict --> page_num > tables > table_num > table data
    self.master_dict = {}

    with open(json_path, encoding="utf-8") as f_obj:
      self.data = json.load(f_obj)
    self.table_attributes()

  def table_attributes(self):
    """
    This function obtains information regarding all the tables.
    For ex. total tables, table header info, table row wise data
		in dataframe format
    """

    if "pages" in self.data.keys():
      # Iterate over pages
      for pg_num, page in enumerate(self.data["pages"]):

        page_data = {}
        if "tables" in page.keys():

          # Iterate over tables
          for table_num, table in enumerate(page["tables"]):

            # extract header(columns)
            if "bodyRows" in table and "headerRows" in table:
              for _, hrow in enumerate(table["headerRows"]):
                header_row = [
                  TableExtractor.get_text(
                    cell["layout"], self.data) for cell in hrow["cells"]
                ]
                columns = []
                for val, conf, cord in header_row:
                  if val is None:
                    columns.append(val, conf, cord)
                  else:
                    columns.append((" ".join(val.split()), conf, cord))
                table_data = {"headers": columns}
                col_data = {}
                try:
                  for row_num, row in enumerate(table["bodyRows"]):
                    row_data = [
                        TableExtractor.get_text(
                          cell["layout"], self.data) for cell in row["cells"]
                    ]
                    for i_col in range(len(header_row)):
                      entity_val, conf, coordinates = row_data[i_col]
                      col_data[i_col] = {
                        "value": entity_val,
                        "extraction_confidence": conf,
                        "value_coordinates": coordinates,
                        "manual_extraction": False,
                        "corrected_value": None
                      }
                    table_data[row_num] = {"rows": deepcopy(col_data)}

                except ValueError as e:
                  Logger.error(e)
                  return "Table Empty !!!"

              page_data[table_num] = table_data
              page_data["height"] = page["dimension"]["height"]
              page_data["width"] = page["dimension"]["width"]
          self.master_dict[pg_num] = page_data
    else:
      Logger.error("no data found in table")
      return None

  @staticmethod
  def get_text(el, data):
    """Convert text offset indexes into text snippets."""
    text = ""

    # Span over the textSegments
    if "textAnchor" in el.keys():
      if "textSegments" in el["textAnchor"].keys():
        for segment in el["textAnchor"]["textSegments"]:
          # Check for startIndex. If not present = 0
          if "startIndex" in segment.keys():
            start_index = segment["startIndex"]
          else:
            start_index = 0
          # Check for endIndex. If not present = 0
          if "endIndex" in segment.keys():
            end_index = segment["endIndex"]
          else:
            end_index = 0
          text += data["text"][int(start_index) : int(end_index)]
          cell_conf = el["confidence"]
          cell_coordinates = el["boundingPoly"]["normalizedVertices"]
          coordinates = []
          for bb_cord in cell_coordinates:
            coordinates.append(deepcopy(bb_cord["x"]))
            coordinates.append(deepcopy(bb_cord["y"]))

    if text in ("", None):
      text = cell_conf = coordinates = None
    return (text, cell_conf, coordinates)

  @staticmethod
  def compare_lists(master_list, sub_list):
    """Compare two list and return the avg match percentage

    Args:
        list1 (list): list with items
        list2 (list): list with items
    """
    x = lambda x: x in master_list

    matched = list(filter(x, sub_list))
    return len(matched)/len(master_list)

  @staticmethod
  def get_table_using_header(page, inp_header):
    """uses the page info to extract the table

    Args:
        page (dict): dict that contains a table info
        inp_header (list): list of column names to
				 match with the header of a table
    """

    for pg_num in page:
      for table_num in page[pg_num]:
        table_dict = page[pg_num][table_num]
        table_header = [val[0] for val in table_dict["headers"]]
        if TableExtractor.compare_lists(table_header, inp_header) >= 0.70:
          return table_dict, table_header
        else:
          continue
    Logger.error("Input headers does not match up to 70% with any table.")
    return None

  def get_entities(self, table_entities):
    """
    Extract data from table based on user specific inputs for a table.

    Args:
      table_entities (list): user specified table parameters

    Returns:
      out(list): extracted entities
    """

    if table_entities["isheader"]:
      inp_header = table_entities["headers"]
      if isinstance(table_entities["table_num"], int):
        table_num = table_entities["table_num"]
      else:
        table_num = 0
      if isinstance(table_entities["page_num"], int):
        page_num = table_entities["page_num"]
      else:
        page_num = 0

      try:
        if table_num > 0 and page_num > 0:
          table_dict = self.master_dict[page_num][table_num]
          columns = [val[0] for val in table_dict["headers"]]
          if TableExtractor.compare_lists(columns, inp_header) < 0.70:
            return "Table does not match with the headers provided"
        # if no table and page info provided.Iterate over all the pages to find
        # the table based on header
        elif page_num == 0 and table_num == 0:
          table_data = TableExtractor.get_table_using_header(
            self.master_dict, inp_header)
          if table_data:
            table_dict, columns = table_data
          else:
            return None

        elif page_num > 0 and table_num == 0:
          if page_num not in self.master_dict:
            return "page not found"
          page_dict = self.master_dict[page_num]
          table_dict, columns = TableExtractor.get_table_using_header(
            page_dict, inp_header)
        else:
          return "Operation cannot be performed. Check your config"
      except ValueError as e:
        Logger.error(e)
        return None
      out = []

      for user_inp in table_entities["entity_extraction"]:
        try:
          entity_data = {}
          suffix, row = user_inp["entity_suffix"], user_inp["row_no"]
          col = user_inp["col"]
          row_dict = table_dict[row]["rows"]
          if suffix in (None, ""):
            suffix = ""
          entity_name = f"{columns[col]} {suffix}"
          entity_data = row_dict[col]
          entity_data["entity"] = entity_name
          entity_data["key_coordinates"] = table_dict["headers"][col][2]

          out.append(deepcopy(entity_data))
        except ValueError as e:
          Logger.error(e)
          continue
      return out
    else:
      Logger.error("No header present in the table. Table not extracted.")
      return None

<|endoftext|>


<commit_before>from datetime import datetime, time
import json
import unicodecsv as csv

from django import http
from django.urls import reverse_lazy
from django.shortcuts import render, get_object_or_404
from django.urls import reverse
from django.contrib.auth.models import User
from django.contrib import messages
from django.conf import settings
from django.utils import timezone
from django.utils.decorators import method_decorator
from django.utils.translation import ugettext as _
from django.views.generic.base import View
from django.views.generic import ListView
from django.views.generic import TemplateView
from django.views.generic.edit import FormView
from django.db.models.expressions import RawSQL
from django.db.models import Count
from django.db.models import Q
from django.db.models import Prefetch
from django.db.models import OuterRef
from django.db.models import Subquery
from django.db.models import F, Case, When, Value, Sum, IntegerField

from studygroups.models import Application
from studygroups.models import StudyGroup
from studygroups.models import Course
from studygroups.models import TeamMembership
from ..decorators import user_is_staff
from learnwithpeople import __version__ as VERSION
from learnwithpeople import GIT_REVISION
from ..tasks import send_community_digest

from studygroups.forms import DigestGenerateForm

@method_decorator(user_is_staff, name='dispatch')
class StaffDashView(TemplateView):
    template_name = 'studygroups/staff_dash.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['version'] = VERSION
        context['git_revision'] = GIT_REVISION
        return context

@method_decorator(user_is_staff, name='dispatch')
class DigestGenerateView(FormView):
    form_class = DigestGenerateForm
    template_name = 'studygroups/digest_form.html'
    success_url = reverse_lazy('studygroups_staff_dash')

    def form_valid(self, form):
        # Find all signups with email and send opt out confirmation
        messages.info(self.request, _('You will shortly receive the community digest.'))
        start_date = datetime.combine(form.cleaned_data['start_date'], time(0,0,0))
        start_date = timezone.utc.localize(start_date)
        end_date = datetime.combine(form.cleaned_data['end_date'], time(0,0,0))
        end_date = timezone.utc.localize(end_date)
        send_community_digest.delay(start_date, end_date)
        return super().form_valid(form)

@method_decorator(user_is_staff, name='dispatch')
class ExportSignupsView(ListView):

    def get_queryset(self):
        return Application.objects.all().prefetch_related('study_group', 'study_group__course')

    def csv(self, **kwargs):
        response = http.HttpResponse(content_type="text/csv")
        ts = timezone.now().utcnow().isoformat()
        response['Content-Disposition'] = 'attachment; filename="signups-{}.csv"'.format(ts)
        signup_questions = ['support', 'goals', 'computer_access']
        field_names = [
            'id', 'uuid', 'study group id', 'study group uuid', 'study group name', 'course',
            'location', 'name', 'email', 'mobile', 'signed up at'
        ] + signup_questions + ['use_internet', 'survey completed']
        writer = csv.writer(response)
        writer.writerow(field_names)
        for signup in self.object_list:
            signup_data = json.loads(signup.signup_questions)
            digital_literacy = 'n/a'
            if signup_data.get('use_internet'):
                digital_literacy = dict(Application.DIGITAL_LITERACY_CHOICES)[signup_data.get('use_internet')]
            writer.writerow(
                [
                    signup.id,
                    signup.uuid,
                    signup.study_group_id,
                    signup.study_group.uuid,
                    signup.study_group.name,
                    signup.study_group.course.title,
                    signup.study_group.venue_name,
                    signup.name,
                    signup.email,
                    signup.mobile,
                    signup.created_at,
                ] +
                [ signup_data.get(key, 'n/a') for key in signup_questions ] +
                [
                    digital_literacy,
                    'yes' if signup.learnersurveyresponse_set.count() else 'no'
                ]
            )
        return response

    def get(self, request, *args, **kwargs):
        self.object_list = self.get_queryset()
        return self.csv(**kwargs)

@method_decorator(user_is_staff, name='dispatch')
class ExportFacilitatorsView(ListView):

    def get_queryset(self):
        learning_circles = StudyGroup.objects.select_related('course').published().filter(cofacilitators__user_id=OuterRef('pk')).order_by('-start_date')
        return User.objects.all().annotate(
            learning_circle_count=Sum(
                Case(
                    When(
                        facilitator__study_group__deleted_at__isnull=True,
                        facilitator__study_group__draft=False,
                        then=Value(1),
                        facilitator__user__id=F('id')
                    ),
                    default=Value(0), output_field=IntegerField()
                )
            )
        ).annotate(
            last_learning_circle_date=Subquery(learning_circles.values('start_date')[:1]),
            last_learning_circle_name=Subquery(learning_circles.values('name')[:1]),
            last_learning_circle_course=Subquery(learning_circles.values('course__title')[:1]),
            last_learning_circle_venue=Subquery(learning_circles.values('venue_name')[:1])
        )

    def csv(self, **kwargs):
        response = http.HttpResponse(content_type="text/csv")
        ts = timezone.now().utcnow().isoformat()
        response['Content-Disposition'] = 'attachment; filename="facilitators-{}.csv"'.format(ts)
        field_names = ['name',
            'email',
            'date joined',
            'last login',
            'communication opt-in',
            'learning circles run',
            'last learning circle date',
            'last learning cirlce name',
            'last learning circle course',
            'last learning circle venue',
        ]
        writer = csv.writer(response)
        writer.writerow(field_names)
        for user in self.object_list:
            data = [
                ' '.join([user.first_name, user.last_name]),
                user.email,
                user.date_joined,
                user.last_login,
                user.profile.communication_opt_in if user.profile else False,
                user.learning_circle_count,
                user.last_learning_circle_date,
                user.last_learning_circle_name,
                user.last_learning_circle_course,
                user.last_learning_circle_venue,
            ]
            writer.writerow(data)
        return response

    def get(self, request, *args, **kwargs):
        self.object_list = self.get_queryset()
        return self.csv(**kwargs)

@method_decorator(user_is_staff, name='dispatch')
class ExportStudyGroupsView(ListView):

    def get_queryset(self):
        return StudyGroup.objects.all().prefetch_related('course', 'facilitator', 'meeting_set').annotate(
            learning_circle_number=RawSQL("RANK() OVER(PARTITION BY facilitator_id ORDER BY created_at ASC)", [])
        )

    def csv(self, **kwargs):
        response = http.HttpResponse(content_type="text/csv")
        ts = timezone.now().utcnow().isoformat()
        response['Content-Disposition'] = 'attachment; filename="learning-circles-{}.csv"'.format(ts)
        field_names = [
            'id',
            'uuid',
            'name',
            'date created',
            'date deleted',
            'draft',
            'course id',
            'course title',
            'facilitator',
            'faciltator email',
            'learning_circle_number',
            'location',
            'city',
            'time',
            'day',
            'last meeting',
            'first meeting',
            'signups',
            'team',
            'facilitator survey',
            'facilitator survey completed',
            'learner survey',
            'learner survey responses',
            'did not happen',
        ]
        writer = csv.writer(response)
        writer.writerow(field_names)
        for sg in self.object_list:
            data = [
                sg.pk,
                sg.uuid,
                sg.name,
                sg.created_at,
                sg.deleted_at,
                'yes' if sg.draft else 'no',
                sg.course.id,
                sg.course.title,
                ' '.join([sg.facilitator.first_name, sg.facilitator.last_name]),
                sg.facilitator.email,
                sg.learning_circle_number,
                ' ' .join([sg.venue_name, sg.venue_address]),
                sg.city,
                sg.meeting_time,
                sg.day(),
            ]
            if sg.meeting_set.active().last():
                data += [sg.meeting_set.active().order_by('meeting_date', 'meeting_time').last().meeting_date]
            elif sg.deleted_at:
                data += [sg.start_date]
            else:
                data += ['']

            if sg.meeting_set.active().first():
                data += [sg.meeting_set.active().order_by('meeting_date', 'meeting_time').first().meeting_date]
            elif sg.deleted_at:
                data += [sg.end_date]
            else:
                data += ['']

            data += [sg.application_set.active().count()]
            # team
            if sg.facilitator.teammembership_set.active().count():
                data += [sg.facilitator.teammembership_set.active().first().team.name]
            else:
                data += ['']

            base_url = f'{settings.PROTOCOL}://{settings.DOMAIN}'
            facilitator_survey =  '{}{}'.format(
                base_url,
                reverse('studygroups_facilitator_survey', args=(sg.uuid,))
            )
            data += [facilitator_survey]
            data += ['yes' if sg.facilitatorsurveyresponse_set.count() else 'no']
            learner_survey = '{}{}'.format(
                base_url,
                reverse('studygroups_learner_survey', args=(sg.uuid,))
            )
            data += [learner_survey]
            data += [sg.learnersurveyresponse_set.count()]
            data += [sg.did_not_happen]

            writer.writerow(data)
        return response

    def get(self, request, *args, **kwargs):
        self.object_list = self.get_queryset()
        return self.csv(**kwargs)

@method_decorator(user_is_staff, name='dispatch')
class ExportCoursesView(ListView):

    def get_queryset(self):
        team_membership = TeamMembership.objects.active().filter(user=OuterRef('created_by'))
        return Course.objects.active()\
            .filter(studygroup__deleted_at__isnull=True, studygroup__draft=False)\
            .filter(facilitatorguide__deleted_at__isnull=True)\
            .annotate(lc_count=Count('studygroup', distinct=True))\
            .annotate(active_lc_count=Count('studygroup', distinct=True, filter=Q(studygroup__end_date__gte=timezone.now())))\
            .annotate(facilitator_guide_count=Count('facilitatorguide', distinct=True))\
            .annotate(team_name=Subquery(team_membership.values('team__name')[:1]))\
            .select_related('created_by')

    def csv(self, **kwargs):
        response = http.HttpResponse(content_type="text/csv")
        ts = timezone.now().utcnow().isoformat()
        response['Content-Disposition'] = 'attachment; filename="courses-{}.csv"'.format(ts)
        db_fields = [
            'id',
            'title',
            'provider',
            'link',
            'caption',
            'on_demand',
            'topics',
            'language',
            'created_by',
            'unlisted',
            'license',
            'created_at',
            'lc_count',
            'active_lc_count',
            'facilitator_guide_count',
            'team_name',
        ]
        writer = csv.writer(response)
        writer.writerow(db_fields)
        for obj in self.object_list:
            data = [
                getattr(obj, field) for field in db_fields
            ]
            writer.writerow(data)
        return response

    def get(self, request, *args, **kwargs):
        self.object_list = self.get_queryset()
        return self.csv(**kwargs)
<commit_msg>Update learning circle export to correctly use team and add facilitator count<commit_after>from datetime import datetime, time
import json
import unicodecsv as csv

from django import http
from django.urls import reverse_lazy
from django.shortcuts import render, get_object_or_404
from django.urls import reverse
from django.contrib.auth.models import User
from django.contrib import messages
from django.conf import settings
from django.utils import timezone
from django.utils.decorators import method_decorator
from django.utils.translation import ugettext as _
from django.views.generic.base import View
from django.views.generic import ListView
from django.views.generic import TemplateView
from django.views.generic.edit import FormView
from django.db.models.expressions import RawSQL
from django.db.models import Count
from django.db.models import Q
from django.db.models import Prefetch
from django.db.models import OuterRef
from django.db.models import Subquery
from django.db.models import F, Case, When, Value, Sum, IntegerField

from studygroups.models import Application
from studygroups.models import StudyGroup
from studygroups.models import Course
from studygroups.models import TeamMembership
from ..decorators import user_is_staff
from learnwithpeople import __version__ as VERSION
from learnwithpeople import GIT_REVISION
from ..tasks import send_community_digest

from studygroups.forms import DigestGenerateForm

@method_decorator(user_is_staff, name='dispatch')
class StaffDashView(TemplateView):
    template_name = 'studygroups/staff_dash.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['version'] = VERSION
        context['git_revision'] = GIT_REVISION
        return context

@method_decorator(user_is_staff, name='dispatch')
class DigestGenerateView(FormView):
    form_class = DigestGenerateForm
    template_name = 'studygroups/digest_form.html'
    success_url = reverse_lazy('studygroups_staff_dash')

    def form_valid(self, form):
        # Find all signups with email and send opt out confirmation
        messages.info(self.request, _('You will shortly receive the community digest.'))
        start_date = datetime.combine(form.cleaned_data['start_date'], time(0,0,0))
        start_date = timezone.utc.localize(start_date)
        end_date = datetime.combine(form.cleaned_data['end_date'], time(0,0,0))
        end_date = timezone.utc.localize(end_date)
        send_community_digest.delay(start_date, end_date)
        return super().form_valid(form)

@method_decorator(user_is_staff, name='dispatch')
class ExportSignupsView(ListView):

    def get_queryset(self):
        return Application.objects.all().prefetch_related('study_group', 'study_group__course')

    def csv(self, **kwargs):
        response = http.HttpResponse(content_type="text/csv")
        ts = timezone.now().utcnow().isoformat()
        response['Content-Disposition'] = 'attachment; filename="signups-{}.csv"'.format(ts)
        signup_questions = ['support', 'goals', 'computer_access']
        field_names = [
            'id', 'uuid', 'study group id', 'study group uuid', 'study group name', 'course',
            'location', 'name', 'email', 'mobile', 'signed up at'
        ] + signup_questions + ['use_internet', 'survey completed']
        writer = csv.writer(response)
        writer.writerow(field_names)
        for signup in self.object_list:
            signup_data = json.loads(signup.signup_questions)
            digital_literacy = 'n/a'
            if signup_data.get('use_internet'):
                digital_literacy = dict(Application.DIGITAL_LITERACY_CHOICES)[signup_data.get('use_internet')]
            writer.writerow(
                [
                    signup.id,
                    signup.uuid,
                    signup.study_group_id,
                    signup.study_group.uuid,
                    signup.study_group.name,
                    signup.study_group.course.title,
                    signup.study_group.venue_name,
                    signup.name,
                    signup.email,
                    signup.mobile,
                    signup.created_at,
                ] +
                [ signup_data.get(key, 'n/a') for key in signup_questions ] +
                [
                    digital_literacy,
                    'yes' if signup.learnersurveyresponse_set.count() else 'no'
                ]
            )
        return response

    def get(self, request, *args, **kwargs):
        self.object_list = self.get_queryset()
        return self.csv(**kwargs)

@method_decorator(user_is_staff, name='dispatch')
class ExportFacilitatorsView(ListView):

    def get_queryset(self):
        learning_circles = StudyGroup.objects.select_related('course').published().filter(cofacilitators__user_id=OuterRef('pk')).order_by('-start_date')
        return User.objects.all().annotate(
            learning_circle_count=Sum(
                Case(
                    When(
                        facilitator__study_group__deleted_at__isnull=True,
                        facilitator__study_group__draft=False,
                        then=Value(1),
                        facilitator__user__id=F('id')
                    ),
                    default=Value(0), output_field=IntegerField()
                )
            )
        ).annotate(
            last_learning_circle_date=Subquery(learning_circles.values('start_date')[:1]),
            last_learning_circle_name=Subquery(learning_circles.values('name')[:1]),
            last_learning_circle_course=Subquery(learning_circles.values('course__title')[:1]),
            last_learning_circle_venue=Subquery(learning_circles.values('venue_name')[:1])
        )

    def csv(self, **kwargs):
        response = http.HttpResponse(content_type="text/csv")
        ts = timezone.now().utcnow().isoformat()
        response['Content-Disposition'] = 'attachment; filename="facilitators-{}.csv"'.format(ts)
        field_names = ['name',
            'email',
            'date joined',
            'last login',
            'communication opt-in',
            'learning circles run',
            'last learning circle date',
            'last learning cirlce name',
            'last learning circle course',
            'last learning circle venue',
        ]
        writer = csv.writer(response)
        writer.writerow(field_names)
        for user in self.object_list:
            data = [
                ' '.join([user.first_name, user.last_name]),
                user.email,
                user.date_joined,
                user.last_login,
                user.profile.communication_opt_in if user.profile else False,
                user.learning_circle_count,
                user.last_learning_circle_date,
                user.last_learning_circle_name,
                user.last_learning_circle_course,
                user.last_learning_circle_venue,
            ]
            writer.writerow(data)
        return response

    def get(self, request, *args, **kwargs):
        self.object_list = self.get_queryset()
        return self.csv(**kwargs)

@method_decorator(user_is_staff, name='dispatch')
class ExportStudyGroupsView(ListView):

    def get_queryset(self):
        return StudyGroup.objects.all().prefetch_related('course', 'facilitator', 'meeting_set').annotate(
            learning_circle_number=RawSQL("RANK() OVER(PARTITION BY facilitator_id ORDER BY created_at ASC)", [])
        )

    def csv(self, **kwargs):
        response = http.HttpResponse(content_type="text/csv")
        ts = timezone.now().utcnow().isoformat()
        response['Content-Disposition'] = 'attachment; filename="learning-circles-{}.csv"'.format(ts)
        field_names = [
            'id',
            'uuid',
            'name',
            'date created',
            'date deleted',
            'draft',
            'course id',
            'course title',
            'facilitator',
            'facilitator email',
            'learning_circle_number',
            'location',
            'city',
            'time',
            'day',
            'last meeting',
            'first meeting',
            'signups',
            'team',
            'facilitator survey',
            'facilitator survey completed',
            'learner survey',
            'learner survey responses',
            'did not happen',
            'facilitator count',
        ]
        writer = csv.writer(response)
        writer.writerow(field_names)
        for sg in self.object_list:
            data = [
                sg.pk,
                sg.uuid,
                sg.name,
                sg.created_at,
                sg.deleted_at,
                'yes' if sg.draft else 'no',
                sg.course.id,
                sg.course.title,
                ' '.join([sg.facilitator.first_name, sg.facilitator.last_name]),
                sg.facilitator.email,
                sg.learning_circle_number,
                ' ' .join([sg.venue_name, sg.venue_address]),
                sg.city,
                sg.meeting_time,
                sg.day(),
            ]
            if sg.meeting_set.active().last():
                data += [sg.meeting_set.active().order_by('meeting_date', 'meeting_time').last().meeting_date]
            elif sg.deleted_at:
                data += [sg.start_date]
            else:
                data += ['']

            if sg.meeting_set.active().first():
                data += [sg.meeting_set.active().order_by('meeting_date', 'meeting_time').first().meeting_date]
            elif sg.deleted_at:
                data += [sg.end_date]
            else:
                data += ['']

            data += [sg.application_set.active().count()]

            if sg.team:
                data += [sg.team.name]
            else:
                data += ['']

            base_url = f'{settings.PROTOCOL}://{settings.DOMAIN}'
            facilitator_survey =  '{}{}'.format(
                base_url,
                reverse('studygroups_facilitator_survey', args=(sg.uuid,))
            )
            data += [facilitator_survey]
            data += ['yes' if sg.facilitatorsurveyresponse_set.count() else 'no']
            learner_survey = '{}{}'.format(
                base_url,
                reverse('studygroups_learner_survey', args=(sg.uuid,))
            )
            data += [learner_survey]
            data += [sg.learnersurveyresponse_set.count()]
            data += [sg.did_not_happen]
            data += [sg.cofacilitators.count()]

            writer.writerow(data)
        return response

    def get(self, request, *args, **kwargs):
        self.object_list = self.get_queryset()
        return self.csv(**kwargs)

@method_decorator(user_is_staff, name='dispatch')
class ExportCoursesView(ListView):

    def get_queryset(self):
        team_membership = TeamMembership.objects.active().filter(user=OuterRef('created_by'))
        return Course.objects.active()\
            .filter(studygroup__deleted_at__isnull=True, studygroup__draft=False)\
            .filter(facilitatorguide__deleted_at__isnull=True)\
            .annotate(lc_count=Count('studygroup', distinct=True))\
            .annotate(active_lc_count=Count('studygroup', distinct=True, filter=Q(studygroup__end_date__gte=timezone.now())))\
            .annotate(facilitator_guide_count=Count('facilitatorguide', distinct=True))\
            .annotate(team_name=Subquery(team_membership.values('team__name')[:1]))\
            .select_related('created_by')

    def csv(self, **kwargs):
        response = http.HttpResponse(content_type="text/csv")
        ts = timezone.now().utcnow().isoformat()
        response['Content-Disposition'] = 'attachment; filename="courses-{}.csv"'.format(ts)
        db_fields = [
            'id',
            'title',
            'provider',
            'link',
            'caption',
            'on_demand',
            'topics',
            'language',
            'created_by',
            'unlisted',
            'license',
            'created_at',
            'lc_count',
            'active_lc_count',
            'facilitator_guide_count',
            'team_name',
        ]
        writer = csv.writer(response)
        writer.writerow(db_fields)
        for obj in self.object_list:
            data = [
                getattr(obj, field) for field in db_fields
            ]
            writer.writerow(data)
        return response

    def get(self, request, *args, **kwargs):
        self.object_list = self.get_queryset()
        return self.csv(**kwargs)
<|endoftext|>


<commit_before>#! /usr/bin/python

import sys
import functools
import time

from functools import partial
from benchmark import Benchmark
from bulk_insert import BulkInserter
from hyperdex.mongo import HyperDatabase, HyperSpace

### CONFIG ###
# Modify this to change the behaviour of the benchmark

# How many atomic add calls?
numCalls = 1000
# How many documents in the space ?
numDocs = 100000
# Gaps between calling
addGap = 1
##############

# Verify Config
assert numCalls > 0
assert numDocs > 0
assert addGap > 0
assert (numCalls * addGap) < numDocs

# Setup
db = HyperDatabase(sys.argv[1], int(sys.argv[2]))
assert addGap > 0
assert (numCalls * addGap) < numDocs

# Setup
db = HyperDatabase(sys.argv[1], int(sys.argv[2]))

def create_doc(num_elems, prefix, pos):
        id = prefix + str(pos)
        doc = {'_id' : id}

        for i in range(num_elems):
                doc['elem' + str(i)] = i

        return doc

# Run
for n in range(1000, 10100, 1000):
        space = db.bench

        generator = partial(create_doc, n, 'myval')
        bulk_inserter = BulkInserter(numDocs, space, generator)
        bulk_inserter.run()

        pendingOps = []

        bench = Benchmark()
        bench.start()

        for i in range(numCalls):
                p = space.async_atomic_add('myval' + str(i*addGap), {'elem50' : 10})
                pendingOps.append(p)

        while len(pendingOps) > 0:
                pendingOps[0].wait()
                pendingOps.pop(0)

        elapsed = bench.stop()

        # Print simple CSV
        print str(n) + ',' + str(elapsed)

        space.clear()
        time.sleep(5)

<commit_msg>Don't vary document size in benchmark<commit_after>#! /usr/bin/python

import sys
import functools
import time

from functools import partial
from benchmark import Benchmark
from bulk_insert import BulkInserter
from hyperdex.mongo import HyperDatabase, HyperSpace

### CONFIG ###
# Modify this to change the behaviour of the benchmark

# How big are the documents (in elements?
documentSize = 1000
# How many atomic add calls?
numCalls = 1000
# How many documents in the space ?
numDocs = 100000
# Gaps between calling
addGap = 1
##############

# Verify Config
assert numCalls > 0
assert numDocs > 0
assert addGap > 0
assert (numCalls * addGap) < numDocs

# Setup
db = HyperDatabase(sys.argv[1], int(sys.argv[2]))
assert addGap > 0
assert (numCalls * addGap) < numDocs

# Setup
db = HyperDatabase(sys.argv[1], int(sys.argv[2]))

def create_doc(num_elems, prefix, pos):
        id = prefix + str(pos)
        doc = {'_id' : id}

        for i in range(num_elems):
                doc['elem' + str(i)] = i

        return doc

# Run
for n in range(0, 10, 1):
        space = db.bench

        generator = partial(create_doc, documentSize, 'myval')
        bulk_inserter = BulkInserter(numDocs, space, generator)
        bulk_inserter.run()

        pendingOps = []

        bench = Benchmark()
        bench.start()

        for i in range(numCalls):
                p = space.async_atomic_add('myval' + str(i*addGap), {'elem50' : 10})
                pendingOps.append(p)

        while len(pendingOps) > 0:
                pendingOps[0].wait()
                pendingOps.pop(0)

        elapsed = bench.stop()

        # Print simple CSV
        print str(elapsed)

        space.clear()
        time.sleep(5)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Nov 26 10:57:30 2013

@author: Christian Muenker

Tab-Widget for exporting / importing and saving / loading data
"""
from __future__ import print_function, division, unicode_literals
import sys, os
from PyQt4 import QtGui
import scipy.io
import numpy as np

# import filterbroker from one level above if this file is run as __main__
# for test purposes
if __name__ == "__main__":
    __cwd__ = os.path.dirname(os.path.abspath(__file__))
    sys.path.append(__cwd__ + '/..')

import filterbroker as fb # importing filterbroker initializes all its globals

class InputInfo(QtGui.QWidget):
    """
    Create the window for entering exporting / importing and saving / loading data
    """
    def __init__(self, DEBUG = True):
        self.DEBUG = DEBUG
        super(InputInfo, self).__init__()

        self.initUI()
        self.showCoeffs()
        self.showInfo()

    def initUI(self):
        """
        Intitialize the widget, consisting of:
        -
        -
        """
        # widget / subwindow for parameter selection
        self.chkFilterInfo = QtGui.QCheckBox()
        self.chkFilterInfo.setToolTip("Display filter info from filter design file.")
        self.labFilterInfo = QtGui.QLabel()
        self.labFilterInfo.setText("Filter Info")

        self.chkCoeffList =  QtGui.QCheckBox()
        self.chkCoeffList.setToolTip("Show filter coefficients as an editable list.")
        self.labCoeffList = QtGui.QLabel()
        self.labCoeffList.setText("Coefficients")

        self.chkPoleZeroList =  QtGui.QCheckBox()
        self.chkPoleZeroList.setToolTip("Show poles and zeros as an editable list.")
        self.labPoleZeroList = QtGui.QLabel()
        self.labPoleZeroList.setText("Poles / Zeros")

        self.listCoeff = QtGui.QListWidget()

        self.labFiltInfo = QtGui.QLabel()
        self.labFiltInfo.setWordWrap(True)

        filtInfoLayout = QtGui.QVBoxLayout()
        filtInfoLayout.addWidget(self.labFiltInfo)

        self.filtInfoFrame = QtGui.QFrame()
        self.filtInfoFrame.setFrameStyle(QtGui.QFrame.StyledPanel|QtGui.QFrame.Sunken)
        self.filtInfoFrame.setLayout(filtInfoLayout)
        self.filtInfoFrame.setSizePolicy(QtGui.QSizePolicy.Minimum,
                                 QtGui.QSizePolicy.Minimum)

        # ============== UI Layout =====================================
        self.chkLayout = QtGui.QHBoxLayout()
        self.chkLayout.addWidget(self.chkFilterInfo) # filter export button
        self.chkLayout.addWidget(self.labFilterInfo)
        self.chkLayout.addStretch(1)
        self.chkLayout.addWidget(self.chkCoeffList)
        self.chkLayout.addWidget(self.labCoeffList)
        self.chkLayout.addStretch(1)
        self.chkLayout.addWidget(self.chkPoleZeroList)
        self.chkLayout.addWidget(self.labPoleZeroList)
        self.chkLayout.addStretch(10)

        vbox = QtGui.QVBoxLayout()
        vbox.addLayout(self.chkLayout)
        vbox.addWidget(self.listCoeff)
        vbox.addWidget(self.filtInfoFrame)
        vbox.addStretch(10)
        self.setLayout(vbox)

        # ============== Signals & Slots ================================
        self.chkFilterInfo.clicked.connect(self.showInfo)
        self.chkCoeffList.clicked.connect(self.showCoeffs)

    def showInfo(self):
        """
        Display info from filter design file
        """
        self.filtInfoFrame.setVisible(self.chkFilterInfo.isChecked())
        pass

    def showCoeffs(self):
        self.listCoeff.setVisible(self.chkCoeffList.isChecked())
        self.listCoeff.clear()
        coeffs = fb.gD["selFilter"]["coeffs"]
        print(coeffs)
        print ("shape", np.shape(coeffs))
        print ("len", len(coeffs))
        print("ndim", np.ndim(coeffs))
        aCoeffs = []
        bCoeffs = []
        if np.ndim(coeffs) == 1:
            print("FIR!")
            for i in range(len(coeffs)):
                print(i, fb.gD["selFilter"]["coeffs"][i])
                bCoeffs.append(str(coeffs[i]))
            self.listCoeff.addItems(bCoeffs)
            print(bCoeffs)
        else:
            print("IIR!")
            for i in range(np.shape(coeffs)[1]):
                print(i, fb.gD["selFilter"]["coeffs"][0][i])
                bCoeffs.append(str(coeffs[0][i]))
                aCoeffs.append(str(coeffs[1][i]))
            self.listCoeff.addItems(bCoeffs)
#            self.listCoeff.addItems(bCoeffs)
            print(bCoeffs)

#------------------------------------------------------------------------------

if __name__ == '__main__':

    app = QtGui.QApplication(sys.argv)
    form = InputInfo()
    form.show()

    app.exec_()<commit_msg>Added printing of filter info text Signed-off-by: chipmuenk <mail@chipmuenk.de><commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Nov 26 10:57:30 2013

@author: Christian Muenker

Tab-Widget for exporting / importing and saving / loading data
"""
from __future__ import print_function, division, unicode_literals
import sys, os
from PyQt4 import QtGui
import scipy.io
import numpy as np

# import filterbroker from one level above if this file is run as __main__
# for test purposes
if __name__ == "__main__":
    __cwd__ = os.path.dirname(os.path.abspath(__file__))
    sys.path.append(__cwd__ + '/..')

import filterbroker as fb # importing filterbroker initializes all its globals

class InputInfo(QtGui.QWidget):
    """
    Create the window for entering exporting / importing and saving / loading data
    """
    def __init__(self, DEBUG = True):
        self.DEBUG = DEBUG
        super(InputInfo, self).__init__()

        self.initUI()
        self.showCoeffs()
        self.showInfo()

    def initUI(self):
        """
        Intitialize the widget, consisting of:
        -
        -
        """
        # widget / subwindow for parameter selection
        self.chkFilterInfo = QtGui.QCheckBox()
        self.chkFilterInfo.setToolTip("Display filter info from filter design file.")
        self.labFilterInfo = QtGui.QLabel()
        self.labFilterInfo.setText("Filter Info")

        self.chkCoeffList =  QtGui.QCheckBox()
        self.chkCoeffList.setToolTip("Show filter coefficients as an editable list.")
        self.labCoeffList = QtGui.QLabel()
        self.labCoeffList.setText("Coefficients")

        self.chkPoleZeroList =  QtGui.QCheckBox()
        self.chkPoleZeroList.setToolTip("Show poles and zeros as an editable list.")
        self.labPoleZeroList = QtGui.QLabel()
        self.labPoleZeroList.setText("Poles / Zeros")

        self.listCoeff = QtGui.QListWidget()

        self.labFiltInfo = QtGui.QLabel()
        self.labFiltInfo.setWordWrap(True)

        filtInfoLayout = QtGui.QVBoxLayout()
        filtInfoLayout.addWidget(self.labFiltInfo)

        self.filtInfoFrame = QtGui.QFrame()
        self.filtInfoFrame.setFrameStyle(QtGui.QFrame.StyledPanel|QtGui.QFrame.Sunken)
        self.filtInfoFrame.setLayout(filtInfoLayout)
        self.filtInfoFrame.setSizePolicy(QtGui.QSizePolicy.Minimum,
                                 QtGui.QSizePolicy.Minimum)

        # ============== UI Layout =====================================
        self.chkLayout = QtGui.QHBoxLayout()
        self.chkLayout.addWidget(self.chkFilterInfo) # filter export button
        self.chkLayout.addWidget(self.labFilterInfo)
        self.chkLayout.addStretch(1)
        self.chkLayout.addWidget(self.chkCoeffList)
        self.chkLayout.addWidget(self.labCoeffList)
        self.chkLayout.addStretch(1)
        self.chkLayout.addWidget(self.chkPoleZeroList)
        self.chkLayout.addWidget(self.labPoleZeroList)
        self.chkLayout.addStretch(10)

        vbox = QtGui.QVBoxLayout()
        vbox.addLayout(self.chkLayout)
        vbox.addWidget(self.listCoeff)
        vbox.addWidget(self.filtInfoFrame)
        vbox.addStretch(10)
        self.setLayout(vbox)

        # ============== Signals & Slots ================================
        self.chkFilterInfo.clicked.connect(self.showInfo)
        self.chkCoeffList.clicked.connect(self.showCoeffs)

    def showInfo(self):
        """
        Display info from filter design file
        """
        self.filtInfoFrame.setVisible(self.chkFilterInfo.isChecked())
        self.labFiltInfo.setText(fb.gD["selFilter"]["inst"].info)

    def showCoeffs(self):
        self.listCoeff.setVisible(self.chkCoeffList.isChecked())
        self.listCoeff.clear()
        coeffs = fb.gD["selFilter"]["coeffs"]
        print(coeffs)
        print ("shape", np.shape(coeffs))
        print ("len", len(coeffs))
        print("ndim", np.ndim(coeffs))
        aCoeffs = []
        bCoeffs = []
        if np.ndim(coeffs) == 1:
            print("FIR!")
            for i in range(len(coeffs)):
                print(i, fb.gD["selFilter"]["coeffs"][i])
                bCoeffs.append(str(coeffs[i]))
            self.listCoeff.addItems(bCoeffs)
            print(bCoeffs)
        else:
            print("IIR!")
            for i in range(np.shape(coeffs)[1]):
                print(i, fb.gD["selFilter"]["coeffs"][0][i])
                bCoeffs.append(str(coeffs[0][i]))
                aCoeffs.append(str(coeffs[1][i]))
            self.listCoeff.addItems(bCoeffs)
#            self.listCoeff.addItems(bCoeffs)
            print(bCoeffs)

#------------------------------------------------------------------------------

if __name__ == '__main__':

    app = QtGui.QApplication(sys.argv)
    form = InputInfo()
    form.show()

    app.exec_()<|endoftext|>


<commit_before>__author__ = "vishvendra singh"
__version__ = 0.1

import sys
import os
import urllib
import basic_functions as bf
import download_subtitle as ds

def print_msg(msg):
    os.system("clear")
    print "msg: "+msg
    cli_menu()

def cli_menu():
    print "1. add folder to watch"
    print "2. remove folder"
    print "3. downloading subtitles"
    print "4. exit"
    choice = input("your choice: ")
    if user_choice_validate(choice):
        interpret_usr_choice(choice)
    else:
        print_msg("msg: choice not valid, possible values are 1, 2, 3, 4")

def user_choice_validate(value):
    """
    >>> user_choice_validate(3)
    True
    >>> user_choice_validate(5)
    False

    :param value:
    :return:
    """
    if isinstance(value, int) and value in [1, 2, 3, 4]:
        return True
    else:
        return False

def interpret_usr_choice(choice):
    if choice == 1:
        cli_add_dir()
    elif choice == 2:
        print_msg("this option is not available till now")
    elif choice == 3:
        cli_dl_subs()
    elif choice == 4:
        sys.exit(0)

def cli_add_dir():
    dir_path = raw_input("input folder path:>>> ")
    if bf.add_folder(dir_path):
        print_msg("folder added successfully")
    else:
        print_msg("given folder not exist, check your input")

def cli_dl_subs():
    if ds.download_subtitles_main():
        print_msg("subs downloaded successfully")
    else:
        print_msg("add folders to list first")

def subliminal_exist_check():
    """
    check for installation of subliminal on user's computer
    >>> subliminal_exist_check()
    True

    :return:
    """
    try:
        import subliminal
    except ImportError:
        return False
    else:
        return True

def internet_connectivity_check():
    """
    checks for internet connectivity on user's computer
    >>> internet_connectivity_check()
    True

    :return:
    """
    try:
        fh = urllib.urlopen("http://www.google.com")
    except IOError:
        return False
    fh.close()
    return True

def start_check():
    """
    when program starts it checks for internet connectivity and installation of subliminal
    >>> start_check()  #when connected to internet and subliminal installation
    True

    :return: status whether start program or not
    """
    if subliminal_exist_check():
        if internet_connectivity_check():
            return True
        else:
            print "internet connectivity problem"
            return False
    else:
        print "subliminal not installed"
        return False

if __name__ == "__main__":
    if start_check():
        cli_menu()
    else:
        print "restart again after resolving given problems"
        sys.exit(0)<commit_msg>start implementing dlsub.py rather than download_subtitle.py in main.py<commit_after>__author__ = "vishvendra singh"
__version__ = 0.1

import sys
import os
import urllib

import basic_functions as bf
import dlsub

def print_msg(msg):
    os.system("clear")
    print "msg: "+msg
    cli_menu()

def cli_menu():
    print "1. add folder to watch"
    print "2. remove folder"
    print "3. downloading subtitles"
    print "4. exit"
    choice = input("your choice: ")
    if user_choice_validate(choice):
        interpret_usr_choice(choice)
    else:
        print_msg("msg: choice not valid, possible values are 1, 2, 3, 4")

def user_choice_validate(value):
    """
    >>> user_choice_validate(3)
    True
    >>> user_choice_validate(5)
    False

    :param value:
    :return:
    """
    if isinstance(value, int) and value in [1, 2, 3, 4]:
        return True
    else:
        return False

def interpret_usr_choice(choice):
    if choice == 1:
        cli_add_dir()
    elif choice == 2:
        print_msg("this option is not available till now")
    elif choice == 3:
        cli_dl_subs()
    elif choice == 4:
        sys.exit(0)

def cli_add_dir():
    dir_path = raw_input("input folder path:>>> ")
    if bf.add_folder(dir_path):
        print_msg("folder added successfully")
    else:
        print_msg("given folder not exist, check your input")

def cli_dl_subs():
    if dlsub.download_subtitles_main():
        print_msg("subs downloaded successfully")
    else:
        print_msg("add folders to list first")

def subliminal_exist_check():
    """
    check for installation of subliminal on user's computer
    >>> subliminal_exist_check()
    True

    :return:
    """
    try:
        import subliminal
    except ImportError:
        return False
    else:
        return True

def internet_connectivity_check():
    """
    checks for internet connectivity on user's computer
    >>> internet_connectivity_check()
    True

    :return:
    """
    try:
        fh = urllib.urlopen("http://www.google.com")
    except IOError:
        return False
    fh.close()
    return True

def start_check():
    """
    when program starts it checks for internet connectivity and installation of subliminal
    >>> start_check()  #when connected to internet and subliminal installation
    True

    :return: status whether start program or not
    """
    if subliminal_exist_check():
        if internet_connectivity_check():
            return True
        else:
            print "internet connectivity problem"
            return False
    else:
        print "subliminal not installed"
        return False

if __name__ == "__main__":
    if start_check():
        cli_menu()
    else:
        print "restart again after resolving given problems"
        sys.exit(0)<|endoftext|>


<commit_before>from pyamg.krylov import bicgstab, cg, cgne, cgnr, cr, fgmres, gmres
from pyamg.krylov._gmres_householder import gmres_householder
from pyamg.krylov._gmres_mgs import gmres_mgs
from numpy import array, zeros, ones
from scipy import mat, random
from scipy.linalg import solve
from pyamg.util.linalg import norm
import pyamg

from numpy.testing import TestCase, assert_array_almost_equal, assert_equal

class TestKrylov(TestCase):
    def setUp(self):
        self.cases = []
        self.spd_cases = []
        self.symm_cases = []

        # self.oblique = [gmres, fgmres, cgnr,
        #                 krylov._gmres_householder.gmres_householder,
        #                 krylov._gmres_mgs.gmres_mgs]
        self.oblique = [gmres_householder, gmres_mgs, gmres, fgmres, cgnr]
        self.symm_oblique = [cr]
        self.orth = [cgne]
        self.inexact = [bicgstab]
        self.spd_orth = [cg]

        # 1x1
        A = mat([[1.2]])
        b = array([3.9]).reshape(-1, 1)
        x0 = zeros((1, 1))
        self.cases.append({'A': A, 'b': b, 'x0': x0, 'tol': 1e-16,
                           'maxiter': 1, 'reduction_factor': 1e-10})
        self.spd_cases.append({'A': A, 'b': b, 'x0': x0, 'tol': 1e-16,
                               'maxiter': 1, 'reduction_factor': 1e-10})
        self.symm_cases.append({'A': A, 'b': b, 'x0': x0, 'tol': 1e-16,
                                'maxiter': 1, 'reduction_factor': 1e-10})

        # 4x4
        A = mat([[1.2,    0.,   0.,     0.],
                 [0.,     4.,   2.,     6.],
                 [0.,     0.,   9.3,  -2.31],
                 [-4.,     0.,   0.,    -11.]])
        b = array([1., 3.9, 0., -1.23]).reshape(-1, 1)
        x0 = zeros((4, 1))
        self.cases.append({'A': A, 'b': b, 'x0': x0, 'tol': 1e-16,
                           'maxiter': 4, 'reduction_factor': 1e-10})
        self.spd_cases.append({'A': A.T*A, 'b': b, 'x0': x0, 'tol': 1e-16,
                               'maxiter': 4, 'reduction_factor': 1e-10})
        self.symm_cases.append({'A': A.T + A, 'b': b, 'x0': x0, 'tol': 1e-16,
                                'maxiter': 4, 'reduction_factor': 1e-10})

        # 4x4 Imaginary
        A = mat(A, dtype=complex)
        A[0, 0] += 3.1j
        A[3, 3] -= 1.34j
        A[1, 3] *= 1.0j
        A[1, 2] += 1.0j
        b = array([1. - 1.0j, 2.0 - 3.9j, 0., -1.23]).reshape(-1, 1)
        x0 = ones((4, 1))
        self.cases.append({'A': A, 'b': b, 'x0': x0, 'tol': 1e-16,
                           'maxiter': 4, 'reduction_factor': 1e-10})
        self.spd_cases.append({'A': A.H*A, 'b': b, 'x0': x0, 'tol': 1e-16,
                               'maxiter': 4, 'reduction_factor': 1e-10})
        self.symm_cases.append({'A': A.H + A, 'b': b, 'x0': x0, 'tol': 1e-16,
                                'maxiter': 4, 'reduction_factor': 1e-10})

        # 10x10
        A = mat([[-1.1,    0.,   0.,   0.,  3.9,  0.,   0.,  11.,  -1.,  0.],
                 [0.,    4.,   2.9,  0.,   0.,  6.8,  0.,  0.,    0.,  0.],
                 [0.,    0.,   9.0,  0.,   0.,  0.8,  1., -2.2,   0.,  9.],
                 [-4.,    0.,   0.0,  0.,   0.,  0.0,  2.,  2.2,   0.,  0.],
                 [0.,    0.,   0.0, 21.,   0.,  0.1,  0.,   0.,   0.,  0.],
                 [0.,    0.,   0.0,  0.,  -4.7, 0.0,  0.,   0.,   0.,  0.],
                 [2.1,   7.,  22.0,  0.,   0.,  0.0,  0.,   0.,   0.,  0.],
                 [0.,    0.,   0.0, 34.,   0.,  0.0,  0.,   0.,  -12.3, 0.],
                 [0.,   3.4,   0.0,  0.,   0., -0.3,  0.,   0.,   0.,  0.],
                 [9.,    0.,   0.0,  0.,  87.,  0.0,  0.,   0.,   0., -11.2]])
        b = array([1., 0., 0.2, 8., 0., -1.9,
                   11.3, 0.0, 0.1, 0.0]).reshape(-1, 1)
        x0 = zeros((10, 1))
        x0[4] = 11.1
        x0[7] = -2.
        self.cases.append({'A': A, 'b': b, 'x0': x0, 'tol': 1e-16,
                           'maxiter': 2, 'reduction_factor': 0.98})
        self.symm_cases.append({'A': A + A.T, 'b': b, 'x0': x0, 'tol': 1e-16,
                                'maxiter': 2, 'reduction_factor': 0.98})
        self.spd_cases.append({'A':
                               mat(pyamg.gallery.poisson((10,)).todense()),
                               'b': b, 'x0': x0, 'tol': 1e-16, 'maxiter': 2,
                               'reduction_factor': 0.98})

    #from numpy.testing import dec
    #@dec.knownfailureif(True, 'MGS and Householder and scipy.sparse.linalg ' +\
    #                          'do not match for GMRES' +\
    #                          'see https://github.com/pyamg/pyamg/issues/158')

    def test_gmres(self):
        # Ensure repeatability
        random.seed(0)

        #  For these small matrices, Householder and MGS GMRES should give the
        #  same result, and for symmetric (but possibly indefinite) matrices CR
        #  and GMRES should give same result
        for maxiter in [1, 2, 3]:
            for case, symm_case in zip(self.cases, self.symm_cases):
                A = case['A']
                b = case['b']
                x0 = case['x0']
                A_symm = symm_case['A']
                b_symm = symm_case['b']
                x0_symm = symm_case['x0']

                # Test agreement between Householder and GMRES
                (x, flag) = gmres_householder(A, b, x0=x0,
                                              maxiter=min(A.shape[0], maxiter))
                (x2, flag2) = gmres_mgs(A, b, x0=x0, maxiter=min(A.shape[0],
                                        maxiter))
                try:
                    assert_array_almost_equal(x/norm(x), x2/norm(x2),
                                              err_msg='Householder GMRES and MGS\
                                                       GMRES gave different\
                                                       results for small matrix')
                except AssertionError:
                    from nose import SkipTest
                    raise SkipTest('MGS and Householder and scipy.sparse.linalg' +\
                                   'do not match for GMRES' +\
                                   'see https://github.com/pyamg/pyamg/issues/158')

                assert_equal(flag, flag2,
                             err_msg='Householder GMRES and MGS GMRES returned\
                                      different convergence flags for small\
                                      matrix')

                # Test agreement between GMRES and CR
                if A_symm.shape[0] > 1:
                    residuals2 = []
                    (x2, flag2) = gmres_mgs(A_symm, b_symm, x0=x0_symm,
                                            maxiter=min(A.shape[0], maxiter),
                                            residuals=residuals2)
                    residuals3 = []
                    (x3, flag2) = cr(A_symm, b_symm, x0=x0_symm,
                                     maxiter=min(A.shape[0], maxiter),
                                     residuals=residuals3)
                    residuals2 = array(residuals2)
                    residuals3 = array(residuals3)
                    assert_array_almost_equal(residuals3/norm(residuals3),
                                              residuals2/norm(residuals2),
                                              err_msg='CR and GMRES yield\
                                                       different residual\
                                                       vectors')
                    assert_array_almost_equal(x2/norm(x2), x3/norm(x3),
                                              err_msg='CR and GMRES yield\
                                                       different answers')

    #from numpy.testing import dec
    #@dec.knownfailureif(True, 'Oblique projectors fail ' +\
    #                          'see https://github.com/pyamg/pyamg/issues/159')

    def test_krylov(self):
        # Oblique projectors reduce the residual
        for method in self.oblique:
            for case in self.cases:
                A = case['A']
                b = case['b']
                x0 = case['x0']
                (xNew, flag) = method(A, b, x0=x0, tol=case['tol'],
                                      maxiter=case['maxiter'])
                xNew = xNew.reshape(-1, 1)
                try:
                    assert_equal((norm(b - A*xNew)/norm(b - A*x0)) <
                                 case['reduction_factor'], True,
                                 err_msg='Oblique Krylov Method Failed Test')
                except AssertionError:
                    from nose import SkipTest
                    raise SkipTest('Oblique projectors fail ' +\
                                   'see https://github.com/pyamg/pyamg/issues/159')

        # Oblique projectors reduce the residual, here we consider oblique
        # projectors for symmetric matrices
        for method in self.symm_oblique:
            for case in self.symm_cases:
                A = case['A']
                b = case['b']
                x0 = case['x0']
                (xNew, flag) = method(A, b, x0=x0, tol=case['tol'],
                                      maxiter=case['maxiter'])
                xNew = xNew.reshape(-1, 1)
                assert_equal((norm(b - A*xNew)/norm(b - A*x0)) <
                             case['reduction_factor'], True,
                             err_msg='Symmetric oblique Krylov Method Failed\
                                      Test')

        # Orthogonal projectors reduce the error
        for method in self.orth:
            for case in self.cases:
                A = case['A']
                b = case['b']
                x0 = case['x0']
                (xNew, flag) = method(A, b, x0=x0, tol=case['tol'],
                                      maxiter=case['maxiter'])
                xNew = xNew.reshape(-1, 1)
                soln = solve(A, b)
                assert_equal((norm(soln - xNew)/norm(soln - x0)) <
                             case['reduction_factor'], True,
                             err_msg='Orthogonal Krylov Method Failed Test')

        # SPD Orthogonal projectors reduce the error
        for method in self.spd_orth:
            for case in self.spd_cases:
                A = case['A']
                b = case['b']
                x0 = case['x0']
                (xNew, flag) = method(A, b, x0=x0, tol=case['tol'],
                                      maxiter=case['maxiter'])
                xNew = xNew.reshape(-1, 1)
                soln = solve(A, b)
                assert_equal((norm(soln - xNew)/norm(soln - x0)) <
                             case['reduction_factor'], True,
                             err_msg='Orthogonal Krylov Method Failed Test')

        # Assume that Inexact Methods reduce the residual for these examples
        for method in self.inexact:
            for case in self.cases:
                A = case['A']
                b = case['b']
                x0 = case['x0']
                (xNew, flag) = method(A, b, x0=x0, tol=case['tol'],
                                      maxiter=A.shape[0])
                xNew = xNew.reshape(-1, 1)
                assert_equal((norm(b - A*xNew)/norm(b - A*x0)) < 0.15, True,
                             err_msg='Inexact Krylov Method Failed Test')
<commit_msg>fix string continuations<commit_after>from pyamg.krylov import bicgstab, cg, cgne, cgnr, cr, fgmres, gmres
from pyamg.krylov._gmres_householder import gmres_householder
from pyamg.krylov._gmres_mgs import gmres_mgs
from numpy import array, zeros, ones
from scipy import mat, random
from scipy.linalg import solve
from pyamg.util.linalg import norm
import pyamg

from numpy.testing import TestCase, assert_array_almost_equal, assert_equal

class TestKrylov(TestCase):
    def setUp(self):
        self.cases = []
        self.spd_cases = []
        self.symm_cases = []

        # self.oblique = [gmres, fgmres, cgnr,
        #                 krylov._gmres_householder.gmres_householder,
        #                 krylov._gmres_mgs.gmres_mgs]
        self.oblique = [gmres_householder, gmres_mgs, gmres, fgmres, cgnr]
        self.symm_oblique = [cr]
        self.orth = [cgne]
        self.inexact = [bicgstab]
        self.spd_orth = [cg]

        # 1x1
        A = mat([[1.2]])
        b = array([3.9]).reshape(-1, 1)
        x0 = zeros((1, 1))
        self.cases.append({'A': A, 'b': b, 'x0': x0, 'tol': 1e-16,
                           'maxiter': 1, 'reduction_factor': 1e-10})
        self.spd_cases.append({'A': A, 'b': b, 'x0': x0, 'tol': 1e-16,
                               'maxiter': 1, 'reduction_factor': 1e-10})
        self.symm_cases.append({'A': A, 'b': b, 'x0': x0, 'tol': 1e-16,
                                'maxiter': 1, 'reduction_factor': 1e-10})

        # 4x4
        A = mat([[1.2,    0.,   0.,     0.],
                 [0.,     4.,   2.,     6.],
                 [0.,     0.,   9.3,  -2.31],
                 [-4.,     0.,   0.,    -11.]])
        b = array([1., 3.9, 0., -1.23]).reshape(-1, 1)
        x0 = zeros((4, 1))
        self.cases.append({'A': A, 'b': b, 'x0': x0, 'tol': 1e-16,
                           'maxiter': 4, 'reduction_factor': 1e-10})
        self.spd_cases.append({'A': A.T*A, 'b': b, 'x0': x0, 'tol': 1e-16,
                               'maxiter': 4, 'reduction_factor': 1e-10})
        self.symm_cases.append({'A': A.T + A, 'b': b, 'x0': x0, 'tol': 1e-16,
                                'maxiter': 4, 'reduction_factor': 1e-10})

        # 4x4 Imaginary
        A = mat(A, dtype=complex)
        A[0, 0] += 3.1j
        A[3, 3] -= 1.34j
        A[1, 3] *= 1.0j
        A[1, 2] += 1.0j
        b = array([1. - 1.0j, 2.0 - 3.9j, 0., -1.23]).reshape(-1, 1)
        x0 = ones((4, 1))
        self.cases.append({'A': A, 'b': b, 'x0': x0, 'tol': 1e-16,
                           'maxiter': 4, 'reduction_factor': 1e-10})
        self.spd_cases.append({'A': A.H*A, 'b': b, 'x0': x0, 'tol': 1e-16,
                               'maxiter': 4, 'reduction_factor': 1e-10})
        self.symm_cases.append({'A': A.H + A, 'b': b, 'x0': x0, 'tol': 1e-16,
                                'maxiter': 4, 'reduction_factor': 1e-10})

        # 10x10
        A = mat([[-1.1,    0.,   0.,   0.,  3.9,  0.,   0.,  11.,  -1.,  0.],
                 [0.,    4.,   2.9,  0.,   0.,  6.8,  0.,  0.,    0.,  0.],
                 [0.,    0.,   9.0,  0.,   0.,  0.8,  1., -2.2,   0.,  9.],
                 [-4.,    0.,   0.0,  0.,   0.,  0.0,  2.,  2.2,   0.,  0.],
                 [0.,    0.,   0.0, 21.,   0.,  0.1,  0.,   0.,   0.,  0.],
                 [0.,    0.,   0.0,  0.,  -4.7, 0.0,  0.,   0.,   0.,  0.],
                 [2.1,   7.,  22.0,  0.,   0.,  0.0,  0.,   0.,   0.,  0.],
                 [0.,    0.,   0.0, 34.,   0.,  0.0,  0.,   0.,  -12.3, 0.],
                 [0.,   3.4,   0.0,  0.,   0., -0.3,  0.,   0.,   0.,  0.],
                 [9.,    0.,   0.0,  0.,  87.,  0.0,  0.,   0.,   0., -11.2]])
        b = array([1., 0., 0.2, 8., 0., -1.9,
                   11.3, 0.0, 0.1, 0.0]).reshape(-1, 1)
        x0 = zeros((10, 1))
        x0[4] = 11.1
        x0[7] = -2.
        self.cases.append({'A': A, 'b': b, 'x0': x0, 'tol': 1e-16,
                           'maxiter': 2, 'reduction_factor': 0.98})
        self.symm_cases.append({'A': A + A.T, 'b': b, 'x0': x0, 'tol': 1e-16,
                                'maxiter': 2, 'reduction_factor': 0.98})
        self.spd_cases.append({'A':
                               mat(pyamg.gallery.poisson((10,)).todense()),
                               'b': b, 'x0': x0, 'tol': 1e-16, 'maxiter': 2,
                               'reduction_factor': 0.98})

    # from numpy.testing import dec
    # @dec.knownfailureif(True, 'MGS and Householder and scipy.sparse.linalg '
    #                          'do not match for GMRES'
    #                          'see https://github.com/pyamg/pyamg/issues/158')

    def test_gmres(self):
        # Ensure repeatability
        random.seed(0)

        #  For these small matrices, Householder and MGS GMRES should give the
        #  same result, and for symmetric (but possibly indefinite) matrices CR
        #  and GMRES should give same result
        for maxiter in [1, 2, 3]:
            for case, symm_case in zip(self.cases, self.symm_cases):
                A = case['A']
                b = case['b']
                x0 = case['x0']
                A_symm = symm_case['A']
                b_symm = symm_case['b']
                x0_symm = symm_case['x0']

                # Test agreement between Householder and GMRES
                (x, flag) = gmres_householder(A, b, x0=x0,
                                              maxiter=min(A.shape[0], maxiter))
                (x2, flag2) = gmres_mgs(A, b, x0=x0, maxiter=min(A.shape[0],
                                        maxiter))
                try:
                    err_msg = ('Householder GMRES and MGS GMRES gave '
                               'different results for small matrix')
                    assert_array_almost_equal(x/norm(x), x2/norm(x2),
                                              err_msg=err_msg)
                except AssertionError:
                    from nose import SkipTest
                    skip_msg = ('MGS and Householder and scipy.sparse.linalg '
                                'do not match for GMRES'
                                'https://github.com/pyamg/pyamg/issues/158')
                    raise SkipTest(skip_msg)

                err_msg = ('Householder GMRES and MGS GMRES returned '
                           'different convergence flags for small matrix')
                assert_equal(flag, flag2, err_msg=err_msg)

                # Test agreement between GMRES and CR
                if A_symm.shape[0] > 1:
                    residuals2 = []
                    (x2, flag2) = gmres_mgs(A_symm, b_symm, x0=x0_symm,
                                            maxiter=min(A.shape[0], maxiter),
                                            residuals=residuals2)
                    residuals3 = []
                    (x3, flag2) = cr(A_symm, b_symm, x0=x0_symm,
                                     maxiter=min(A.shape[0], maxiter),
                                     residuals=residuals3)
                    residuals2 = array(residuals2)
                    residuals3 = array(residuals3)

                    err_msg = 'CR and GMRES yield different residual vectors'
                    assert_array_almost_equal(residuals3/norm(residuals3),
                                              residuals2/norm(residuals2),
                                              err_msg=err_msg)

                    err_msg = 'CR and GMRES yield different answers'
                    assert_array_almost_equal(x2/norm(x2), x3/norm(x3),
                                              err_msg=err_msg)

    # from numpy.testing import dec
    # @dec.knownfailureif(True, 'Oblique projectors fail '
    #                          'see https://github.com/pyamg/pyamg/issues/159')

    def test_krylov(self):
        # Oblique projectors reduce the residual
        for method in self.oblique:
            for case in self.cases:
                A = case['A']
                b = case['b']
                x0 = case['x0']
                (xNew, flag) = method(A, b, x0=x0, tol=case['tol'],
                                      maxiter=case['maxiter'])
                xNew = xNew.reshape(-1, 1)
                try:
                    assert_equal((norm(b - A*xNew)/norm(b - A*x0)) <
                                 case['reduction_factor'], True,
                                 err_msg='Oblique Krylov Method Failed Test')
                except AssertionError:
                    from nose import SkipTest
                    skip_msg = ('Oblique projectors fail '
                                'https://github.com/pyamg/pyamg/issues/159')
                    raise SkipTest(skip_msg)

        # Oblique projectors reduce the residual, here we consider oblique
        # projectors for symmetric matrices
        for method in self.symm_oblique:
            for case in self.symm_cases:
                A = case['A']
                b = case['b']
                x0 = case['x0']
                (xNew, flag) = method(A, b, x0=x0, tol=case['tol'],
                                      maxiter=case['maxiter'])
                xNew = xNew.reshape(-1, 1)
                assert_equal((norm(b - A*xNew)/norm(b - A*x0)) <
                             case['reduction_factor'], True,
                             err_msg='Symmetric oblique Krylov Method Failed')

        # Orthogonal projectors reduce the error
        for method in self.orth:
            for case in self.cases:
                A = case['A']
                b = case['b']
                x0 = case['x0']
                (xNew, flag) = method(A, b, x0=x0, tol=case['tol'],
                                      maxiter=case['maxiter'])
                xNew = xNew.reshape(-1, 1)
                soln = solve(A, b)
                assert_equal((norm(soln - xNew)/norm(soln - x0)) <
                             case['reduction_factor'], True,
                             err_msg='Orthogonal Krylov Method Failed Test')

        # SPD Orthogonal projectors reduce the error
        for method in self.spd_orth:
            for case in self.spd_cases:
                A = case['A']
                b = case['b']
                x0 = case['x0']
                (xNew, flag) = method(A, b, x0=x0, tol=case['tol'],
                                      maxiter=case['maxiter'])
                xNew = xNew.reshape(-1, 1)
                soln = solve(A, b)
                assert_equal((norm(soln - xNew)/norm(soln - x0)) <
                             case['reduction_factor'], True,
                             err_msg='Orthogonal Krylov Method Failed Test')

        # Assume that Inexact Methods reduce the residual for these examples
        for method in self.inexact:
            for case in self.cases:
                A = case['A']
                b = case['b']
                x0 = case['x0']
                (xNew, flag) = method(A, b, x0=x0, tol=case['tol'],
                                      maxiter=A.shape[0])
                xNew = xNew.reshape(-1, 1)
                assert_equal((norm(b - A*xNew)/norm(b - A*x0)) < 0.15, True,
                             err_msg='Inexact Krylov Method Failed Test')
<|endoftext|>


<commit_before>''' Helper functions for the unittests are defined here.
'''

import numpy as np
import tables as tb

def get_array_differences(first_array, second_array):
    '''Takes two numpy.ndarrays and compares them on a column basis.
    Different column data types, missing columns and columns with different values are returned in a string.

    Parameters
    ----------
    first_array : numpy.ndarray
    second_array : numpy.ndarray

    Returns
    -------
    string
    '''
    if first_array.dtype.names is None:  # normal nd.array
        return ': Sum first array: ' + str(np.sum(first_array)) + ', Sum second array: ' + str(np.sum(second_array))
    else:
        return_str = ''
        for column_name in first_array.dtype.names:
            first_column = first_array[column_name]
            try:
                second_column = second_array[column_name]
            except ValueError:
                return_str += 'No ' + column_name + ' column found. '
                continue
            if (first_column.dtype != second_column.dtype):
                return_str += 'Column ' + column_name + ' has different data type. '
            try:
                if not (first_column == second_column).all():  # check if the data of the column is equal
                    return_str += 'Column ' + column_name + ' not equal. '
            except AttributeError:
                if not (first_column == second_column):
                    return_str += 'Column ' + column_name + ' not equal. '
        for column_name in second_array.dtype.names:
            try:
                first_array[column_name]
            except ValueError:
                return_str += 'Additional column ' + column_name + ' found. '
                continue
        return ': ' + return_str

def array_close(array_1, array_2, rtol=1.e-5, atol=1.e-8):
    '''Compares two numpy arrays elementwise for similarity with small differences.'''
    if not array_1.dtype.names:
        try:
            return np.allclose(array_1, array_2, rtol=1.e-5, atol=1.e-8, equal_nan=True)  # Only works on non recarrays
        except ValueError:  # Raised if shape is incompatible
            return False
    results = []
    for column in array_1.dtype.names:
        results.append(np.allclose(array_1[column], array_2[column], rtol=1.e-5, atol=1.e-8))
    return np.all(results)

def compare_h5_files(first_file, second_file, node_names=None, detailed_comparison=True, exact=True, rtol=1.e-5, atol=1.e-8, chunk_size=1000000):
    '''Takes two hdf5 files and check for equality of all nodes.
    Returns true if the node data is equal and the number of nodes is the number of expected nodes.
    It also returns a error string containing the names of the nodes that are not equal.

    Parameters
    ----------
    first_file : string
        Path to the first file.
    second_file : string
        Path to the second file.
    node_names : list, tuple
        Iterable of node names that are required to exist and will be compared.
        If None, compare all existing nodes and fail if nodes are not existing.
    detailed_comparison : boolean
        Print reason why the comparison failed
    exact : boolean
        True if the results have to match exactly. E.g. False for fit results.
    rtol, atol: number
        From numpy.allclose:
        rtol : float
            The relative tolerance parameter (see Notes).
        atol : float
            The absolute tolerance parameter (see Notes).

    Returns
    -------
    (bool, string)
    '''
    checks_passed = True
    error_msg = ""
    with tb.open_file(first_file, 'r') as first_h5_file:
        with tb.open_file(second_file, 'r') as second_h5_file:
            fist_file_nodes = [node.name for node in first_h5_file.root]  # get node names
            second_file_nodes = [node.name for node in second_h5_file.root]  # get node names
            if node_names is None:
                additional_first_file_nodes = set(fist_file_nodes) - set(second_file_nodes)
                additional_second_file_nodes = set(second_file_nodes) - set(fist_file_nodes)
                if additional_first_file_nodes:
                    checks_passed = False
                    error_msg += 'File %s has additional nodes: %s\n' % (first_file, ', '.join(additional_first_file_nodes))
                if additional_second_file_nodes:
                    checks_passed = False
                    error_msg += 'File %s has additional nodes: %s\n' % (second_file, ', '.join(additional_second_file_nodes))
                common_nodes = set(fist_file_nodes) & set(second_file_nodes)
            else:
                missing_first_file_nodes = set(node_names) - set(fist_file_nodes)
                if missing_first_file_nodes:
                    checks_passed = False
                    error_msg += 'File %s is missing nodes: %s\n' % (first_file, ', '.join(missing_first_file_nodes))
                missing_second_file_nodes = set(node_names) - set(second_file_nodes)
                if missing_second_file_nodes:
                    checks_passed = False
                    error_msg += 'File %s is missing nodes: %s\n' % (second_file, ', '.join(missing_second_file_nodes))
                common_nodes = (set(fist_file_nodes) & set(second_file_nodes)) & set(node_names)
            for node_name in common_nodes:  # loop over all nodes and compare each node, do not abort if one node is wrong
                nrows = first_h5_file.get_node(first_h5_file.root, node_name).nrows
                index_start = 0
                while index_start < nrows:
                    # reduce memory footprint by taken array dimension into account
                    read_nrows = max(1, int(chunk_size / np.prod(first_h5_file.get_node(first_h5_file.root, node_name).shape[1:])))
                    index_stop = index_start + read_nrows
                    first_file_data = first_h5_file.get_node(first_h5_file.root, node_name).read(index_start, index_stop)
                    second_file_data = second_h5_file.get_node(second_h5_file.root, node_name).read(index_start, index_stop)
                    if exact:
                        try:
                            np.testing.assert_array_equal(first_file_data, second_file_data)
                        except AssertionError as e:
                            checks_passed = False
                            error_msg += node_name
                            if detailed_comparison:
                                error_msg += get_array_differences(first_file_data, second_file_data)
                                error_msg += str(e)
                            error_msg += '\n'
                            break
                    else:
                        if not array_close(first_file_data, second_file_data, rtol, atol):
                            checks_passed = False
                            error_msg += node_name
                            if detailed_comparison:
                                error_msg += get_array_differences(first_file_data, second_file_data)
                            error_msg += '\n'
                            break
                    index_start += read_nrows
    return checks_passed, error_msg
<commit_msg>ENH: enhancements for array and HDF5 comparison tests<commit_after>''' Helper functions for the unittests are defined here.
'''

import numpy as np
import tables as tb

def nan_equal(first_array, second_array):
    ''' Compares two arrays and test for equality.

    Works with recarrays.
    NaNs are considered equal.

    Parameters
    ----------
    first_array : numpy.ndarray
    second_array : numpy.ndarray

    Returns
    -------
    boolean
    '''
    # Check for shape
    if first_array.shape != second_array.shape:
        return False
    if first_array.dtype.names is None:  # Not a recarray
        # Check for same dtypes
        if first_array.dtype != second_array.dtype:
            return False
    else:
        # Check for same column names
        if set(first_array.dtype.names) != set(second_array.dtype.names):
            return False
        for column in first_array.dtype.names:
            # Check for same dtypes
            if first_array[column].dtype != second_array[column].dtype:
                return False
    # Check for equality
    try:
        np.testing.assert_equal(actual=first_array, desired=second_array)
    except AssertionError:
        return False
    return True

def nan_close(first_array, second_array, rtol=1.e-5, atol=1.e-8, equal_nan=True):
    ''' Compares two arrays and test for similarity.

    Works with recarrays.

    Parameters
    ----------
    first_array : numpy.ndarray
    second_array : numpy.ndarray
    rtol : float
    atol : float
    equal_nan : boolean
        If True, NaNs are considered equal.

    Returns
    -------
    boolean
    '''
    # Check for shape
    if first_array.shape != second_array.shape:
        return False
    if first_array.dtype.names is None:  # Not a recarray
        # Check for same dtypes
        if first_array.dtype != second_array.dtype:
            return False
        try:
            return np.allclose(a=first_array, b=second_array, rtol=rtol, atol=atol, equal_nan=equal_nan)
        except ValueError:
            return False
    else:
        # Check for same column names
        if set(first_array.dtype.names) != set(second_array.dtype.names):
            return False
        results = []
        for column in first_array.dtype.names:
            # Check for same dtypes
            if first_array[column].dtype != second_array[column].dtype:
                return False
            # Check for similarity
            results.append(np.allclose(a=first_array[column], b=second_array[column], rtol=rtol, atol=atol, equal_nan=equal_nan))
        return all(results)

def get_array_differences(first_array, second_array, exact=True, rtol=1e-5, atol=1e-8, equal_nan=True):
    '''Takes two numpy.ndarrays and compares them on a column basis.
    Different column data types, missing columns and columns with different values are returned in a string.

    Parameters
    ----------
    first_array : numpy.ndarray
    second_array : numpy.ndarray

    Returns
    -------
    string
    '''
    def compare_arrays(actual, desired, exact, rtol, atol, equal_nan):
        compare_str = ''
        if actual.dtype != desired.dtype:
            compare_str += 'Type:\nfirst: %s\nsecond: %s\n' % (str(actual.dtype), str(desired.dtype))
        if actual.shape != desired.shape:
            compare_str += 'Shape:\nfirst: %s\nsecond: %s\n' % (str(actual.shape), str(desired.shape))
        if np.sum(actual) != np.sum(desired):
            compare_str += 'Sum:\nfirst: %s\nsecond: %s\n' % (str(np.sum(actual)), str(np.sum(desired)))
        if exact:
            try:
                np.testing.assert_equal(actual=first_array, desired=second_array)
            except AssertionError as e:
                compare_str += e + "\n"
        else:
            try:
                np.testing.assert_allclose(actual=first_array, desired=second_array, rtol=rtol, atol=atol, equal_nan=equal_nan)
            except AssertionError as e:
                compare_str += e + "\n"
        if compare_str:
            compare_str = "Difference:\n" + compare_str
        else:
            compare_str = "No Difference\n"
        return compare_str

    if first_array.dtype.names is None:  # Not a recarray
        return compare_arrays(actual=first_array, desired=second_array, exact=exact, rtol=rtol, atol=atol, equal_nan=equal_nan)
    else:
        return_str = ''
        first_array_column_names = first_array.dtype.names
        second_array_column_names = second_array.dtype.names
        additional_first_array_column_names = set(first_array_column_names) - set(second_array_column_names)
        additional_second_array_column_names = set(second_array_column_names) - set(first_array_column_names)
        if additional_first_array_column_names:
            error_msg += 'First array has additional columns: %s\n' % ', '.join(additional_first_array_column_names)
        if additional_second_array_column_names:
            error_msg += 'Second array has additional columns: %s\n' % ', '.join(additional_second_array_column_names)
        common_columns = set(first_array_column_names) & set(second_array_column_names)
        for column_name in common_columns:  # loop over all nodes and compare each node, do not abort if one node is wrong
            first_column_data = first_array[column_name]
            second_column_data = second_array[column_name]
            col_str = compare_arrays(actual=first_column_data, desired=second_column_data, exact=exact, rtol=rtol, atol=atol, equal_nan=equal_nan)
            return_str += "Column %s:\n%s" % (column_name, col_str)
        return return_str

def compare_h5_files(first_file, second_file, node_names=None, detailed_comparison=True, exact=True, rtol=1e-5, atol=1e-8, chunk_size=1000000):
    '''Takes two hdf5 files and check for equality of all nodes.
    Returns true if the node data is equal and the number of nodes is the number of expected nodes.
    It also returns a error string containing the names of the nodes that are not equal.

    Parameters
    ----------
    first_file : string
        Path to the first file.
    second_file : string
        Path to the second file.
    node_names : list, tuple
        Iterable of node names that are required to exist and will be compared.
        If None, compare all existing nodes and fail if nodes are not existing.
    detailed_comparison : boolean
        Print reason why the comparison failed
    exact : boolean
        True if the results have to match exactly. E.g. False for fit results.
    rtol, atol: number
        From numpy.allclose:
        rtol : float
            The relative tolerance parameter (see Notes).
        atol : float
            The absolute tolerance parameter (see Notes).

    Returns
    -------
    (bool, string)
    '''
    checks_passed = True
    error_msg = ""
    with tb.open_file(first_file, 'r') as first_h5_file:
        with tb.open_file(second_file, 'r') as second_h5_file:
            fist_file_nodes = [node.name for node in first_h5_file.root]  # get node names
            second_file_nodes = [node.name for node in second_h5_file.root]  # get node names
            if node_names is None:
                additional_first_file_nodes = set(fist_file_nodes) - set(second_file_nodes)
                additional_second_file_nodes = set(second_file_nodes) - set(fist_file_nodes)
                if additional_first_file_nodes:
                    checks_passed = False
                    if detailed_comparison:
                        error_msg += 'File %s has additional nodes: %s\n' % (first_file, ', '.join(additional_first_file_nodes))
                if additional_second_file_nodes:
                    checks_passed = False
                    if detailed_comparison:
                        error_msg += 'File %s has additional nodes: %s\n' % (second_file, ', '.join(additional_second_file_nodes))
                common_nodes = set(fist_file_nodes) & set(second_file_nodes)
            else:
                missing_first_file_nodes = set(node_names) - set(fist_file_nodes)
                if missing_first_file_nodes:
                    checks_passed = False
                    if detailed_comparison:
                        error_msg += 'File %s is missing nodes: %s\n' % (first_file, ', '.join(missing_first_file_nodes))
                missing_second_file_nodes = set(node_names) - set(second_file_nodes)
                if missing_second_file_nodes:
                    checks_passed = False
                    if detailed_comparison:
                        error_msg += 'File %s is missing nodes: %s\n' % (second_file, ', '.join(missing_second_file_nodes))
                common_nodes = (set(fist_file_nodes) & set(second_file_nodes)) & set(node_names)
            for node_name in common_nodes:  # loop over all nodes and compare each node, do not abort if one node is wrong
                nrows = first_h5_file.get_node(first_h5_file.root, node_name).nrows
                index_start = 0
                while index_start < nrows:
                    # reduce memory footprint by taken array dimension into account
                    read_nrows = max(1, int(chunk_size / np.prod(first_h5_file.get_node(first_h5_file.root, node_name).shape[1:])))
                    index_stop = index_start + read_nrows
                    first_file_data = first_h5_file.get_node(first_h5_file.root, node_name).read(index_start, index_stop)
                    second_file_data = second_h5_file.get_node(second_h5_file.root, node_name).read(index_start, index_stop)
                    if exact:
                        if not nan_equal(first_array=first_file_data, second_array=second_file_data):
                            checks_passed = False
                            if detailed_comparison:
                                error_msg += ('Node %s:\n' % node_name) + get_array_differences(first_array=first_file_data, second_array=second_file_data, exact=True)
                            break
                    else:
                        if not nan_close(first_array=first_file_data, second_array=second_file_data, rtol=rtol, atol=atol, equal_nan=True):
                            checks_passed = False
                            if detailed_comparison:
                                error_msg += ('Node %s:\n' % node_name) + get_array_differences(first_array=first_file_data, second_array=second_file_data, exact=False, rtol=rtol, atol=atol, equal_nan=True)
                            break
                    index_start += read_nrows
    if checks_passed:
        error_msg = 'Comparing file %s and %s: OK\n%s' % (first_file, second_file, error_msg)
    else:
        error_msg = 'Comparing file %s and %s: FAILED\n%s' % (first_file, second_file, error_msg)
    return checks_passed, error_msg
<|endoftext|>