<commit_before>from .core.objects import Row, banned_columns
from .core.objects.apiobjects import MainSelection
from .core.exceptions.messages import LoadingMsg as msg

import io
import csv

def fill_empty(n, lst):
    x = len(lst)
    for _ in lst:
        yield _
    yield from ("" for b in (n - x))

def csv_size_limit(size):
    """Changes the csv field size limit.
        :param size: The size limit of the csv data.
        :type size: :class:`type`
    """
    csv.field_size_limit(size)

def forbidden_columns(columns, msg=msg):
    for x in columns:
        if x in banned_columns:
            raise ValueError(
                msg.forbidden_column.format(x))

def _loads(csvdoc, columns=None, cls=Row, delimiter=",", quotechar='"',
          typetransfer=False, csv_size_max=None, newline="\n"):
    """Loads csv, but as a python string

        Note: Due to way python's internal csv library works, identical headers will overwrite each other.
    """
    if csv_size_max:
        csv_size_limit(csv_size_max)
    if isinstance(csvdoc, str):
        csvfile = io.StringIO()
        csvfile.write(csvdoc)
        data = csv.DictReader(csvdoc.split(newline),
                              delimiter=delimiter, quotechar=quotechar)
        if not columns:
            columns = tuple(next(csv.reader(csvdoc.split(
                newline), delimiter=delimiter, quotechar=quotechar)))
    else:
        data = csvdoc
    if columns:
        forbidden_columns(columns)
    elif (not columns) and isinstance(csvdoc, tuple):
        forbidden_columns(csvdoc[0].keys())
    api = MainSelection(data, columns=(
        columns),  cls=cls, typetransfer=typetransfer)
    return api

def _new(columns=None, cls=Row, custom_columns=False):
    # Empty Main Selection
    if isinstance(columns, str):
        columns = [columns]
    if columns:
        forbidden_columns(columns)
    if not columns:
        columns = tuple()
    if not isinstance(columns, tuple):
        columns = tuple(columns)
    return MainSelection(columns=columns, cls=cls, custom_columns=custom_columns)

def convert_to_psv(finalcolumns, csvfile):
    lenfinal_column = len(finalcolumns)
    for row in csvfile:
        yield dict(zip(finalcolumns, fill_empty(lenfinal_column, row)))

def figure_out_columns(columns):
    finalcolumns = []

    # We don't need to completely clean this up, just prevent unusable columns.
    # PSV will automatically handle collisions
    # We need to keep these as close as possible to the original
    # For custom_columns
    empty_counter = 0
    for x in (x.strip() for x in columns):
        result = x
        if result in banned_columns:
            result = "psv_banned_columns_{}".format(x)
        if not result:
            result = "psv_empty_column_{}".format(empty_counter)
            empty_counter += 1
        finalcolumns.append(result)
    return finalcolumns

def _safe_load(csvfile, columns, cls, custom_columns):
    # Implements a much slower DictWriter
    # This function kills the gc, have to do it pretty
    # much manually due to the gc not firing

    try:
        raw_columns = next(csvfile)
    except StopIteration:
        # This is an Empty File, just use _new
        # pass in columns in case of columns being provided
        return _new(columns=columns)

    finalcolumns = figure_out_columns(columns)
    dataset = tuple(convert_to_psv(finalcolumns, csvfile))
    return _loads(dataset, finalcolumns)
<commit_msg>Fixed fill_empty implementation (Added missing range)<commit_after>from .core.objects import Row, banned_columns
from .core.objects.apiobjects import MainSelection
from .core.exceptions.messages import LoadingMsg as msg

import io
import csv

def fill_empty(n, lst):
    x = len(lst)
    for _ in lst:
        yield _
    yield from ("" for b in range(n - x))

def csv_size_limit(size):
    """Changes the csv field size limit.
        :param size: The size limit of the csv data.
        :type size: :class:`type`
    """
    csv.field_size_limit(size)

def forbidden_columns(columns, msg=msg):
    for x in columns:
        if x in banned_columns:
            raise ValueError(
                msg.forbidden_column.format(x))

def _loads(csvdoc, columns=None, cls=Row, delimiter=",", quotechar='"',
          typetransfer=False, csv_size_max=None, newline="\n"):
    """Loads csv, but as a python string

        Note: Due to way python's internal csv library works, identical headers will overwrite each other.
    """
    if csv_size_max:
        csv_size_limit(csv_size_max)
    if isinstance(csvdoc, str):
        csvfile = io.StringIO()
        csvfile.write(csvdoc)
        data = csv.DictReader(csvdoc.split(newline),
                              delimiter=delimiter, quotechar=quotechar)
        if not columns:
            columns = tuple(next(csv.reader(csvdoc.split(
                newline), delimiter=delimiter, quotechar=quotechar)))
    else:
        data = csvdoc
    if columns:
        forbidden_columns(columns)
    elif (not columns) and isinstance(csvdoc, tuple):
        forbidden_columns(csvdoc[0].keys())
    api = MainSelection(data, columns=(
        columns),  cls=cls, typetransfer=typetransfer)
    return api

def _new(columns=None, cls=Row, custom_columns=False):
    # Empty Main Selection
    if isinstance(columns, str):
        columns = [columns]
    if columns:
        forbidden_columns(columns)
    if not columns:
        columns = tuple()
    if not isinstance(columns, tuple):
        columns = tuple(columns)
    return MainSelection(columns=columns, cls=cls, custom_columns=custom_columns)

def convert_to_psv(finalcolumns, csvfile):
    lenfinal_column = len(finalcolumns)
    for row in csvfile:
        yield dict(zip(finalcolumns, fill_empty(lenfinal_column, row)))

def figure_out_columns(columns):
    finalcolumns = []

    # We don't need to completely clean this up, just prevent unusable columns.
    # PSV will automatically handle collisions
    # We need to keep these as close as possible to the original
    # For custom_columns
    empty_counter = 0
    for x in (x.strip() for x in columns):
        result = x
        if result in banned_columns:
            result = "psv_banned_columns_{}".format(x)
        if not result:
            result = "psv_empty_column_{}".format(empty_counter)
            empty_counter += 1
        finalcolumns.append(result)
    return finalcolumns

def _safe_load(csvfile, columns, cls, custom_columns):
    # Implements a much slower DictWriter
    # This function kills the gc, have to do it pretty
    # much manually due to the gc not firing

    try:
        raw_columns = next(csvfile)
    except StopIteration:
        # This is an Empty File, just use _new
        # pass in columns in case of columns being provided
        return _new(columns=columns)

    finalcolumns = figure_out_columns(columns)
    dataset = tuple(convert_to_psv(finalcolumns, csvfile))
    return _loads(dataset, finalcolumns)
<|endoftext|>


<commit_before>import pysam

from .bam_io import index_bam
from .bwa import SimpleAligner
from .cap3 import Cap3Assembly
from .cigar import alternative_alignment_cigar_is_better
from .readtagger import SamTagProcessor

class AssemblyRealigner(object):
    """Assemble and realign reads in a cluster."""

    def __init__(self,
                 input_alignment_file,
                 genome_bwa_index,
                 transposon_bwa_index,
                 tmp_dir=None):  # Make that configurable ...
        """Assemble reads and align contigs in a cluster to improve breakpoints."""
        self.input_alignment_file = input_alignment_file
        self.reference_genome_index = genome_bwa_index
        self.transposon_index = transposon_bwa_index
        self.genome_aligner = SimpleAligner(bwa_index=genome_bwa_index, tmp_dir=tmp_dir)
        self.transposon_aligner = SimpleAligner(bwa_index=transposon_bwa_index, tmp_dir=tmp_dir)

    def collect_reads(self, cluster):
        """Collect reads that could be useful for determining a clusters breakpoint via assembly."""
        if len(cluster.orientation_switches) > 1:
            # For now only use this strategy to refine potential TSDs,
            # may be interesting for improving insertions with single-sided evidence as well.
            start = cluster.min
            end = cluster.max
        else:
            return []
        index_bam(self.input_alignment_file)  # Should not be necessary, but tests fail without this :(
        additional_read_gen = pysam.AlignmentFile(self.input_alignment_file).fetch(tid=cluster.tid, start=start, end=end)
        additional_reads = (r for r in additional_read_gen if r.query_name not in cluster.read_index)
        reads = {}
        for r in additional_reads:
            if r.has_tag('MS') and not r.is_duplicate:
                qname = r.query_name
                if r.is_read1:
                    current_qname = "%s.1" % qname
                    other_qname = "%s.2" % qname
                else:
                    current_qname = "%s.2" % qname
                    other_qname = "%s.1" % qname
                reads[current_qname] = r.query_sequence
                reads[other_qname] = r.get_tag('MS')
        return self.assemble_reads(reads)

    def assemble_reads(self, reads):
        """Assemble potentially informative reads, align and set tags for contigs."""
        if len(reads) > 500:
            return []
        assembly = Cap3Assembly(reads)
        contig_sequences = {i: contig.sequence for i, contig in enumerate(assembly.contigs)}
        genome_aligned_contigs, genome_header = self.genome_aligner.align_contigs(contig_sequences)
        transposon_aligned_contigs, transposon_header = self.transposon_aligner.align_contigs(contig_sequences)
        informative_reads = []
        if transposon_aligned_contigs and genome_aligned_contigs:
            transposon_tags = SamTagProcessor(source_bam=transposon_aligned_contigs, header=transposon_header, tag_mate=False)
            for gc in genome_aligned_contigs:
                if gc.query_name in transposon_tags.result:
                    tag = transposon_tags.result[gc.query_name][False]['s']
                    if alternative_alignment_cigar_is_better(current_cigar=gc.cigar,
                                                             alternative_cigar=tag.cigar,
                                                             same_orientation=gc.is_reverse == tag.is_reverse):
                        gc.set_tag('AD', str(tag))
                        gc.set_tag('AR', str(tag.reference_name()))
                        gc.set_tag('AC', gc.query_name)
                        informative_reads.append(gc)
        return informative_reads
<commit_msg>Use pysam.AlignmentFile in AssemblyRealigner<commit_after>import pysam

from .bam_io import index_bam
from .bwa import SimpleAligner
from .cap3 import Cap3Assembly
from .cigar import alternative_alignment_cigar_is_better
from .readtagger import SamTagProcessor

class AssemblyRealigner(object):
    """Assemble and realign reads in a cluster."""

    def __init__(self,
                 input_alignment_file,
                 genome_bwa_index,
                 transposon_bwa_index,
                 tmp_dir=None):  # Make that configurable ...
        """Assemble reads and align contigs in a cluster to improve breakpoints."""
        self.input_alignment_file = input_alignment_file
        self.reference_genome_index = genome_bwa_index
        self.transposon_index = transposon_bwa_index
        self.genome_aligner = SimpleAligner(bwa_index=genome_bwa_index, tmp_dir=tmp_dir)
        self.transposon_aligner = SimpleAligner(bwa_index=transposon_bwa_index, tmp_dir=tmp_dir)

    def collect_reads(self, cluster):
        """Collect reads that could be useful for determining a clusters breakpoint via assembly."""
        informative_reads = []
        if len(cluster.orientation_switches) > 1:
            # For now only use this strategy to refine potential TSDs,
            # may be interesting for improving insertions with single-sided evidence as well.
            start = cluster.min
            end = cluster.max
            reads = {}
            index_bam(self.input_alignment_file)  # Should not be necessary, but tests fail without this :(
            with pysam.AlignmentFile(self.input_alignment_file) as reader:
                for r in reader.fetch(tid=cluster.tid, start=start, end=end):
                    if r.query_name not in cluster.read_index and r.has_tag('MS') and not r.is_duplicate:
                        qname = r.query_name
                        if r.is_read1:
                            current_qname = "%s.1" % qname
                            other_qname = "%s.2" % qname
                        else:
                            current_qname = "%s.2" % qname
                            other_qname = "%s.1" % qname
                        reads[current_qname] = r.query_sequence
                        reads[other_qname] = r.get_tag('MS')
            if len(reads) < 500:
                informative_reads = self.assemble_reads(reads)
        return informative_reads

    def assemble_reads(self, reads):
        """Assemble potentially informative reads, align and set tags for contigs."""
        assembly = Cap3Assembly(reads)
        contig_sequences = {i: contig.sequence for i, contig in enumerate(assembly.contigs)}
        genome_aligned_contigs, genome_header = self.genome_aligner.align_contigs(contig_sequences)
        transposon_aligned_contigs, transposon_header = self.transposon_aligner.align_contigs(contig_sequences)
        informative_reads = []
        if transposon_aligned_contigs and genome_aligned_contigs:
            transposon_tags = SamTagProcessor(source_bam=transposon_aligned_contigs, header=transposon_header, tag_mate=False)
            for gc in genome_aligned_contigs:
                if gc.query_name in transposon_tags.result:
                    tag = transposon_tags.result[gc.query_name][False]['s']
                    if alternative_alignment_cigar_is_better(current_cigar=gc.cigar,
                                                             alternative_cigar=tag.cigar,
                                                             same_orientation=gc.is_reverse == tag.is_reverse):
                        gc.set_tag('AD', str(tag))
                        gc.set_tag('AR', str(tag.reference_name()))
                        gc.set_tag('AC', gc.query_name)
                        informative_reads.append(gc)
        return informative_reads
<|endoftext|>


<commit_before>"""
Middleware for Proxito.

This is used to take the request and map the host to the proper project slug.

Additional processing is done to get the project from the URL in the ``views.py`` as well.
"""
import logging

from django.conf import settings
from django.shortcuts import render
from django.utils.deprecation import MiddlewareMixin

from readthedocs.projects.models import Domain, Project

log = logging.getLogger(__name__)  # noqa

def map_host_to_project_slug(request):
    """
    Take the request and map the host to the proper project slug.

    We check, in order:

    * The ``HTTP_X_RTD_SLUG`` host header for explicit Project mapping
        - This sets ``request.rtdheader`` True
    * The ``PUBLIC_DOMAIN`` where we can use the subdomain as the project name
        - This sets ``request.subdomain`` True
    * The hostname without port information, which maps to ``Domain`` objects
        - This sets ``request.cname`` True
    """

    host = request.get_host().lower().split(':')[0]
    public_domain = settings.PUBLIC_DOMAIN.lower().split(':')[0]
    host_parts = host.split('.')
    public_domain_parts = public_domain.split('.')

    project_slug = None

    # Explicit Project slug being passed in
    if 'HTTP_X_RTD_SLUG' in request.META:
        project_slug = request.META['HTTP_X_RTD_SLUG'].lower()
        if Project.objects.filter(slug=project_slug).exists():
            request.rtdheader = True
            log.info('Setting project based on X_RTD_SLUG header: %s' % project_slug)
            return project_slug

    if public_domain in host or host == 'proxito':
        # Serve from the PUBLIC_DOMAIN, ensuring it looks like `foo.PUBLIC_DOMAIN`
        if public_domain_parts == host_parts[1:]:
            project_slug = host_parts[0]
            request.subdomain = True
            log.debug('Proxito Public Domain: host=%s', host)
            return project_slug
        else:
            # TODO: This can catch some possibly valid domains (docs.readthedocs.io.com) for example
            # But these feel like they might be phishing, etc. so let's block them for now.
            log.warning('Weird variation on our hostname: host=%s', host)
            return render(
                request, 'core/dns-404.html', context={'host': host}, status=400
            )

    # Serve CNAMEs
    domain = Domain.objects.filter(domain=host).first()
    if domain:
        project_slug = domain.project.slug
        request.cname = True
        log.debug('Proxito CNAME: host=%s', host)
        return project_slug
    else:
        # Some person is CNAMEing to us without configuring a domain - 404.
        log.debug('CNAME 404: host=%s', host)
        return render(
            request, 'core/dns-404.html', context={'host': host}, status=404
        )

class ProxitoMiddleware(MiddlewareMixin):

    """The actual middleware we'll be using in prod."""

    def process_request(self, request):  # noqa
        if any([not settings.USE_SUBDOMAIN, 'localhost' in request.get_host(),
                'testserver' in request.get_host()]):
            log.debug('Not processing Proxito middleware')
            return None

        ret = map_host_to_project_slug(request)

        # Handle returning a response
        if hasattr(ret, 'status_code'):
            return ret

        log.debug('Proxito Project: slug=%s', ret)

        # Otherwise set the slug on the request
        request.host_project_slug = request.slug = ret

        return None
<commit_msg>Fix lint<commit_after>"""
Middleware for Proxito.

This is used to take the request and map the host to the proper project slug.

Additional processing is done to get the project from the URL in the ``views.py`` as well.
"""
import logging

from django.conf import settings
from django.shortcuts import render
from django.utils.deprecation import MiddlewareMixin

from readthedocs.projects.models import Domain, Project

log = logging.getLogger(__name__)  # noqa

def map_host_to_project_slug(request):
    """
    Take the request and map the host to the proper project slug.

    We check, in order:

    * The ``HTTP_X_RTD_SLUG`` host header for explicit Project mapping
        - This sets ``request.rtdheader`` True
    * The ``PUBLIC_DOMAIN`` where we can use the subdomain as the project name
        - This sets ``request.subdomain`` True
    * The hostname without port information, which maps to ``Domain`` objects
        - This sets ``request.cname`` True
    """

    host = request.get_host().lower().split(':')[0]
    public_domain = settings.PUBLIC_DOMAIN.lower().split(':')[0]
    host_parts = host.split('.')
    public_domain_parts = public_domain.split('.')

    project_slug = None

    # Explicit Project slug being passed in
    if 'HTTP_X_RTD_SLUG' in request.META:
        project_slug = request.META['HTTP_X_RTD_SLUG'].lower()
        if Project.objects.filter(slug=project_slug).exists():
            request.rtdheader = True
            log.info('Setting project based on X_RTD_SLUG header: %s', project_slug)
            return project_slug

    if public_domain in host or host == 'proxito':
        # Serve from the PUBLIC_DOMAIN, ensuring it looks like `foo.PUBLIC_DOMAIN`
        if public_domain_parts == host_parts[1:]:
            project_slug = host_parts[0]
            request.subdomain = True
            log.debug('Proxito Public Domain: host=%s', host)
            return project_slug
        # TODO: This can catch some possibly valid domains (docs.readthedocs.io.com) for example
        # But these feel like they might be phishing, etc. so let's block them for now.
        log.warning('Weird variation on our hostname: host=%s', host)
        return render(
            request, 'core/dns-404.html', context={'host': host}, status=400
        )

    # Serve CNAMEs
    domain = Domain.objects.filter(domain=host).first()
    if domain:
        project_slug = domain.project.slug
        request.cname = True
        log.debug('Proxito CNAME: host=%s', host)
        return project_slug

    # Some person is CNAMEing to us without configuring a domain - 404.
    log.debug('CNAME 404: host=%s', host)
    return render(
        request, 'core/dns-404.html', context={'host': host}, status=404
    )

class ProxitoMiddleware(MiddlewareMixin):

    """The actual middleware we'll be using in prod."""

    def process_request(self, request):  # noqa
        if any([not settings.USE_SUBDOMAIN, 'localhost' in request.get_host(),
                'testserver' in request.get_host()]):
            log.debug('Not processing Proxito middleware')
            return None

        ret = map_host_to_project_slug(request)

        # Handle returning a response
        if hasattr(ret, 'status_code'):
            return ret

        log.debug('Proxito Project: slug=%s', ret)

        # Otherwise set the slug on the request
        request.host_project_slug = request.slug = ret

        return None
<|endoftext|>


<commit_before>import csv
import forms

from django.contrib.auth.decorators import user_passes_test
from django.shortcuts import render
from django.core.urlresolvers import reverse
from django.http import HttpResponse
from functools import wraps

from registrasion import views

''' A list of report views objects that can be used to load a list of
reports. '''
_all_report_views = []

class Report(object):

    def __init__(self):
        pass

    def title():
        raise NotImplementedError

    def headings():
        ''' Returns the headings for the report. '''
        raise NotImplementedError

    def rows(content_type):
        '''

        Arguments:
            content_type (str): The content-type for the output format of this
            report.

        Returns:
            An iterator, which yields each row of the data. Each row should
            be an iterable containing the cells, rendered appropriately for
            content_type.
        '''
        raise NotImplementedError

    def _linked_text(self, content_type, address, text):
        '''

        Returns:
            an HTML linked version of text, if the content_type for this report
            is HTMLish, otherwise, the text.
        '''

        if content_type == "text/html":
            return Report._html_link(address, text)
        else:
            return text

    @staticmethod
    def _html_link(address, text):
        return '%s' % (address, text)

class _ReportTemplateWrapper(object):
    ''' Used internally to pass `Report` objects to templates. They effectively
    are used to specify the content_type for a report. '''

    def __init__(self, content_type, report):
        self.content_type = content_type
        self.report = report

    def title(self):
        return self.report.title()

    def headings(self):
        return self.report.headings()

    def rows(self):
        return self.report.rows(self.content_type)

class BasicReport(Report):

    def __init__(self, title, headings, link_view=None):
        super(BasicReport, self).__init__()
        self._title = title
        self._headings = headings
        self._link_view = link_view

    def title(self):
        ''' Returns the title for this report. '''
        return self._title

    def headings(self):
        ''' Returns the headings for the table. '''
        return self._headings

    def cell_text(self, content_type, index, text):
        if index > 0 or not self._link_view:
            return text
        else:
            address = self.get_link(text)
            return self._linked_text(content_type, address, text)

    def get_link(self, argument):
        return reverse(self._link_view, args=[argument])

class ListReport(BasicReport):

    def __init__(self, title, headings, data, link_view=None):
        super(ListReport, self).__init__(title, headings, link_view=link_view)
        self._data = data

    def rows(self, content_type):
        ''' Returns the data rows for the table. '''

        for row in self._data:
            yield [
                self.cell_text(content_type, i, cell)
                for i, cell in enumerate(row)
            ]

class QuerysetReport(BasicReport):

    def __init__(self, title, attributes, queryset, headings=None,
                 link_view=None):
        super(QuerysetReport, self).__init__(
            title, headings, link_view=link_view
        )
        self._attributes = attributes
        self._queryset = queryset

    def headings(self):
        if self._headings is not None:
            return self._headings

        return [
            " ".join(i.split("_")).capitalize() for i in self._attributes
        ]

    def rows(self, content_type):

        def rgetattr(item, attr):
            for i in attr.split("__"):
                item = getattr(item, i)

            if callable(item):
                try:
                    return item()
                except TypeError:
                    pass

            return item

        for row in self._queryset:
            yield [
                self.cell_text(content_type, i, rgetattr(row, attribute))
                for i, attribute in enumerate(self._attributes)
            ]

class Links(Report):

    def __init__(self, title, links):
        '''
        Arguments:
            links ([tuple, ...]): a list of 2-tuples:
                (url, link_text)

        '''
        self._title = title
        self._links = links

    def title(self):
        return self._title

    def headings(self):
        return []

    def rows(self, content_type):
        print self._links
        for url, link_text in self._links:
            yield [
                self._linked_text(content_type, url, link_text)
            ]

def report_view(title, form_type=None):
    ''' Decorator that converts a report view function into something that
    displays a Report.

    Arguments:
        title (str):
            The title of the report.
        form_type (Optional[forms.Form]):
            A form class that can make this report display things. If not
            supplied, no form will be displayed.

    '''

    # Create & return view
    def _report(view):
        report_view = ReportView(view, title, form_type)
        report_view = user_passes_test(views._staff_only)(report_view)
        report_view = wraps(view)(report_view)

        # Add this report to the list of reports.
        _all_report_views.append(report_view)

        return report_view

    return _report

class ReportView(object):

    def __init__(self, inner_view, title, form_type):
        # Consolidate form_type so it has content type and section
        self.inner_view = inner_view
        self.title = title
        self.form_type = form_type

    def __call__(self, request, *a, **k):
        data = ReportViewRequestData(self, request, *a, **k)
        return self.render(data)

    def get_form(self, request):

        # Create a form instance
        if self.form_type is not None:
            form = self.form_type(request.GET)

            # Pre-validate it
            form.is_valid()
        else:
            form = None

        return form

    @classmethod
    def wrap_reports(cls, reports, content_type):
        reports = [
            _ReportTemplateWrapper(content_type, report)
            for report in reports
        ]

        return reports

    def render(self, data):
        renderers = {
            "text/csv": self._render_as_csv,
            "text/html": self._render_as_html,
            None: self._render_as_html,
        }
        render = renderers[data.content_type]
        return render(data)

    def _render_as_html(self, data):
        ctx = {
            "title": self.title,
            "form": data.form,
            "reports": data.reports,
        }

        return render(data.request, "registrasion/report.html", ctx)

    def _render_as_csv(self, data):
        report = data.reports[data.section]

        # Create the HttpResponse object with the appropriate CSV header.
        response = HttpResponse(content_type='text/csv')

        writer = csv.writer(response)
        encode = lambda i: i.encode("utf8") if isinstance(i, unicode) else i
        writer.writerow(list(encode(i) for i in report.headings()))
        for row in report.rows():
            writer.writerow(list(encode(i) for i in row))

        return response

class ReportViewRequestData(object):

    def __init__(self, report_view, request, *a, **k):
        self.report_view = report_view
        self.request = request

        # Calculate other data
        self.form = report_view.get_form(request)

        # Content type and section come from request.GET
        self.content_type = request.GET.get("content_type")
        self.section = request.GET.get("section")
        self.section = int(self.section) if self.section else None

        if self.content_type is None:
            self.content_type = "text/html"

        # Reports come from calling the inner view
        reports = report_view.inner_view(request, self.form, *a, **k)

        # Normalise to a list
        if isinstance(reports, Report):
            reports = [reports]

        # Wrap them in appropriate format
        reports = ReportView.wrap_reports(reports, self.content_type)

        self.reports = reports

def get_all_reports():
    ''' Returns all the views that have been registered with @report '''

    return list(_all_report_views)
<commit_msg>Adds some reporting documentation<commit_after>import csv
import forms

from django.contrib.auth.decorators import user_passes_test
from django.shortcuts import render
from django.core.urlresolvers import reverse
from django.http import HttpResponse
from functools import wraps

from registrasion import views

''' A list of report views objects that can be used to load a list of
reports. '''
_all_report_views = []

class Report(object):

    def __init__(self):
        pass

    def title():
        raise NotImplementedError

    def headings():
        ''' Returns the headings for the report. '''
        raise NotImplementedError

    def rows(content_type):
        '''

        Arguments:
            content_type (str): The content-type for the output format of this
            report.

        Returns:
            An iterator, which yields each row of the data. Each row should
            be an iterable containing the cells, rendered appropriately for
            content_type.
        '''
        raise NotImplementedError

    def _linked_text(self, content_type, address, text):
        '''

        Returns:
            an HTML linked version of text, if the content_type for this report
            is HTMLish, otherwise, the text.
        '''

        if content_type == "text/html":
            return Report._html_link(address, text)
        else:
            return text

    @staticmethod
    def _html_link(address, text):
        return '%s' % (address, text)

class _ReportTemplateWrapper(object):
    ''' Used internally to pass `Report` objects to templates. They effectively
    are used to specify the content_type for a report. '''

    def __init__(self, content_type, report):
        self.content_type = content_type
        self.report = report

    def title(self):
        return self.report.title()

    def headings(self):
        return self.report.headings()

    def rows(self):
        return self.report.rows(self.content_type)

class BasicReport(Report):

    def __init__(self, title, headings, link_view=None):
        super(BasicReport, self).__init__()
        self._title = title
        self._headings = headings
        self._link_view = link_view

    def title(self):
        ''' Returns the title for this report. '''
        return self._title

    def headings(self):
        ''' Returns the headings for the table. '''
        return self._headings

    def cell_text(self, content_type, index, text):
        if index > 0 or not self._link_view:
            return text
        else:
            address = self.get_link(text)
            return self._linked_text(content_type, address, text)

    def get_link(self, argument):
        return reverse(self._link_view, args=[argument])

class ListReport(BasicReport):

    def __init__(self, title, headings, data, link_view=None):
        super(ListReport, self).__init__(title, headings, link_view=link_view)
        self._data = data

    def rows(self, content_type):
        ''' Returns the data rows for the table. '''

        for row in self._data:
            yield [
                self.cell_text(content_type, i, cell)
                for i, cell in enumerate(row)
            ]

class QuerysetReport(BasicReport):

    def __init__(self, title, attributes, queryset, headings=None,
                 link_view=None):
        super(QuerysetReport, self).__init__(
            title, headings, link_view=link_view
        )
        self._attributes = attributes
        self._queryset = queryset

    def headings(self):
        if self._headings is not None:
            return self._headings

        return [
            " ".join(i.split("_")).capitalize() for i in self._attributes
        ]

    def rows(self, content_type):

        def rgetattr(item, attr):
            for i in attr.split("__"):
                item = getattr(item, i)

            if callable(item):
                try:
                    return item()
                except TypeError:
                    pass

            return item

        for row in self._queryset:
            yield [
                self.cell_text(content_type, i, rgetattr(row, attribute))
                for i, attribute in enumerate(self._attributes)
            ]

class Links(Report):

    def __init__(self, title, links):
        '''
        Arguments:
            links ([tuple, ...]): a list of 2-tuples:
                (url, link_text)

        '''
        self._title = title
        self._links = links

    def title(self):
        return self._title

    def headings(self):
        return []

    def rows(self, content_type):
        print self._links
        for url, link_text in self._links:
            yield [
                self._linked_text(content_type, url, link_text)
            ]

def report_view(title, form_type=None):
    ''' Decorator that converts a report view function into something that
    displays a Report.

    Arguments:
        title (str):
            The title of the report.
        form_type (Optional[forms.Form]):
            A form class that can make this report display things. If not
            supplied, no form will be displayed.

    '''

    # Create & return view
    def _report(view):
        report_view = ReportView(view, title, form_type)
        report_view = user_passes_test(views._staff_only)(report_view)
        report_view = wraps(view)(report_view)

        # Add this report to the list of reports.
        _all_report_views.append(report_view)

        return report_view

    return _report

class ReportView(object):
    ''' View objects that can render report data into HTML or CSV. '''

    def __init__(self, inner_view, title, form_type):
        '''

        Arguments:
            inner_view: Callable that returns either a Report or a sequence of
                Report objects.

            title: The title that appears at the top of all of the reports.

            form_type: A Form class that can be used to query the report.

        '''

        # Consolidate form_type so it has content type and section
        self.inner_view = inner_view
        self.title = title
        self.form_type = form_type

    def __call__(self, request, *a, **k):
        data = ReportViewRequestData(self, request, *a, **k)
        return self.render(data)

    def get_form(self, request):

        ''' Creates an instance of self.form_type using request.GET '''

        # Create a form instance
        if self.form_type is not None:
            form = self.form_type(request.GET)

            # Pre-validate it
            form.is_valid()
        else:
            form = None

        return form

    @classmethod
    def wrap_reports(cls, reports, content_type):
        ''' Wraps the reports in a _ReportTemplateWrapper for the given
        content_type -- this allows data to be returned as HTML links, for
        instance. '''

        reports = [
            _ReportTemplateWrapper(content_type, report)
            for report in reports
        ]

        return reports

    def render(self, data):
        ''' Renders the reports based on data.content_type's value.

        Arguments:
            data (ReportViewRequestData): The report data. data.content_type
                is used to determine how the reports are rendered.

        Returns:
            HTTPResponse: The rendered version of the report.

        '''
        renderers = {
            "text/csv": self._render_as_csv,
            "text/html": self._render_as_html,
            None: self._render_as_html,
        }
        render = renderers[data.content_type]
        return render(data)

    def _render_as_html(self, data):
        ctx = {
            "title": self.title,
            "form": data.form,
            "reports": data.reports,
        }

        return render(data.request, "registrasion/report.html", ctx)

    def _render_as_csv(self, data):
        report = data.reports[data.section]

        # Create the HttpResponse object with the appropriate CSV header.
        response = HttpResponse(content_type='text/csv')

        writer = csv.writer(response)
        encode = lambda i: i.encode("utf8") if isinstance(i, unicode) else i
        writer.writerow(list(encode(i) for i in report.headings()))
        for row in report.rows():
            writer.writerow(list(encode(i) for i in row))

        return response

class ReportViewRequestData(object):
    '''

    Attributes:
        form (Form): form based on request
        reports ([Report, ...]): The reports rendered from the request

    Arguments:
        report_view (ReportView): The ReportView to call back to.
        request (HTTPRequest): A django HTTP request

    '''

    def __init__(self, report_view, request, *a, **k):

        self.report_view = report_view
        self.request = request

        # Calculate other data
        self.form = report_view.get_form(request)

        # Content type and section come from request.GET
        self.content_type = request.GET.get("content_type")
        self.section = request.GET.get("section")
        self.section = int(self.section) if self.section else None

        if self.content_type is None:
            self.content_type = "text/html"

        # Reports come from calling the inner view
        reports = report_view.inner_view(request, self.form, *a, **k)

        # Normalise to a list
        if isinstance(reports, Report):
            reports = [reports]

        # Wrap them in appropriate format
        reports = ReportView.wrap_reports(reports, self.content_type)

        self.reports = reports

def get_all_reports():
    ''' Returns all the views that have been registered with @report '''

    return list(_all_report_views)
<|endoftext|>


<commit_before>'''
-----------------------------------------------------------------------   |
update github with all of the changes you have made!!!
create a pull request with the branch you made and commited all changes
delete unnesessary folders from github

'''
import re
import spacy
nlp = spacy.load('en')
from nlpSpacy import pos_tup_list, remove_POS_matches

def subordinate_removal(s):
    def conjunct_lst_found(lst,type_find,conjunction):
        match_found = []
        if type_find == "Upper":
            match_found.append(lst[0])
            for i in range(1,len(lst)):
                word = lst[i][0]
                if (word != ","):
                    match_found.append(lst[i])
                else:
                    match_found.append(lst[i])
                    return match_found
        else:
            for i in range(0,len(lst)):
                word = lst[i][0]
                if word == conjunction:
                    match_found.append(lst[i])
                    for j in range(i+1,len(lst)-1):
                        match_found.append(lst[j])
                    return match_found

    def subordinate_remove(s):
        complete_pos_lst = pos_tup_list(s)
        conjunct_upper = re.compile('(After|Although|As|Because|Before|' \
                                    'If|Once|Since|Than|That|Though|' \
                                    'Unless|Until|When|Whenever|Where|' \
                                    'Whereas|Wherever|Whether|While|Why|' \
                                    'Even if|Even though|Provided that|' \
                                    'Rather than|So that|In order that)(?=\s)')
        conjunct_lower = re.compile('(?<=\s)(after|although|as|because|' \
                                    'before|if|once|since|than|that|though|' \
                                    'unless|until|when|whenever|where|' \
                                    'whereas|wherever|whether|while|why|' \
                                    'even if|even though|provided that|' \
                                    'rather than|so that|in order that)(?=\s)')
        match_upper = conjunct_upper.search(s)
        match_lower = conjunct_lower.search(s)
        if (match_upper != None and s.find(",") > -1):
            conjunc_found = conjunct_lst_found(complete_pos_lst,
                                               "Upper",
                                               "")
            upd_pos_list = remove_POS_matches(complete_pos_lst,
                                              conjunc_found,
                                              "")
            return (complete_pos_lst,upd_pos_list,conjunc_found)
        elif (match_lower != None):
            word_match = match_lower.group(0)
            conjunc_found = conjunct_lst_found(complete_pos_lst,
                                               "Lower",
                                               word_match)
            upd_pos_list = remove_POS_matches(complete_pos_lst,
                                              conjunc_found,
                                              "")
            return (complete_pos_lst,upd_pos_list,conjunc_found)
        else:
            return ("ERROR", complete_pos_lst)

    return subordinate_remove(s)
<commit_msg>Added additional comments.<commit_after>'''
The main function in this file is subordinate_removal(s). This function
handles sentences that contain subordinate conjunctions in them. It
removes the subordinate conjunction from the sentence in the following
ways: If a subordinate conjunction is in the beginnig of a sentence,
then it removes that word and all of the words following it until a
comma is reached. In the sentence "After we went to the park, Billy
went swimming." the string "After we went to the park," would be
removed. In the sentence "I think we should go hiking after the
football game" the words "after the football game" would be removed.
Args:
    s: A string that is a complete sentence. I will refer to the string
    as a sentence below.
Returns:
    A tuple with three inputs:
    The first input is list of tuples describing all of the words in
    the sentence. Each tuple in this list contains the word,
    dependency, and pos for every word in the sentence, note that
    punctuation is also considered a word.
    The second input contains a similar list as the first input in the
    return statement, but it does not contain the subordinate conjunction
    match that was found.
    The third input is list of a list of tuples that contain the
    subordinate conjunction match found.
'''
import re
import spacy
nlp = spacy.load('en')
from nlpSpacy import pos_tup_list, remove_POS_matches

def subordinate_removal(s):
    def conjunct_lst_found(lst,type_find,conjunction):
        #this function returns a list of tuples that make up a
        #subordinate conjunction
        match_found = []
        if type_find == "Upper":
            match_found.append(lst[0])
            for i in range(1,len(lst)):
                word = lst[i][0]
                if (word != ","):
                    match_found.append(lst[i])
                else:
                    match_found.append(lst[i])
                    return match_found
        else:
            for i in range(0,len(lst)):
                word = lst[i][0]
                if word == conjunction:
                    match_found.append(lst[i])
                    for j in range(i+1,len(lst)-1):
                        match_found.append(lst[j])
                    return match_found

    def subordinate_remove(s):
        #this removes a subordinate conjunction from a string
        complete_pos_lst = pos_tup_list(s)
        conjunct_upper = re.compile('(After|Although|As|Because|Before|' \
                                    'If|Once|Since|Than|That|Though|' \
                                    'Unless|Until|When|Whenever|Where|' \
                                    'Whereas|Wherever|Whether|While|Why|' \
                                    'Even if|Even though|Provided that|' \
                                    'Rather than|So that|In order that)(?=\s)')
        conjunct_lower = re.compile('(?<=\s)(after|although|as|because|' \
                                    'before|if|once|since|than|that|though|' \
                                    'unless|until|when|whenever|where|' \
                                    'whereas|wherever|whether|while|why|' \
                                    'even if|even though|provided that|' \
                                    'rather than|so that|in order that)(?=\s)')
        match_upper = conjunct_upper.search(s)
        match_lower = conjunct_lower.search(s)
        if (match_upper != None and s.find(",") > -1):
            conjunc_found = conjunct_lst_found(complete_pos_lst,
                                               "Upper",
                                               "")
            upd_pos_list = remove_POS_matches(complete_pos_lst,
                                              conjunc_found,
                                              "")
            return (complete_pos_lst,upd_pos_list,conjunc_found)
        elif (match_lower != None):
            word_match = match_lower.group(0)
            conjunc_found = conjunct_lst_found(complete_pos_lst,
                                               "Lower",
                                               word_match)
            upd_pos_list = remove_POS_matches(complete_pos_lst,
                                              conjunc_found,
                                              "")
            return (complete_pos_lst,upd_pos_list,conjunc_found)
        else:
            return ("ERROR", complete_pos_lst)

    return subordinate_remove(s)
<|endoftext|>


<commit_before>

from paste.script.templates import Template, var
from paste.util.template import paste_script_template_renderer

class FalconTemplate(Template):

    _template_dir = 'falcon'
    summary = "A basic daixm microserivce falcon package"
    vars = [
        var('version', 'Version (like 0.1)'),
        var('description', 'One-line description of the package'),
        var('long_description', 'Multi-line description (in reST)'),
        var('keywords', 'Space-separated keywords/tags'),
        var('author', 'Author name'),
        var('author_email', 'Author email'),
        var('url', 'URL of homepage'),
        ]

    template_renderer = staticmethod(paste_script_template_renderer)

<commit_msg>gitignore<commit_after>

import os
from paste.script.templates import Template, var
from paste.util.template import paste_script_template_renderer

class FalconTemplate(Template):

    _template_dir = 'falcon'
    summary = "A basic daixm microserivce falcon package"
    vars = [
        var('version', 'Version (like 0.1)'),
        var('description', 'One-line description of the package'),
        var('long_description', 'Multi-line description (in reST)'),
        var('keywords', 'Space-separated keywords/tags'),
        var('author', 'Author name'),
        var('author_email', 'Author email'),
        var('url', 'URL of homepage'),
        ]

    template_renderer = staticmethod(paste_script_template_renderer)

    def post(self, command, output_dir, vars):
        os.rename('%s/gitignore' % output_dir, '%s/.gitignore' % output_dir)
<|endoftext|>


<commit_before>"""
Part of the code below is copied and modified from:

- dask 0.14.3 (Copyright (c) 2014-2015, Continuum Analytics, Inc.
  and contributors)
  Licensed under the BSD 3 License
  http://dask.pydata.org

"""
from collections import OrderedDict

from .variable import (AbstractVariable, Variable, ForeignVariable,
                       UndefinedVariable, VariableList)
from .process import Process
from ..core.utils import AttrMapping
from ..core.formatting import (_calculate_col_width, pretty_print,
                               maybe_truncate)

def _set_process_names(processes):
    # type: Dict[str, Process]
    """Set process names using keys of the mapping."""
    for k, p in processes.items():
        p._name = k

def _get_foreign_vars(processes):
    # type: Dict[str, Process] -> Dict[str, List[ForeignVariable]]

    foreign_vars = {}

    for proc_name, proc in processes.items():
        foreign_vars[proc_name] = []

        for var in proc._variables.values():
            if isinstance(var, (tuple, list)):
                foreign_vars[proc_name] += [
                    v for v in var if isinstance(v, ForeignVariable)
                ]
            elif isinstance(var, ForeignVariable):
                foreign_vars[proc_name].append(var)

    return foreign_vars

def _link_foreign_vars(processes):
    """Assign process instances to foreign variables."""
    proc_lookup = {v.__class__: v for v in processes.values()}

    for variables in _get_foreign_vars(processes).values():
        for var in variables:
            var._other_process_obj = proc_lookup[var._other_process_cls]

def _get_input_vars(processes):
    # type: Dict[str, Process] -> Dict[str, Dict[str, Variable]]

    input_vars = {}

    for proc_name, proc in processes.items():
        input_vars[proc_name] = {}

        for k, var in proc._variables.items():
            if isinstance(var, Variable) and not var.provided:
                input_vars[proc_name][k] = var

    # case of variables provided by other processes
    foreign_vars = _get_foreign_vars(processes)
    for variables in foreign_vars.values():
        for var in variables:
            if input_vars[var.ref_process.name].get(var.var_name, False):
                if var.provided:
                    del input_vars[var.ref_process.name][var.var_name]

    return {k: v for k, v in input_vars.items() if v}

def _get_process_dependencies(processes):
    # type: Dict[str, Process] -> Dict[str, List[Process]]

    dep_processes = {k: set() for k in processes}
    foreign_vars = _get_foreign_vars(processes)

    for proc_name, variables in foreign_vars.items():
        for var in variables:
            if var.provided:
                dep_processes[var.ref_process.name].add(proc_name)
            else:
                ref_var = var.ref_var
                if ref_var.provided or getattr(ref_var, 'optional', False):
                    dep_processes[proc_name].add(var.ref_process.name)

    return {k: list(v) for k, v in dep_processes.items()}

def _sort_processes(dep_processes):
    # type: Dict[str, List[Process]] -> List[str]
    """Stack-based depth-first search traversal.

    This is based on Tarjan's method for topological sorting.
    Code taken and modified from Dask (New BSD License,
    https://github.com/dask/dask).

    """
    ordered = []

    # Nodes whose descendents have been completely explored.
    # These nodes are guaranteed to not be part of a cycle.
    completed = set()

    # All nodes that have been visited in the current traversal.  Because
    # we are doing depth-first search, going "deeper" should never result
    # in visiting a node that has already been seen.  The `seen` and
    # `completed` sets are mutually exclusive; it is okay to visit a node
    # that has already been added to `completed`.
    seen = set()

    for key in dep_processes:
        if key in completed:
            continue
        nodes = [key]
        while nodes:
            # Keep current node on the stack until all descendants are visited
            cur = nodes[-1]
            if cur in completed:
                # Already fully traversed descendants of cur
                nodes.pop()
                continue
            seen.add(cur)

            # Add direct descendants of cur to nodes stack
            next_nodes = []
            for nxt in dep_processes[cur]:
                if nxt not in completed:
                    if nxt in seen:
                        # Cycle detected!
                        cycle = [nxt]
                        while nodes[-1] != nxt:
                            cycle.append(nodes.pop())
                        cycle.append(nodes.pop())
                        cycle.reverse()
                        cycle = '->'.join(cycle)
                        raise RuntimeError(
                            "Cycle detected in process graph: %s" % cycle
                        )
                    next_nodes.append(nxt)

            if next_nodes:
                nodes.extend(next_nodes)
            else:
                # cur has no more descendants to explore, so we're done with it
                ordered.append(cur)
                completed.add(cur)
                seen.remove(cur)
                nodes.pop()
    return ordered

class Model(AttrMapping):
    """An immutable collection (mapping) of process units that together
    form a computational model.

    This collection is ordered such that the computational flow is
    consistent with process inter-dependencies.

    Ordering doesn't need to be explicitly provided ; it is dynamically
    computed using the processes interfaces.

    Processes interfaces are also used for automatically retrieving
    the model inputs, i.e., all the variables which require setting a
    value before running the model.

    """
    def __init__(self, processes):
        """
        Parameters
        ----------
        processes : dict
            Dictionnary with process names as keys and `Process`
            objects as values. The order doesn't matter.

        """
        for k, p in processes.items():
            if not isinstance(p, Process):
                raise TypeError("%s is not a Process object" % p)

        _set_process_names(processes)
        _link_foreign_vars(processes)

        self._input_vars = _get_input_vars(processes)
        self._dep_processes = _get_process_dependencies(processes)
        self._processes = OrderedDict(
            [(k, processes[k]) for k in _sort_processes(self._dep_processes)]
        )
        self._time_processes = OrderedDict(
            [(k, proc) for k, proc in self._processes.items()
             if proc.meta['time_dependent']]
        )

        super(Model, self).__init__(self._processes)
        self._initialized = True

    def _get_proc_var_name(self, variable):
        # type: AbstractVariable -> Union[tuple[str, str], None]
        for proc_name, variables in self._processes.items():
            for var_name, var in variables.items():
                if var is variable:
                    return proc_name, var_name
        return None

    @property
    def input_vars(self):
        """Returns all variables that require setting a
        value before running the model.

        These variables are grouped by process name (dict of dicts).

        """
        return self._input_vars

    def is_input(self, variable):
        """Returns True if the variable is a model input.

        Parameters
        ----------
        variable : object or tuple
            Either a Variable object or a (str, str) tuple
            corresponding to process name and variable name.

        """
        if isinstance(variable, AbstractVariable):
            proc_name, var_name = self._get_proc_var_name(variable)
        elif isinstance(variable, VariableList):
            # VariableList objects are never model inputs
            return False
        else:
            proc_name, var_name = variable

        return self._input_vars.get(proc_name, {}).get(var_name, False)

    def visualize(self, show_only_variable=None, show_inputs=False,
                  show_variables=False):
        """Render the model as a graph using dot (require graphviz).

        Parameters
        ----------
        show_only_variable : object or tuple, optional
            Show only a variable (and all other linked variables) given either
            as a Variable object or a tuple corresponding to process name and
            variable name. Deactivated by default.
        show_inputs : bool, optional
            If True, show all input variables in the graph (default: False).
            Ignored if `show_only_variable` is not None.
        show_variabless : bool, optional
            If True, show also the other variables (default: False).
            Ignored if `show_only_variable` is not None.

        See Also
        --------
        dot.dot_graph

        """
        from .dot import dot_graph
        return dot_graph(self, show_only_variable=show_only_variable,
                         show_inputs=show_inputs,
                         show_variables=show_variables)

    def _get_dsk(self, func='run_step'):
        return {k: (getattr(self._processes[k], func), v)
                for k, v in self._dep_processes.items()}

    def initalize(self):
        """Initalize a model run for all processes in the model."""
        for proc in self._processes.values():
            proc.initialize()

    def run_step(self, step):
        """Run a (time) step."""
        for proc in self._time_processes.values():
            proc.run_step(step)

    def finalize(self):
        """Finalize a model run for all processes in the model."""
        for proc in self._processes.values():
            proc.finalize()

    def _create_new_model(self, processes):
        """Create a new Model object with new process instances."""
        new_processes = {}

        for proc_name, proc in processes.items():
            proc_cls = type(proc)
            undef_var_names = [k for k, v in proc_cls.variables.items()
                               if isinstance(v, UndefinedVariable)]
            undef_vars = {k: proc.variables[k] for k in undef_var_names}
            new_processes[proc_name] = type(proc)(**undef_vars)

        return type(self)(new_processes)

    def update_processes(self, processes):
        """Add or replace processe(s) in this model.

        Parameters
        ----------
        processes : dict
            Dictionnary with process names as keys and
            `Process` objects as values.

        Returns
        -------
        updated : Model
            New Model instance with updated processes.

        """
        new_processes = self._processes.copy()
        new_processes.update(processes)
        return self._create_new_model(new_processes)

    def drop_processes(self, names):
        """Drop processe(s) from this model.

        Parameters
        ----------
        names : str or list of str
            Name(s) of the processes to drop.

        Returns
        -------
        dropped : Model
            New Model instance with dropped processes.

        """
        if isinstance(names, str):
            names = [names]

        new_processes = self._processes.copy()
        for n in names:
            del new_processes[n]
        return self._create_new_model(new_processes)

    def __repr__(self):
        n_inputs = sum([len(v) for v in self._input_vars.values()])

        hdr = ("<fastscape.models.Model (%d processes, %d inputs)>\n"
               % (len(self._processes), n_inputs))

        max_line_length = 70
        col_width = max([_calculate_col_width(var)
                         for var in self._input_vars.values()])

        blocks = []
        for proc_name in self._processes:
            proc_str = "%s" % proc_name

            inputs = self._input_vars.get(proc_name, {})
            lines = []
            for name, var in inputs.items():
                line = pretty_print("    %s " % name, col_width)
                line += maybe_truncate("(in) %s" % var.description,
                                       max_line_length - col_width)
                lines.append(line)

            if lines:
                proc_str += '\n' + '\n'.join(lines)
            blocks.append(proc_str)

        return hdr + '\n'.join(blocks)
<commit_msg>add Model.run method that accepts an xarray.Dataset object<commit_after>"""
Part of the code below is copied and modified from:

- dask 0.14.3 (Copyright (c) 2014-2015, Continuum Analytics, Inc.
  and contributors)
  Licensed under the BSD 3 License
  http://dask.pydata.org

"""
from collections import OrderedDict

from .variable import (AbstractVariable, Variable, ForeignVariable,
                       UndefinedVariable, VariableList)
from .process import Process
from ..core.utils import AttrMapping
from ..core.formatting import (_calculate_col_width, pretty_print,
                               maybe_truncate)

def _set_process_names(processes):
    # type: Dict[str, Process]
    """Set process names using keys of the mapping."""
    for k, p in processes.items():
        p._name = k

def _get_foreign_vars(processes):
    # type: Dict[str, Process] -> Dict[str, List[ForeignVariable]]

    foreign_vars = {}

    for proc_name, proc in processes.items():
        foreign_vars[proc_name] = []

        for var in proc._variables.values():
            if isinstance(var, (tuple, list)):
                foreign_vars[proc_name] += [
                    v for v in var if isinstance(v, ForeignVariable)
                ]
            elif isinstance(var, ForeignVariable):
                foreign_vars[proc_name].append(var)

    return foreign_vars

def _link_foreign_vars(processes):
    """Assign process instances to foreign variables."""
    proc_lookup = {v.__class__: v for v in processes.values()}

    for variables in _get_foreign_vars(processes).values():
        for var in variables:
            var._other_process_obj = proc_lookup[var._other_process_cls]

def _get_input_vars(processes):
    # type: Dict[str, Process] -> Dict[str, Dict[str, Variable]]

    input_vars = {}

    for proc_name, proc in processes.items():
        input_vars[proc_name] = {}

        for k, var in proc._variables.items():
            if isinstance(var, Variable) and not var.provided:
                input_vars[proc_name][k] = var

    # case of variables provided by other processes
    foreign_vars = _get_foreign_vars(processes)
    for variables in foreign_vars.values():
        for var in variables:
            if input_vars[var.ref_process.name].get(var.var_name, False):
                if var.provided:
                    del input_vars[var.ref_process.name][var.var_name]

    return {k: v for k, v in input_vars.items() if v}

def _get_process_dependencies(processes):
    # type: Dict[str, Process] -> Dict[str, List[Process]]

    dep_processes = {k: set() for k in processes}
    foreign_vars = _get_foreign_vars(processes)

    for proc_name, variables in foreign_vars.items():
        for var in variables:
            if var.provided:
                dep_processes[var.ref_process.name].add(proc_name)
            else:
                ref_var = var.ref_var
                if ref_var.provided or getattr(ref_var, 'optional', False):
                    dep_processes[proc_name].add(var.ref_process.name)

    return {k: list(v) for k, v in dep_processes.items()}

def _sort_processes(dep_processes):
    # type: Dict[str, List[Process]] -> List[str]
    """Stack-based depth-first search traversal.

    This is based on Tarjan's method for topological sorting.
    Code taken and modified from Dask (New BSD License,
    https://github.com/dask/dask).

    """
    ordered = []

    # Nodes whose descendents have been completely explored.
    # These nodes are guaranteed to not be part of a cycle.
    completed = set()

    # All nodes that have been visited in the current traversal.  Because
    # we are doing depth-first search, going "deeper" should never result
    # in visiting a node that has already been seen.  The `seen` and
    # `completed` sets are mutually exclusive; it is okay to visit a node
    # that has already been added to `completed`.
    seen = set()

    for key in dep_processes:
        if key in completed:
            continue
        nodes = [key]
        while nodes:
            # Keep current node on the stack until all descendants are visited
            cur = nodes[-1]
            if cur in completed:
                # Already fully traversed descendants of cur
                nodes.pop()
                continue
            seen.add(cur)

            # Add direct descendants of cur to nodes stack
            next_nodes = []
            for nxt in dep_processes[cur]:
                if nxt not in completed:
                    if nxt in seen:
                        # Cycle detected!
                        cycle = [nxt]
                        while nodes[-1] != nxt:
                            cycle.append(nodes.pop())
                        cycle.append(nodes.pop())
                        cycle.reverse()
                        cycle = '->'.join(cycle)
                        raise RuntimeError(
                            "Cycle detected in process graph: %s" % cycle
                        )
                    next_nodes.append(nxt)

            if next_nodes:
                nodes.extend(next_nodes)
            else:
                # cur has no more descendants to explore, so we're done with it
                ordered.append(cur)
                completed.add(cur)
                seen.remove(cur)
                nodes.pop()
    return ordered

class Model(AttrMapping):
    """An immutable collection (mapping) of process units that together
    form a computational model.

    This collection is ordered such that the computational flow is
    consistent with process inter-dependencies.

    Ordering doesn't need to be explicitly provided ; it is dynamically
    computed using the processes interfaces.

    Processes interfaces are also used for automatically retrieving
    the model inputs, i.e., all the variables which require setting a
    value before running the model.

    """
    def __init__(self, processes):
        """
        Parameters
        ----------
        processes : dict
            Dictionnary with process names as keys and `Process`
            objects as values. The order doesn't matter.

        """
        for k, p in processes.items():
            if not isinstance(p, Process):
                raise TypeError("%s is not a Process object" % p)

        _set_process_names(processes)
        _link_foreign_vars(processes)

        self._input_vars = _get_input_vars(processes)
        self._dep_processes = _get_process_dependencies(processes)
        self._processes = OrderedDict(
            [(k, processes[k]) for k in _sort_processes(self._dep_processes)]
        )
        self._time_processes = OrderedDict(
            [(k, proc) for k, proc in self._processes.items()
             if proc.meta['time_dependent']]
        )

        super(Model, self).__init__(self._processes)
        self._initialized = True

    def _get_proc_var_name(self, variable):
        # type: AbstractVariable -> Union[tuple[str, str], None]
        for proc_name, variables in self._processes.items():
            for var_name, var in variables.items():
                if var is variable:
                    return proc_name, var_name
        return None

    @property
    def input_vars(self):
        """Returns all variables that require setting a
        value before running the model.

        These variables are grouped by process name (dict of dicts).

        """
        return self._input_vars

    def is_input(self, variable):
        """Returns True if the variable is a model input.

        Parameters
        ----------
        variable : object or tuple
            Either a Variable object or a (str, str) tuple
            corresponding to process name and variable name.

        """
        if isinstance(variable, AbstractVariable):
            proc_name, var_name = self._get_proc_var_name(variable)
        elif isinstance(variable, VariableList):
            # VariableList objects are never model inputs
            return False
        else:
            proc_name, var_name = variable

        return self._input_vars.get(proc_name, {}).get(var_name, False)

    def visualize(self, show_only_variable=None, show_inputs=False,
                  show_variables=False):
        """Render the model as a graph using dot (require graphviz).

        Parameters
        ----------
        show_only_variable : object or tuple, optional
            Show only a variable (and all other linked variables) given either
            as a Variable object or a tuple corresponding to process name and
            variable name. Deactivated by default.
        show_inputs : bool, optional
            If True, show all input variables in the graph (default: False).
            Ignored if `show_only_variable` is not None.
        show_variabless : bool, optional
            If True, show also the other variables (default: False).
            Ignored if `show_only_variable` is not None.

        See Also
        --------
        dot.dot_graph

        """
        from .dot import dot_graph
        return dot_graph(self, show_only_variable=show_only_variable,
                         show_inputs=show_inputs,
                         show_variables=show_variables)

    def _get_dsk(self, func='run_step'):
        return {k: (getattr(self._processes[k], func), v)
                for k, v in self._dep_processes.items()}

    def _set_inputs_values(self, ds):
        """Set model inputs values from xarray.Dataset."""
        for name, var in ds.data_vars.items():
            proc_name, var_name = name.split('__')

            if self.is_input((proc_name, var_name)):
                self[proc_name][var_name].value = var.values

    def initalize(self):
        """Run `.initalize()` for each processes in the model."""
        for proc in self._processes.values():
            proc.initialize()

    def run_step(self, step):
        """Run `.run_step()` for each time dependent processes in the model.
        """
        for proc in self._time_processes.values():
            proc.run_step(step)

    def finalize_step(self):
        """Run `.finalize_step()` for each time dependent processes
        in the model.
        """
        for proc in self._time_processes.values():
            proc.finalize_step()

    def finalize(self):
        """Run `.finalize()` for each processes in the model."""
        for proc in self._processes.values():
            proc.finalize()

    def run(self, ds, time_dim='time'):
        """Run the model.

        Parameters
        ----------
        ds : xarray.Dataset object
            Dataset to use as model input.
        time_dim : str, optional
            Name of the dimension in the Dataset that is used to set
            the time steps (default: 'time'). The dimension must have absolute
            time coordinates.

        Returns
        -------
        out_ds : xarray.Dataset object
            Another Dataset with model inputs and outputs.

        """
        ds_no_time = ds.filter(lambda v: time_dim not in v.dims)
        self._set_inputs_values(ds_no_time)

        self.initalize()

        ds_time = ds.filter(lambda v: time_dim in v.dims)
        has_time_var = bool(ds_time.data_vars)

        time_steps = ds[time_dim].diff(time_dim).values

        for i, dt in enumerate(time_steps):
            if has_time_var:
                ds_step = ds_time.isel(**{time_dim: i})
                self._set_inputs_values(ds_step)

            self.run_step(dt)
            self.finalize_step()

        self.finalize()

        out_ds = ds.copy()

        return out_ds

    def _create_new_model(self, processes):
        """Create a new Model object with new process instances."""
        new_processes = {}

        for proc_name, proc in processes.items():
            proc_cls = type(proc)
            undef_var_names = [k for k, v in proc_cls.variables.items()
                               if isinstance(v, UndefinedVariable)]
            undef_vars = {k: proc.variables[k] for k in undef_var_names}
            new_processes[proc_name] = type(proc)(**undef_vars)

        return type(self)(new_processes)

    def update_processes(self, processes):
        """Add or replace processe(s) in this model.

        Parameters
        ----------
        processes : dict
            Dictionnary with process names as keys and
            `Process` objects as values.

        Returns
        -------
        updated : Model
            New Model instance with updated processes.

        """
        new_processes = self._processes.copy()
        new_processes.update(processes)
        return self._create_new_model(new_processes)

    def drop_processes(self, names):
        """Drop processe(s) from this model.

        Parameters
        ----------
        names : str or list of str
            Name(s) of the processes to drop.

        Returns
        -------
        dropped : Model
            New Model instance with dropped processes.

        """
        if isinstance(names, str):
            names = [names]

        new_processes = self._processes.copy()
        for n in names:
            del new_processes[n]
        return self._create_new_model(new_processes)

    def __repr__(self):
        n_inputs = sum([len(v) for v in self._input_vars.values()])

        hdr = ("<fastscape.models.Model (%d processes, %d inputs)>\n"
               % (len(self._processes), n_inputs))

        max_line_length = 70
        col_width = max([_calculate_col_width(var)
                         for var in self._input_vars.values()])

        blocks = []
        for proc_name in self._processes:
            proc_str = "%s" % proc_name

            inputs = self._input_vars.get(proc_name, {})
            lines = []
            for name, var in inputs.items():
                line = pretty_print("    %s " % name, col_width)
                line += maybe_truncate("(in) %s" % var.description,
                                       max_line_length - col_width)
                lines.append(line)

            if lines:
                proc_str += '\n' + '\n'.join(lines)
            blocks.append(proc_str)

        return hdr + '\n'.join(blocks)
<|endoftext|>


<commit_before>from collections import defaultdict
from copy import copy
import logging

logger = logging.getLogger(__name__)

LOG_STEP = 500

class FeatureEvaluator(object):
    """Simple feature evaluator"""

    def __init__(self, features):
        self.features = features

    def fit(self, X, y=None):
        self.alive_features = tuple(self.features)
        return self

    def fit_transform(self, X, y=None):
        self.fit(X, y)
        return self.transform(X)

    def transform(self, X, y=None):
        for d in X:
            yield tuple((f(d) for f in self.alive_features))

class TolerantFeatureEvaluator(object):
    """Feature Evaluator that tolerates broken features or samples when fitting.
     Tolerance to failures (only done during fit or fit_transform):
     a) Samples are always discarded when failing:
         If a given sample fails when evaluating a feature with it, no matter
         what, no matter when, the sample is discarded.
     b) Features can be discarded or not, depending on the configuration:
         - When evaluating the first N data-points, if there's an error with a
           feature, that feature is automatically excluded (STRICT mode).
         - If error occurs on later data-points and FEATURE_MAX_ERRORS_ALLOWED
           was exceeded for that feature, the feature is discarded. If not
           exceeded, an internal counter will be increated for that feature.
        Each time that a feature is discarded, the following things happen:
          - every sampled that was discarded because of this feature,
            is re-considered.
          - every previously evaluated sample is stripped off of the result
            of this feature.
    """
    FEATURE_STRICT_UNTIL = 100
    FEATURE_MAX_ERRORS_ALLOWED = 5

    class NoFeaturesLeftError(Exception):
        pass

    def __init__(self, features):
        self.features = features
        self.fitted = False

    def fit(self, X, y=None):
        self._fit_failure_stats = {
            'discarded_samples': [],
            'features': defaultdict(list)
        }
        self.alive_features = self.features[:]

        dataset = X
        last_sample_idx = -1  # Caution to not work in strict mode when retrying
        while dataset:
            self._samples_to_retry = []
            for i, d in enumerate(dataset, last_sample_idx + 1):
                for feature in self.alive_features[:]:
                    try:
                        feature(d)
                    except Exception, e:
                        self.process_failure([], e, feature, d, i)
                        break
            last_sample_idx = i
            dataset = self._samples_to_retry

        self.alive_features = tuple(self.alive_features)
        self.fitted = True
        return self

    def transform(self, X, y=None):
        for d in X:
            yield tuple((f(d) for f in self.alive_features))

    def fit_transform(self, X, y=None):
        # Very similar to fit alone, but buffers samples evaluation for two
        # reasons:
        #   - simply because this is also a transform, so we need to return that
        #   - to be able to patch them if at some given point a Feature that
        #     was working is killed.
        self._fit_failure_stats = {
            'discarded_samples': [],
            'features': defaultdict(list)
        }
        self.alive_features = self.features[:]
        result = []

        dataset = X
        last_sample_idx = -1  # Caution to not work in strict mode when retrying
        while dataset:
            self._samples_to_retry = []
            for i, d in enumerate(dataset, last_sample_idx + 1):
                r = []
                for feature in self.alive_features[:]:
                    try:
                        r.append(feature(d))
                    except Exception, e:
                        self.process_failure(result, e, feature, d, i)
                        break
                else:
                    result.append(r)
            last_sample_idx = i
            dataset = self._samples_to_retry

        self.alive_features = tuple(self.alive_features)
        self.fitted = True
        return (tuple(r) for r in result)

    def process_failure(self, partial_eval, error, feature, dpoint, d_index):
        logger.warning(u'Fail evaluating %s: %s %s' % (feature,
                                                       type(error), error))
        self._fit_failure_stats['discarded_samples'].append(
            dpoint.get('pk', 'PK-NOT-FOUND'))
        feature_errors = self._fit_failure_stats['features'][feature]
        feature_errors.append(dpoint)
        if d_index < self.FEATURE_STRICT_UNTIL:
            self.exclude_feature(feature, partial_eval)
        elif len(feature_errors) > self.FEATURE_MAX_ERRORS_ALLOWED:
            self.exclude_feature(feature, partial_eval)

    def exclude_feature(self, feature, partial_evaluation):
        idx = self.alive_features.index(feature)
        self.alive_features.remove(feature)
        if not self.alive_features:
            raise self.NoFeaturesLeftError()
        for evaluation in partial_evaluation:
            evaluation.pop(idx)
        discarded_samples = self._fit_failure_stats['features'][feature]
        self._samples_to_retry += discarded_samples
<commit_msg>Changed use of deprecated exception syntax<commit_after>from collections import defaultdict
from copy import copy
import logging

logger = logging.getLogger(__name__)

LOG_STEP = 500

class FeatureEvaluator(object):
    """Simple feature evaluator"""

    def __init__(self, features):
        self.features = features

    def fit(self, X, y=None):
        self.alive_features = tuple(self.features)
        return self

    def fit_transform(self, X, y=None):
        self.fit(X, y)
        return self.transform(X)

    def transform(self, X, y=None):
        for d in X:
            yield tuple((f(d) for f in self.alive_features))

class TolerantFeatureEvaluator(object):
    """Feature Evaluator that tolerates broken features or samples when fitting.
     Tolerance to failures (only done during fit or fit_transform):
     a) Samples are always discarded when failing:
         If a given sample fails when evaluating a feature with it, no matter
         what, no matter when, the sample is discarded.
     b) Features can be discarded or not, depending on the configuration:
         - When evaluating the first N data-points, if there's an error with a
           feature, that feature is automatically excluded (STRICT mode).
         - If error occurs on later data-points and FEATURE_MAX_ERRORS_ALLOWED
           was exceeded for that feature, the feature is discarded. If not
           exceeded, an internal counter will be increated for that feature.
        Each time that a feature is discarded, the following things happen:
          - every sampled that was discarded because of this feature,
            is re-considered.
          - every previously evaluated sample is stripped off of the result
            of this feature.
    """
    FEATURE_STRICT_UNTIL = 100
    FEATURE_MAX_ERRORS_ALLOWED = 5

    class NoFeaturesLeftError(Exception):
        pass

    def __init__(self, features):
        self.features = features
        self.fitted = False

    def fit(self, X, y=None):
        self._fit_failure_stats = {
            'discarded_samples': [],
            'features': defaultdict(list)
        }
        self.alive_features = self.features[:]

        dataset = X
        last_sample_idx = -1  # Caution to not work in strict mode when retrying
        while dataset:
            self._samples_to_retry = []
            for i, d in enumerate(dataset, last_sample_idx + 1):
                for feature in self.alive_features[:]:
                    try:
                        feature(d)
                    except Exception as e:
                        self.process_failure([], e, feature, d, i)
                        break
            last_sample_idx = i
            dataset = self._samples_to_retry

        self.alive_features = tuple(self.alive_features)
        self.fitted = True
        return self

    def transform(self, X, y=None):
        for d in X:
            yield tuple((f(d) for f in self.alive_features))

    def fit_transform(self, X, y=None):
        # Very similar to fit alone, but buffers samples evaluation for two
        # reasons:
        #   - simply because this is also a transform, so we need to return that
        #   - to be able to patch them if at some given point a Feature that
        #     was working is killed.
        self._fit_failure_stats = {
            'discarded_samples': [],
            'features': defaultdict(list)
        }
        self.alive_features = self.features[:]
        result = []

        dataset = X
        last_sample_idx = -1  # Caution to not work in strict mode when retrying
        while dataset:
            self._samples_to_retry = []
            for i, d in enumerate(dataset, last_sample_idx + 1):
                r = []
                for feature in self.alive_features[:]:
                    try:
                        r.append(feature(d))
                    except Exception, e:
                        self.process_failure(result, e, feature, d, i)
                        break
                else:
                    result.append(r)
            last_sample_idx = i
            dataset = self._samples_to_retry

        self.alive_features = tuple(self.alive_features)
        self.fitted = True
        return (tuple(r) for r in result)

    def process_failure(self, partial_eval, error, feature, dpoint, d_index):
        logger.warning(u'Fail evaluating %s: %s %s' % (feature,
                                                       type(error), error))
        self._fit_failure_stats['discarded_samples'].append(
            dpoint.get('pk', 'PK-NOT-FOUND'))
        feature_errors = self._fit_failure_stats['features'][feature]
        feature_errors.append(dpoint)
        if d_index < self.FEATURE_STRICT_UNTIL:
            self.exclude_feature(feature, partial_eval)
        elif len(feature_errors) > self.FEATURE_MAX_ERRORS_ALLOWED:
            self.exclude_feature(feature, partial_eval)

    def exclude_feature(self, feature, partial_evaluation):
        idx = self.alive_features.index(feature)
        self.alive_features.remove(feature)
        if not self.alive_features:
            raise self.NoFeaturesLeftError()
        for evaluation in partial_evaluation:
            evaluation.pop(idx)
        discarded_samples = self._fit_failure_stats['features'][feature]
        self._samples_to_retry += discarded_samples
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright © 2007  Red Hat, Inc. All rights reserved.
#
# This copyrighted material is made available to anyone wishing to use, modify,
# copy, or redistribute it subject to the terms and conditions of the GNU
# General Public License v.2.  This program is distributed in the hope that it
# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.  You should have
# received a copy of the GNU General Public License along with this program;
# if not, write to the Free Software Foundation, Inc., 51 Franklin Street,
# Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat trademarks that are
# incorporated in the source code or documentation are not subject to the GNU
# General Public License and may only be used or replicated with the express
# permission of Red Hat, Inc.
#
# Red Hat Author(s): Luke Macken <lmacken@redhat.com>
#                    Toshio Kuratomi <tkuratom@redhat.com>
#

'''
fedora.client is used to interact with Fedora Services.
'''

class FedoraServiceError(Exception):
    '''Base Exception for any problem talking with the Service.'''
    pass

class ServerError(FedoraServiceError):
    '''Unable to talk to the server properly.'''
    pass

class AuthError(FedoraServiceError):
    '''Error during authentication.  For instance, invalid password.'''
    pass

class AppError(FedoraServiceError):
    '''Error condition that the server is passing back to the client.'''
    def __init__(self, name, message):
        super(AppError, self).__init__(name, message)
        self.name = name
        self.message = message

class DictContainer(dict):
    '''dict whose members can be accessed via attribute.

    One thing to note: You can have an entry in your container that is visible
    instead of a standard dict method.  So, for instance, you can have this
    happen::
        >>> d = DictContainer({'keys': 'key'})
        >>> d.keys()
        Traceback (most recent call last):
          File "<stdin>", line 1, in <module>
        TypeError: 'str' object is not callable

    So, as a safety precaution, you should be sure to access things via the
    dict methods::
        >>> d = DictContainer({'keys': 'key'})
        >>> dict.keys(d)
        ['keys']

    The special methods like __getitem__(), __getattr__(), setattr(), etc
    that are invoked through alternate syntax rather than called directly as
    a method are immune to this so you can do this with no ill effects::
        >>> d.__setattr__ = 1000
        >>> d.__getattr__ = 10
        >>> print d.__setattr__
        1000
        >>> print d.__getattr__
        10
    '''
    def __getitem__(self, key):
        '''Return the value for the key in the dictionary.

        If the value is a dict, convert it to a DictContainer first.
        '''
        value = super(DictContainer, self).__getitem__(key)
        if type(value) == dict:
            value = DictContainer(value)
            self[key] = value
        return value

    def __getattribute__(self, key):
        '''Return the value from the dictionary or from the real attributes.

        If the key exists in the dict, then return that.
        Otherwise, if it exists in the values, return that.
            If the value is a dict convert it to a DictContainer first.
        Otherwise, raise AttributeError.
        '''
        if key in self:
            value = super(DictContainer, self).__getitem__(key)
            if type(value) == dict:
                value = DictContainer(value)
                self[key] = value
            return value
        return super(DictContainer, self).__getattribute__(key)

    def __setattr__(self, key, value):
        '''Set a key to a value.

        If the key is not an existing value, set it into the dict instead of
        as an attribute.
        '''
        if hasattr(self, key) and key not in self:
            super(DictContainer, self).__setattr__(key, value)
        else:
            self[key] = value

from fedora.client.proxyclient import ProxyClient
from fedora.client.baseclient import BaseClient

__all__ = ('FedoraServiceError', 'ServerError', 'AuthError', 'AppError',
        'ProxyClient', 'BaseClient', 'DictContainer')
<commit_msg>Add all of our clients to this level so from fedora.client import FooClient works.<commit_after># -*- coding: utf-8 -*-
#
# Copyright © 2007  Red Hat, Inc. All rights reserved.
#
# This copyrighted material is made available to anyone wishing to use, modify,
# copy, or redistribute it subject to the terms and conditions of the GNU
# General Public License v.2.  This program is distributed in the hope that it
# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.  You should have
# received a copy of the GNU General Public License along with this program;
# if not, write to the Free Software Foundation, Inc., 51 Franklin Street,
# Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat trademarks that are
# incorporated in the source code or documentation are not subject to the GNU
# General Public License and may only be used or replicated with the express
# permission of Red Hat, Inc.
#
# Red Hat Author(s): Luke Macken <lmacken@redhat.com>
#                    Toshio Kuratomi <tkuratom@redhat.com>
#

'''
fedora.client is used to interact with Fedora Services.
'''

class FedoraServiceError(Exception):
    '''Base Exception for any problem talking with the Service.'''
    pass

class ServerError(FedoraServiceError):
    '''Unable to talk to the server properly.'''
    pass

class AuthError(FedoraServiceError):
    '''Error during authentication.  For instance, invalid password.'''
    pass

class AppError(FedoraServiceError):
    '''Error condition that the server is passing back to the client.'''
    def __init__(self, name, message):
        super(AppError, self).__init__(name, message)
        self.name = name
        self.message = message

class DictContainer(dict):
    '''dict whose members can be accessed via attribute.

    One thing to note: You can have an entry in your container that is visible
    instead of a standard dict method.  So, for instance, you can have this
    happen::
        >>> d = DictContainer({'keys': 'key'})
        >>> d.keys()
        Traceback (most recent call last):
          File "<stdin>", line 1, in <module>
        TypeError: 'str' object is not callable

    So, as a safety precaution, you should be sure to access things via the
    dict methods::
        >>> d = DictContainer({'keys': 'key'})
        >>> dict.keys(d)
        ['keys']

    The special methods like __getitem__(), __getattr__(), setattr(), etc
    that are invoked through alternate syntax rather than called directly as
    a method are immune to this so you can do this with no ill effects::
        >>> d.__setattr__ = 1000
        >>> d.__getattr__ = 10
        >>> print d.__setattr__
        1000
        >>> print d.__getattr__
        10
    '''
    def __getitem__(self, key):
        '''Return the value for the key in the dictionary.

        If the value is a dict, convert it to a DictContainer first.
        '''
        value = super(DictContainer, self).__getitem__(key)
        if type(value) == dict:
            value = DictContainer(value)
            self[key] = value
        return value

    def __getattribute__(self, key):
        '''Return the value from the dictionary or from the real attributes.

        If the key exists in the dict, then return that.
        Otherwise, if it exists in the values, return that.
            If the value is a dict convert it to a DictContainer first.
        Otherwise, raise AttributeError.
        '''
        if key in self:
            value = super(DictContainer, self).__getitem__(key)
            if type(value) == dict:
                value = DictContainer(value)
                self[key] = value
            return value
        return super(DictContainer, self).__getattribute__(key)

    def __setattr__(self, key, value):
        '''Set a key to a value.

        If the key is not an existing value, set it into the dict instead of
        as an attribute.
        '''
        if hasattr(self, key) and key not in self:
            super(DictContainer, self).__setattr__(key, value)
        else:
            self[key] = value

# We want people to be able to import fedora.client.*Client directly
# pylint: disable-msg=W0611
from fedora.client.proxyclient import ProxyClient
from fedora.client.baseclient import BaseClient
from fedora.client.fas2 import AccountSystem, FASError, CLAError
from fedora.client.pkgdb import PkgdbClient
from fedora.client.bodhi import BodhiClient, BodhiClientException
# pylint: enable-msg=W0611

__all__ = ('FedoraServiceError', 'ServerError', 'AuthError', 'AppError',
        'DictContainer',
        'FASError', 'CLAError', 'BodhiClientException',
        'ProxyClient', 'BaseClient', 'AccountSystem', 'PkgdbClient',
        'BodhiClient')
<|endoftext|>


<commit_before>#License#
#bitHopper by Colin Rice is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License.
#Based on a work at github.com.

from twisted.internet.task import LoopingCall

class Speed():
    def __init__(self,bitHopper):
        self.bitHopper= bitHopper
        self.shares = 0
        call = LoopingCall(self.update_rate)
        call.start(60)
        self.rate = 0

    def add_shares(self, share):
        self.shares += 1

    def update_rate(self):
        self.rate = int((float(self.shares) * (2**32)) / (60 * 10 * 1000000))
        self.shares = 0

    def get_rate(self):
        return int(self.rate)
<commit_msg>Fixed bug in hash display<commit_after>#License#
#bitHopper by Colin Rice is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License.
#Based on a work at github.com.

from twisted.internet.task import LoopingCall

class Speed():
    def __init__(self,bitHopper):
        self.bitHopper= bitHopper
        self.shares = 0
        call = LoopingCall(self.update_rate)
        call.start(60)
        self.rate = 0

    def add_shares(self, share):
        self.shares += 1

    def update_rate(self):
        self.rate = int((float(self.shares) * (2**32)) / (60 * 1000000))
        self.shares = 0

    def get_rate(self):
        return int(self.rate)
<|endoftext|>