<commit_before>import re
import csv
import copy
from django.http import (HttpResponse, HttpResponseGone, HttpResponseRedirect,
                         HttpResponseNotFound)
from django.template import RequestContext
from django.views import generic
from django.shortcuts import (render_to_response, get_object_or_404, render,
                              redirect)
from django.db.models import Q, Count, Max, Min
from django.core.serializers import serialize
from django.core.urlresolvers import reverse
from django.contrib.syndication.views import Feed
from django.templatetags.static import static
from django.utils.feedgenerator import Atom1Feed
from django.conf import settings
from dateutil import rrule
from datetime import datetime
from pynaco.naco import normalizeSimplified
from name.decorators import jsonp
from name.models import Name, Identifier, Location
from . import serializers
from rest_framework.renderers import JSONRenderer

VOCAB_DOMAIN = settings.VOCAB_DOMAIN

class JSONResponse(HttpResponse):
    def __init__(self, data, **kwargs):
        content = JSONRenderer().render(data, renderer_context={'indent': 4})
        kwargs['content_type'] = 'application/json'
        super(JSONResponse, self).__init__(content, **kwargs)

def normalize_query(query_string):
    """Splits the query string in individual keywords, removing
    unnecessary spaces and grouping quoted words together.

    Example:
    >>> normalize_query('some random words "with quotes" and spaces')
        ['some', 'random', 'words', 'with quotes', 'and', 'spaces']
    """
    findterms = re.compile(r'"([^"]+)"|(\S+)').findall
    normspace = re.compile(r'\s{2,}').sub

    terms = (findterms(query_string))

    return [normspace(' ', (t[0] or t[1]).strip()) for t in terms]

def compose_query(query_string):
    """Composes a Q object from the query_string.

    This will first normalize the query_sting and split the string up
    into words. Then it will iterate over the list of words creating Q
    object for each one. Last, it reduces the generated Q objects into
    a single Q object by combining them using the & operator.

    The reduce function essential performs the following operation for
    a query_string that contains 4 words,

        (((Q1 & Q2) & Q3) & Q4)

    where Q1-4 are instances of the Q object.
    """
    # Normalize the query_string which will also split the single string
    # into a list of terms.
    terms = normalize_query(query_string)

    # Create a generator that produces an instance of the Q object
    # for each term.
    qs = (Q(name__icontains=term) for term in terms)

    # Use bitwise AND to compose the objects into a single instance of Q.
    return reduce(lambda x, y: x & y, qs)

def label(request, name_value):
    """Find a name by label.

    If the label matches a single name, users are redirected
    to the name_entry_detail page. Otherwise, if the label
    does not exist, or the specified label matches multiple Name
    objects, a status code 404 is returned.
    """
    if not name_value:
        return HttpResponseNotFound()

    normalized_name = normalizeSimplified(name_value)
    try:
        name = Name.objects.get(normalized_name=normalized_name)

        return HttpResponseRedirect(
            reverse('name_entry_detail', args=[name.name_id]))

    except Name.DoesNotExist:
        return HttpResponseNotFound(
            u'No matching term found - authoritative, or variant - for \"{0}\"'
            .format(name_value))

    except Name.MultipleObjectsReturned:
        return HttpResponseNotFound(
            u'There are multiple Name objects with the same name: \'{0}\'.'
            .format(normalized_name))

class AtomSiteNewsFeed(Feed):
    """Sets up atom feed"""

    feed_type = Atom1Feed
    link = "/name/feed/"
    title = "Name App"
    subtitle = "new records"

    def items(self):
        # last 5 added items
        return Name.objects.order_by('-date_created')[:20]

    def item_title(self, item):
        return item.name

    def item_location(self, item):
        """
        Returns an extra keyword arguments dictionary that is used
        with the `add_item` call of the feed generator. Add the
        'content' field of the 'Entry' item, to be used by the custom
        feed generator.
        """
        location_set = []
        for l in Location.objects.filter(belong_to_name=item):
            location_set.append(
                'georss:point', "%s %s" % (l.latitude, l.longitude)
            )
        return location_set

    def item_description(self, item):
        return "Name Type: %s" % NAME_TYPE_CHOICES[item.name_type][1]

    def item_link(self, item):
        return VOCAB_DOMAIN + 'name/' + item.name_id + '/'

def entry_detail(request, name_id):
    """Name Entry Detail View."""
    queryset = (
        Name.objects.select_related().
        prefetch_related('identifier_set__type')
    )

    name_entry = get_object_or_404(queryset, name_id=name_id)

    if name_entry.merged_with:
        return redirect(name_entry.merged_with)

    elif name_entry.is_suppressed():
        return HttpResponseNotFound()

    elif name_entry.is_deleted():
        return HttpResponseGone('The requested record has been deleted!')

    return render(request, 'name/name_detail.html', {'name': name_entry})

def export(request):
    """Exports Names as TSV file."""
    # Create a CSV response.
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="data.tsv"'

    writer = csv.writer(response, delimiter='\t', quoting=csv.QUOTE_MINIMAL)

    # Iterate through the visible Names that are not merged with any other
    # Name records and write a row for each record.
    for n in Name.objects.visible():
        writer.writerow([
            n.get_name_type_label().lower(),
            n.name.encode('utf-8'),
            request.build_absolute_uri(n.get_absolute_url())
        ])

    return response

def opensearch(request):
    """Renders Opensearch XML file.

    Opensearch URL templates are composed in the view
    because it is easier than creating them in the template.
    """
    # Query templates for the URLs.
    search_query = '{0}?q_type=Personal&q={{searchTerms}}'
    search_json_query = '{0}?q={{searchTerms}}'

    # Build the absolute URLs.
    search_url = request.build_absolute_uri(reverse('name_search'))
    search_json_url = request.build_absolute_uri(reverse('name_names'))

    # Compose the full URLs  that will be sent to the template.
    search = search_query.format(search_url)
    search_json = search_json_query.format(search_json_url)
    image = request.build_absolute_uri(static('name/img/favicon.png'))

    urls = dict(search=search, search_json=search_json, image=image)

    return render(request, 'name/opensearch.xml',
                  urls, content_type='text/xml')

def about(request):
    """Renders the name about page to the user."""
    return render(request, 'name/about.html')

def prepare_graph_date_range():
    """
    Several functions use the same code to prepare the dates and values
    for the graphing of event data, so we can make a function for it.
    DRY 4 LYPHE

    returns list of lists
    """

    name_objects = Name.objects.all()
    # grab the dates for the bounds of the graphs
    if name_objects.count() > 0:
        dates = Name.objects.all().aggregate(
            Min("date_created"), Max('date_created'))
        system_start_date = dates['date_created__min']
        system_end_date = dates['date_created__max']

        # setup list for month call data
        daily_edit_counts = []
        # we need to first fill in the first month, because it won't
        # grab that when the for loop iterates to a new month.
        month = system_start_date.strftime('%Y-%m')
        daily_edit_counts.append([datetime.strptime(month, '%Y-%m'), 0])
        # start with a zero count and reset this as we cross a month
        for dt in rrule.rrule(
            rrule.DAILY,
            dtstart=system_start_date,
            until=system_end_date,
        ):
            # if we change months
            if month != dt.strftime('%Y-%m'):
                month = dt.strftime('%Y-%m')
                # make the year and drop in if i doenst exist
                daily_edit_counts.append(
                    [datetime.strptime(month, '%Y-%m'), 0])
        return daily_edit_counts
    else:
        raise Exception('no name objects to construct graph')

def calc_total_by_month(**kwargs):
    """
    totals for the bag / file count by month.
    Returns an ordered dict
    """

    if kwargs['created']:
        # get the daily Name totals
        daily_counts = Name.objects.extra(
            select={'day': 'date_created'}
        ).values('day').annotate(num=Count('date_created'))
    elif kwargs['edited']:
        # get the daily Name totals
        daily_counts = Name.objects.extra(
            select={'day': 'last_modified'}
        ).values('day').annotate(num=Count('last_modified'))
    # make current total events count
    current_total = 0
    month_skeleton = copy.deepcopy(prepare_graph_date_range())
    # fill in totals
    for u in month_skeleton:
        # replace existing value of zero with sum of nums in certain
        # month
        current_month_counts = [e for e in daily_counts if datetime.strftime(
            e['day'], '%Y-%m') == datetime.strftime(u[0], '%Y-%m')]

        for n in current_month_counts:
            current_total += n['num']
        u[1] = current_total
        if not kwargs['running']:
            current_total = 0
    return month_skeleton

def stats(request):
    """
    Renders the name stats page to the user
    """

    name_type_counts = {
        'personal': Name.objects.filter(name_type=0).count(),
        'organization': Name.objects.filter(name_type=1).count(),
        'event': Name.objects.filter(name_type=2).count(),
        'software': Name.objects.filter(name_type=3).count(),
    }
    name_line_graph_data = {
        'creation_running': calc_total_by_month(created=True,
                                                edited=False, running=True),
        'creation_monthly': calc_total_by_month(created=True,
                                                edited=False, running=False),
        'edited_running': calc_total_by_month(created=False,
                                              edited=True, running=True),
        'edited_monthly': calc_total_by_month(created=False,
                                              edited=True, running=False),
    }
    return render_to_response(
        'name/stats.html',
        {
            'name_line_graph_data': name_line_graph_data,
            'name_type_counts': name_type_counts,
            'total_names': Name.objects.count(),
            'total_links': Identifier.objects.count(),
        },
        context_instance=RequestContext(request)
    )

def resolve_type(q_type):
    """Resolve the Name Types passed to the request, and returns
    a list of Name Type IDs.

    The input is expected to be the raw q_type query string value passed
    in with the request.  The input is expected to be a comma delimited
    list such as, `Personal,Event,Software`.  This function will lookup
    up the ID of the Name Type based on the string and return a list
    of Name Type IDs.
    """
    if not q_type:
        return []

    q_type = q_type.split(',')

    # Iterate through the Name Types and create a list of Name IDs
    # using the corresponding strings values in q_type
    return [k for k, v in NAME_TYPE_CHOICES if v in q_type]

# TODO: Look at reducing the number of queries.
def filter_names(request):
    """Return the set of Name objects filtered on the query
    parameters passed with the request.
    """
    q = request.GET.get('q', None)
    name_types = resolve_type(request.GET.get('q_type', None))

    # Retrieve the visible names from the database.
    names = Name.objects.visible()

    # Filter by name type if it is passed with the request.
    if name_types:
        names = names.filter(name_type__in=name_types)

    # Do further filtering if the q parameter is present.
    if q:
        # All names that fit query OR all variants that contain query
        query_filter = compose_query(q) | Q(variant__variant__icontains=q)
        names = names.filter(query_filter).distinct()

    return names

@jsonp
def get_names(request):
    """Gets the JSON encoded version of the Names matching
    the query parameters.

    This also provides the endpoint used for autocompletion
    during search.
    """
    names = filter_names(request)

    # If the request is AJAX, then it is most likely for autocompletion,
    # so we will only return first 10 names.
    if request.is_ajax():
        names = names[:10]

    data = serializers.NameSearchSerializer(
        names, many=True, context={'request': request})

    response = JSONResponse(data.data)
    response['Access-Control-Allow-Origin'] = '*'
    response['Access-Control-Allow-Headers'] = 'X-Requested-With'

    return response

def landing(request):
    """View for the landing page of the Name App.

    Sends Name count statistics in the context to display on the page.
    """
    counts = {'counts': Name.objects.active_type_counts()}
    return render(request, 'name/landing.html', counts)

def map_json(request):
    """Presents the Locations and related Names serialized into JSON."""
    if request.is_ajax():
        data = serialize(
            'json',
            Location.objects.all().filter(status=0),
            use_natural_foreign_keys=True,
            indent=4)

        return HttpResponse(data, content_type='application/json')
    return HttpResponseNotFound()

def map(request):
    """Renders the map template."""
    return render(request, 'name/map.html')

class SearchView(generic.ListView):
    model = Name
    template_name = 'name/search.html'
    paginate_by = 15

    VALID_SORTS = {
        'name_a': 'name',
        'name_d': '-name',
        'begin_a': 'begin',
        'begin_d': '-begin',
        'end_a': 'end',
        'end_d': '-end',
    }

    DEFAULT_SORT = VALID_SORTS['name_a']

    def get_sort_method(self):
        order = self.request.GET.get('order', '')
        return self.VALID_SORTS.get(order, self.DEFAULT_SORT)

    def get_queryset(self):
        query = self.request.GET
        if query.get('q') or query.get('q_type'):
            return filter_names(self.request).order_by(self.get_sort_method())
        return Name.objects.none()

def name_json(request, name_id):
    """Returns result of name query in json format."""
    name = get_object_or_404(Name, name_id=name_id)
    data = serializers.NameSerializer(name, context={'request': request})

    return JSONResponse(data.data)

def mads_serialize(request, name_id):
    queryset = (
        Name.objects.select_related().
        prefetch_related('identifier_set__type', 'note_set', 'variant_set')
    )

    context = dict(name=get_object_or_404(queryset, name_id=name_id))

    return render(request, 'name/name.mads.xml',
                  context, content_type='text/xml')
<commit_msg>Use the name_type label helper method.<commit_after>import re
import csv
import copy
from django.http import (HttpResponse, HttpResponseGone, HttpResponseRedirect,
                         HttpResponseNotFound)
from django.template import RequestContext
from django.views import generic
from django.shortcuts import (render_to_response, get_object_or_404, render,
                              redirect)
from django.db.models import Q, Count, Max, Min
from django.core.serializers import serialize
from django.core.urlresolvers import reverse
from django.contrib.syndication.views import Feed
from django.templatetags.static import static
from django.utils.feedgenerator import Atom1Feed
from django.conf import settings
from dateutil import rrule
from datetime import datetime
from pynaco.naco import normalizeSimplified
from name.decorators import jsonp
from name.models import Name, Identifier, Location
from . import serializers
from rest_framework.renderers import JSONRenderer

VOCAB_DOMAIN = settings.VOCAB_DOMAIN

class JSONResponse(HttpResponse):
    def __init__(self, data, **kwargs):
        content = JSONRenderer().render(data, renderer_context={'indent': 4})
        kwargs['content_type'] = 'application/json'
        super(JSONResponse, self).__init__(content, **kwargs)

def normalize_query(query_string):
    """Splits the query string in individual keywords, removing
    unnecessary spaces and grouping quoted words together.

    Example:
    >>> normalize_query('some random words "with quotes" and spaces')
        ['some', 'random', 'words', 'with quotes', 'and', 'spaces']
    """
    findterms = re.compile(r'"([^"]+)"|(\S+)').findall
    normspace = re.compile(r'\s{2,}').sub

    terms = (findterms(query_string))

    return [normspace(' ', (t[0] or t[1]).strip()) for t in terms]

def compose_query(query_string):
    """Composes a Q object from the query_string.

    This will first normalize the query_sting and split the string up
    into words. Then it will iterate over the list of words creating Q
    object for each one. Last, it reduces the generated Q objects into
    a single Q object by combining them using the & operator.

    The reduce function essential performs the following operation for
    a query_string that contains 4 words,

        (((Q1 & Q2) & Q3) & Q4)

    where Q1-4 are instances of the Q object.
    """
    # Normalize the query_string which will also split the single string
    # into a list of terms.
    terms = normalize_query(query_string)

    # Create a generator that produces an instance of the Q object
    # for each term.
    qs = (Q(name__icontains=term) for term in terms)

    # Use bitwise AND to compose the objects into a single instance of Q.
    return reduce(lambda x, y: x & y, qs)

def label(request, name_value):
    """Find a name by label.

    If the label matches a single name, users are redirected
    to the name_entry_detail page. Otherwise, if the label
    does not exist, or the specified label matches multiple Name
    objects, a status code 404 is returned.
    """
    if not name_value:
        return HttpResponseNotFound()

    normalized_name = normalizeSimplified(name_value)
    try:
        name = Name.objects.get(normalized_name=normalized_name)

        return HttpResponseRedirect(
            reverse('name_entry_detail', args=[name.name_id]))

    except Name.DoesNotExist:
        return HttpResponseNotFound(
            u'No matching term found - authoritative, or variant - for \"{0}\"'
            .format(name_value))

    except Name.MultipleObjectsReturned:
        return HttpResponseNotFound(
            u'There are multiple Name objects with the same name: \'{0}\'.'
            .format(normalized_name))

class AtomSiteNewsFeed(Feed):
    """Sets up atom feed"""

    feed_type = Atom1Feed
    link = "/name/feed/"
    title = "Name App"
    subtitle = "new records"

    def items(self):
        # last 5 added items
        return Name.objects.order_by('-date_created')[:20]

    def item_title(self, item):
        return item.name

    def item_location(self, item):
        """
        Returns an extra keyword arguments dictionary that is used
        with the `add_item` call of the feed generator. Add the
        'content' field of the 'Entry' item, to be used by the custom
        feed generator.
        """
        location_set = []
        for l in Location.objects.filter(belong_to_name=item):
            location_set.append(
                'georss:point', "%s %s" % (l.latitude, l.longitude)
            )
        return location_set

    def item_description(self, item):
        return "Name Type: %s" % item.get_name_type_label()

    def item_link(self, item):
        return VOCAB_DOMAIN + 'name/' + item.name_id + '/'

def entry_detail(request, name_id):
    """Name Entry Detail View."""
    queryset = (
        Name.objects.select_related().
        prefetch_related('identifier_set__type')
    )

    name_entry = get_object_or_404(queryset, name_id=name_id)

    if name_entry.merged_with:
        return redirect(name_entry.merged_with)

    elif name_entry.is_suppressed():
        return HttpResponseNotFound()

    elif name_entry.is_deleted():
        return HttpResponseGone('The requested record has been deleted!')

    return render(request, 'name/name_detail.html', {'name': name_entry})

def export(request):
    """Exports Names as TSV file."""
    # Create a CSV response.
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="data.tsv"'

    writer = csv.writer(response, delimiter='\t', quoting=csv.QUOTE_MINIMAL)

    # Iterate through the visible Names that are not merged with any other
    # Name records and write a row for each record.
    for n in Name.objects.visible():
        writer.writerow([
            n.get_name_type_label().lower(),
            n.name.encode('utf-8'),
            request.build_absolute_uri(n.get_absolute_url())
        ])

    return response

def opensearch(request):
    """Renders Opensearch XML file.

    Opensearch URL templates are composed in the view
    because it is easier than creating them in the template.
    """
    # Query templates for the URLs.
    search_query = '{0}?q_type=Personal&q={{searchTerms}}'
    search_json_query = '{0}?q={{searchTerms}}'

    # Build the absolute URLs.
    search_url = request.build_absolute_uri(reverse('name_search'))
    search_json_url = request.build_absolute_uri(reverse('name_names'))

    # Compose the full URLs  that will be sent to the template.
    search = search_query.format(search_url)
    search_json = search_json_query.format(search_json_url)
    image = request.build_absolute_uri(static('name/img/favicon.png'))

    urls = dict(search=search, search_json=search_json, image=image)

    return render(request, 'name/opensearch.xml',
                  urls, content_type='text/xml')

def about(request):
    """Renders the name about page to the user."""
    return render(request, 'name/about.html')

def prepare_graph_date_range():
    """
    Several functions use the same code to prepare the dates and values
    for the graphing of event data, so we can make a function for it.
    DRY 4 LYPHE

    returns list of lists
    """

    name_objects = Name.objects.all()
    # grab the dates for the bounds of the graphs
    if name_objects.count() > 0:
        dates = Name.objects.all().aggregate(
            Min("date_created"), Max('date_created'))
        system_start_date = dates['date_created__min']
        system_end_date = dates['date_created__max']

        # setup list for month call data
        daily_edit_counts = []
        # we need to first fill in the first month, because it won't
        # grab that when the for loop iterates to a new month.
        month = system_start_date.strftime('%Y-%m')
        daily_edit_counts.append([datetime.strptime(month, '%Y-%m'), 0])
        # start with a zero count and reset this as we cross a month
        for dt in rrule.rrule(
            rrule.DAILY,
            dtstart=system_start_date,
            until=system_end_date,
        ):
            # if we change months
            if month != dt.strftime('%Y-%m'):
                month = dt.strftime('%Y-%m')
                # make the year and drop in if i doenst exist
                daily_edit_counts.append(
                    [datetime.strptime(month, '%Y-%m'), 0])
        return daily_edit_counts
    else:
        raise Exception('no name objects to construct graph')

def calc_total_by_month(**kwargs):
    """
    totals for the bag / file count by month.
    Returns an ordered dict
    """

    if kwargs['created']:
        # get the daily Name totals
        daily_counts = Name.objects.extra(
            select={'day': 'date_created'}
        ).values('day').annotate(num=Count('date_created'))
    elif kwargs['edited']:
        # get the daily Name totals
        daily_counts = Name.objects.extra(
            select={'day': 'last_modified'}
        ).values('day').annotate(num=Count('last_modified'))
    # make current total events count
    current_total = 0
    month_skeleton = copy.deepcopy(prepare_graph_date_range())
    # fill in totals
    for u in month_skeleton:
        # replace existing value of zero with sum of nums in certain
        # month
        current_month_counts = [e for e in daily_counts if datetime.strftime(
            e['day'], '%Y-%m') == datetime.strftime(u[0], '%Y-%m')]

        for n in current_month_counts:
            current_total += n['num']
        u[1] = current_total
        if not kwargs['running']:
            current_total = 0
    return month_skeleton

def stats(request):
    """
    Renders the name stats page to the user
    """

    name_type_counts = {
        'personal': Name.objects.filter(name_type=0).count(),
        'organization': Name.objects.filter(name_type=1).count(),
        'event': Name.objects.filter(name_type=2).count(),
        'software': Name.objects.filter(name_type=3).count(),
    }
    name_line_graph_data = {
        'creation_running': calc_total_by_month(created=True,
                                                edited=False, running=True),
        'creation_monthly': calc_total_by_month(created=True,
                                                edited=False, running=False),
        'edited_running': calc_total_by_month(created=False,
                                              edited=True, running=True),
        'edited_monthly': calc_total_by_month(created=False,
                                              edited=True, running=False),
    }
    return render_to_response(
        'name/stats.html',
        {
            'name_line_graph_data': name_line_graph_data,
            'name_type_counts': name_type_counts,
            'total_names': Name.objects.count(),
            'total_links': Identifier.objects.count(),
        },
        context_instance=RequestContext(request)
    )

def resolve_type(q_type):
    """Resolve the Name Types passed to the request, and returns
    a list of Name Type IDs.

    The input is expected to be the raw q_type query string value passed
    in with the request.  The input is expected to be a comma delimited
    list such as, `Personal,Event,Software`.  This function will lookup
    up the ID of the Name Type based on the string and return a list
    of Name Type IDs.
    """
    if not q_type:
        return []

    q_type = q_type.split(',')

    # Iterate through the Name Types and create a list of Name IDs
    # using the corresponding strings values in q_type
    return [k for k, v in NAME_TYPE_CHOICES if v in q_type]

# TODO: Look at reducing the number of queries.
def filter_names(request):
    """Return the set of Name objects filtered on the query
    parameters passed with the request.
    """
    q = request.GET.get('q', None)
    name_types = resolve_type(request.GET.get('q_type', None))

    # Retrieve the visible names from the database.
    names = Name.objects.visible()

    # Filter by name type if it is passed with the request.
    if name_types:
        names = names.filter(name_type__in=name_types)

    # Do further filtering if the q parameter is present.
    if q:
        # All names that fit query OR all variants that contain query
        query_filter = compose_query(q) | Q(variant__variant__icontains=q)
        names = names.filter(query_filter).distinct()

    return names

@jsonp
def get_names(request):
    """Gets the JSON encoded version of the Names matching
    the query parameters.

    This also provides the endpoint used for autocompletion
    during search.
    """
    names = filter_names(request)

    # If the request is AJAX, then it is most likely for autocompletion,
    # so we will only return first 10 names.
    if request.is_ajax():
        names = names[:10]

    data = serializers.NameSearchSerializer(
        names, many=True, context={'request': request})

    response = JSONResponse(data.data)
    response['Access-Control-Allow-Origin'] = '*'
    response['Access-Control-Allow-Headers'] = 'X-Requested-With'

    return response

def landing(request):
    """View for the landing page of the Name App.

    Sends Name count statistics in the context to display on the page.
    """
    counts = {'counts': Name.objects.active_type_counts()}
    return render(request, 'name/landing.html', counts)

def map_json(request):
    """Presents the Locations and related Names serialized into JSON."""
    if request.is_ajax():
        data = serialize(
            'json',
            Location.objects.all().filter(status=0),
            use_natural_foreign_keys=True,
            indent=4)

        return HttpResponse(data, content_type='application/json')
    return HttpResponseNotFound()

def map(request):
    """Renders the map template."""
    return render(request, 'name/map.html')

class SearchView(generic.ListView):
    model = Name
    template_name = 'name/search.html'
    paginate_by = 15

    VALID_SORTS = {
        'name_a': 'name',
        'name_d': '-name',
        'begin_a': 'begin',
        'begin_d': '-begin',
        'end_a': 'end',
        'end_d': '-end',
    }

    DEFAULT_SORT = VALID_SORTS['name_a']

    def get_sort_method(self):
        order = self.request.GET.get('order', '')
        return self.VALID_SORTS.get(order, self.DEFAULT_SORT)

    def get_queryset(self):
        query = self.request.GET
        if query.get('q') or query.get('q_type'):
            return filter_names(self.request).order_by(self.get_sort_method())
        return Name.objects.none()

def name_json(request, name_id):
    """Returns result of name query in json format."""
    name = get_object_or_404(Name, name_id=name_id)
    data = serializers.NameSerializer(name, context={'request': request})

    return JSONResponse(data.data)

def mads_serialize(request, name_id):
    queryset = (
        Name.objects.select_related().
        prefetch_related('identifier_set__type', 'note_set', 'variant_set')
    )

    context = dict(name=get_object_or_404(queryset, name_id=name_id))

    return render(request, 'name/name.mads.xml',
                  context, content_type='text/xml')
<|endoftext|>


<commit_before>#!/usr/bin/env python3
'''
# Replace the given text in file(s) or stdin.
'''

# pylint: disable=C0111     # docstrings are always outdated and wrong
# pylint: disable=W0511     # todo is encouraged
# pylint: disable=C0301     # line too long
# pylint: disable=R0902     # too many instance attributes
# pylint: disable=C0302     # too many lines in module
# pylint: disable=C0103     # single letter var names, func name too descriptive
# pylint: disable=R0911     # too many return statements
# pylint: disable=R0912     # too many branches
# pylint: disable=R0915     # too many statements
# pylint: disable=R0913     # too many arguments
# pylint: disable=R1702     # too many nested blocks
# pylint: disable=R0914     # too many local variables
# pylint: disable=R0903     # too few public methods
# pylint: disable=E1101     # no member for base
# pylint: disable=W0201     # attribute defined outside __init__

import os
import shutil
import stat
import sys
import tempfile
from pathlib import Path

#from icecream import ic  # too many deps
import click
from asserttool import maxone
from colorama import Fore
from colorama import Style

#note adding deps requires changes to sendgentoo

def eprint(*args, **kwargs):
    if 'file' in kwargs.keys():
        kwargs.pop('file')
    print(*args, file=sys.stderr, **kwargs)

try:
    from icecream import ic  # https://github.com/gruns/icecream
except ImportError:
    ic = eprint

def is_regular_file(path):
    mode = os.stat(path, follow_symlinks=False)[stat.ST_MODE]
    if stat.S_ISREG(mode):
        return True
    return False

def all_files_iter(p):
    if isinstance(p, str):
        p = Path(p)
    elif isinstance(p, bytes):
        p = Path(p.decode())
    assert isinstance(p, Path)
    yield p.absolute()
    for sub in p.iterdir():
        # eprint("sub:", sub)  # todo: read by terminal, so bell etc happens.... eprint bug?
        if sub.is_symlink():  # must be before is_dir()
            yield sub.absolute()
        elif sub.is_dir():
            yield from all_files_iter(sub)
        else:
            yield sub.absolute()

def replace_text_line(*,
                      file_to_modify: Path,
                      match: str,
                      replacement: str,
                      verbose: bool,
                      debug: bool,
                      ):

    assert isinstance(file_to_modify, Path)
    if verbose:
        eprint(file_to_modify)

    #file_to_modify_basename = os.path.basename(file_to_modify)
    file_to_modify_dir = os.path.dirname(file_to_modify)
    temp_file = tempfile.NamedTemporaryFile(mode='w',
                                            prefix='tmp-',
                                            dir=file_to_modify_dir,
                                            delete=False)

    # this cant handle binary files... or files with mixed newlines
    modified = False
    with open(file_to_modify, 'rU') as file_to_modify_fh:
        try:
            for line in file_to_modify_fh:
                if match in line:
                    modified = True
                new_line = line.replace(match, replacement)
                temp_file.write(new_line)
                #if file_to_modify_fh.newlines == '\n':      # LF (Unix)
                #    temp_file.write("%s\n" % new_line)
                #    continue
                #elif file_to_modify_fh.newlines == '\r\n':  # CR+LF (DOS/Win)
                #    temp_file.write("%s\r\n" % new_line)
                #    continue
                #elif file_to_modify_fh.newlines == '\r':    # CR (Mac OS <= v9)
                #    temp_file.write("%s\r" % new_line)
        except UnicodeDecodeError as e:
            print("UnicodeDecodeError:", file_to_modify, file=sys.stderr)
            raise e

        temp_file_name = temp_file.name
        temp_file.close()
        if modified:
            shutil.copystat(file_to_modify, temp_file_name)
            shutil.move(temp_file_name, file_to_modify)
        else:
            os.unlink(temp_file_name)

def replace_text_bytes(*,
                       file_to_modify: Path,
                       match: bytes,
                       replacement: bytes,
                       verbose: bool,
                       debug: bool,
                       ):

    assert isinstance(file_to_modify, Path)
    if verbose:
        ic(file_to_modify)

    assert isinstance(match, bytes)
    assert isinstance(replacement, bytes)

    window_size = len(match) # need to expand a matching block by an arb amount, replacement can be any size

    #file_to_modify_basename = os.path.basename(file_to_modify)
    file_to_modify_dir = os.path.dirname(file_to_modify)
    temp_file = tempfile.NamedTemporaryFile(mode='wb',
                                            prefix='tmp-',
                                            dir=file_to_modify_dir,
                                            delete=False)

    # this cant handle binary files... or files with mixed newlines
    modified = False
    location_read = 0
    location_write = 0
    window = []
    with open(file_to_modify, 'rb') as fh:
        while True:
            # window starts off empty
            eprint(len(window), location_read)
            fh.seek(location_read)
            next_byte = fh.read(1)
            ic(next_byte)
            if next_byte is None:
                break

            window.append(next_byte)
            location_read += 1
            # first byte in window, or EOF

            #ic(b''.join(window))

            if len(window) < len(match):
                # keep filling the window
                continue

            # if it's possible to do a match, see if there is one
            if len(window) == len(match):
                #ic(len(window))
                print('\n')
                eprint('match :', repr(match))
                eprint('window:', repr(b''.join(window)))
                #ic(window)
                # if there is a match, we know the whole window gets replaced, =>< the current window
                if b''.join(window) == match:
                    eprint("matched")
                    window = replacement
                    ic(window)
                    modified = True
                    temp_file.write(window)  # flush the replacement to disk
                    window = []  # start a new window, dont want to match on the replacement
                    continue
                else:
                    # here the window was full, but it did not match, so the window must be shifted by one byte, and the byte that fell off must be written
                    # the same thing needs to happen when the window was not full, so there is nothing to do down here
                    pass

            if (len(window) - 1) == len(match):     # window needs to move
                temp_file.write(window[0])
                window = window[1:]
                assert len(window) == window_size   # only time window_size is used

            assert len(window) <= len(match)

        temp_file_name = temp_file.name
        temp_file.close()
        if modified:
            shutil.copystat(file_to_modify, temp_file_name)
            shutil.move(temp_file_name, file_to_modify)
        else:
            os.unlink(temp_file_name)

        ic(modified)

def replace_text(file_to_modify: Path,
                 match,
                 replacement,
                 verbose: bool,
                 debug: bool,
                 ):

    if isinstance(match, bytes):
        replace_text_bytes(file_to_modify=file_to_modify,
                           match=match,
                           replacement=replacement,
                           verbose=verbose,
                           debug=debug,
                           )
    else:
        replace_text_line(file_to_modify=file_to_modify,
                          match=match,
                          replacement=replacement,
                          verbose=verbose,
                          debug=debug,
                          )

def get_thing(*,
              prompt: str,
              match: str,
              match_file: str,
              ask: bool,
              verbose: bool,
              debug: bool,
              ):

    result = None
    assert prompt in ['match', 'replacement']
    if not maxone([match, match_file, ask]):
        raise ValueError('--{0} --{0}-file and --ask-{0} are mutually exclusive'.format(prompt))
    if match:
        assert len(match) > 0
        result = match
    if match_file:
        match_file = Path(match_file)
        with open(match_file.as_posix(), 'rb') as fh:
            file_bytes = fh.read()
        assert len(file_bytes) > 0
        #result = file_bytes.decode('utf8')
        result = file_bytes
    if ask:
        match = input(prompt)
        assert len(match) > 0
        result = match

    if result:
        if verbose:
            ic(result)
        return result

    raise ValueError('one of --{0} --{0}-file or --ask-{0} is required'.format(prompt))

@click.command()
@click.argument("files", nargs=-1, required=False)
@click.option("--match", type=str)
@click.option("--replacement", type=str)
@click.option('--match-file', type=str)
@click.option('--replacement-file', type=str)
@click.option('--recursive', '-r', is_flag=True)
@click.option('--endswith', type=str)
@click.option('--recursive-dotfiles', '-d', is_flag=True)
@click.option('--verbose', is_flag=True)
@click.option('--debug', is_flag=True)
@click.option('--ask-match', is_flag=True, help="escape from shell escaping")
@click.option('--ask-replacement', is_flag=True, help="escape from shell escaping")
def cli(files,
        match,
        replacement,
        match_file,
        replacement_file,
        recursive,
        recursive_dotfiles,
        endswith,
        verbose,
        debug,
        ask_match,
        ask_replacement,
        ):

    match = get_thing(prompt='match',
                      match=match,
                      match_file=match_file,
                      ask=ask_match,
                      verbose=verbose,
                      debug=debug,)

    replacement = get_thing(prompt='replacement',
                            match=replacement,
                            match_file=replacement_file,
                            ask=ask_replacement,
                            verbose=verbose,
                            debug=debug,)

    if isinstance(match, bytes):
        assert isinstance(replacement, bytes)

    if not files:  # todo: enumerate_input
        for line in sys.stdin:
            print(line.replace(match, replacement), end='')

    files = list(files)

    for file_to_modify in files:
        if verbose:
            ic(file_to_modify)
        if endswith:
            if not file_to_modify.endswith(endswith):
                continue
        if os.path.isdir(file_to_modify):
            if not recursive:
                print("Warning: skipping folder:",
                      file_to_modify,
                      "specify --recursive to decend into it.", file=sys.stderr)
                continue

            for sub_file in all_files_iter(file_to_modify):
                if is_regular_file(sub_file):
                    if '.' in os.fsdecode(sub_file.parent):
                        if not recursive_dotfiles:
                            if verbose:
                                eprint("skipping:", sub_file, "due to dot '.' in parent")
                            continue
                    replace_text(file_to_modify=Path(sub_file),
                                 match=match,
                                 replacement=replacement,
                                 verbose=verbose,
                                 debug=debug,)

        else:
            if is_regular_file(file_to_modify):
                try:
                    replace_text(file_to_modify=Path(file_to_modify),
                                 match=match,
                                 replacement=replacement,
                                 verbose=verbose,
                                 debug=debug,)
                except UnicodeDecodeError:
                    pass
<commit_msg>auto-commit<commit_after>#!/usr/bin/env python3
'''
# Replace the given text in file(s) or stdin.
'''

# pylint: disable=C0111     # docstrings are always outdated and wrong
# pylint: disable=W0511     # todo is encouraged
# pylint: disable=C0301     # line too long
# pylint: disable=R0902     # too many instance attributes
# pylint: disable=C0302     # too many lines in module
# pylint: disable=C0103     # single letter var names, func name too descriptive
# pylint: disable=R0911     # too many return statements
# pylint: disable=R0912     # too many branches
# pylint: disable=R0915     # too many statements
# pylint: disable=R0913     # too many arguments
# pylint: disable=R1702     # too many nested blocks
# pylint: disable=R0914     # too many local variables
# pylint: disable=R0903     # too few public methods
# pylint: disable=E1101     # no member for base
# pylint: disable=W0201     # attribute defined outside __init__

import os
import shutil
import stat
import sys
import tempfile
from pathlib import Path

#from icecream import ic  # too many deps
import click
from asserttool import maxone
from colorama import Fore
from colorama import Style

#note adding deps requires changes to sendgentoo

def eprint(*args, **kwargs):
    if 'file' in kwargs.keys():
        kwargs.pop('file')
    print(*args, file=sys.stderr, **kwargs)

try:
    from icecream import ic  # https://github.com/gruns/icecream
except ImportError:
    ic = eprint

def is_regular_file(path):
    mode = os.stat(path, follow_symlinks=False)[stat.ST_MODE]
    if stat.S_ISREG(mode):
        return True
    return False

def all_files_iter(p):
    if isinstance(p, str):
        p = Path(p)
    elif isinstance(p, bytes):
        p = Path(p.decode())
    assert isinstance(p, Path)
    yield p.absolute()
    for sub in p.iterdir():
        # eprint("sub:", sub)  # todo: read by terminal, so bell etc happens.... eprint bug?
        if sub.is_symlink():  # must be before is_dir()
            yield sub.absolute()
        elif sub.is_dir():
            yield from all_files_iter(sub)
        else:
            yield sub.absolute()

def replace_text_line(*,
                      file_to_modify: Path,
                      match: str,
                      replacement: str,
                      verbose: bool,
                      debug: bool,
                      ):

    assert isinstance(file_to_modify, Path)
    if verbose:
        eprint(file_to_modify)

    #file_to_modify_basename = os.path.basename(file_to_modify)
    file_to_modify_dir = os.path.dirname(file_to_modify)
    temp_file = tempfile.NamedTemporaryFile(mode='w',
                                            prefix='tmp-',
                                            dir=file_to_modify_dir,
                                            delete=False)

    # this cant handle binary files... or files with mixed newlines
    modified = False
    with open(file_to_modify, 'rU') as file_to_modify_fh:
        try:
            for line in file_to_modify_fh:
                if match in line:
                    modified = True
                new_line = line.replace(match, replacement)
                temp_file.write(new_line)
                #if file_to_modify_fh.newlines == '\n':      # LF (Unix)
                #    temp_file.write("%s\n" % new_line)
                #    continue
                #elif file_to_modify_fh.newlines == '\r\n':  # CR+LF (DOS/Win)
                #    temp_file.write("%s\r\n" % new_line)
                #    continue
                #elif file_to_modify_fh.newlines == '\r':    # CR (Mac OS <= v9)
                #    temp_file.write("%s\r" % new_line)
        except UnicodeDecodeError as e:
            print("UnicodeDecodeError:", file_to_modify, file=sys.stderr)
            raise e

        temp_file_name = temp_file.name
        temp_file.close()
        if modified:
            shutil.copystat(file_to_modify, temp_file_name)
            shutil.move(temp_file_name, file_to_modify)
        else:
            os.unlink(temp_file_name)

def replace_text_bytes(*,
                       file_to_modify: Path,
                       match: bytes,
                       replacement: bytes,
                       verbose: bool,
                       debug: bool,
                       ):

    assert isinstance(file_to_modify, Path)
    if verbose:
        ic(file_to_modify)

    assert isinstance(match, bytes)
    assert isinstance(replacement, bytes)

    window_size = len(match) # need to expand a matching block by an arb amount, replacement can be any size

    #file_to_modify_basename = os.path.basename(file_to_modify)
    file_to_modify_dir = os.path.dirname(file_to_modify)
    temp_file = tempfile.NamedTemporaryFile(mode='wb',
                                            prefix='tmp-',
                                            dir=file_to_modify_dir,
                                            delete=False)

    # this cant handle binary files... or files with mixed newlines
    modified = False
    location_read = 0
    location_write = 0
    window = []
    with open(file_to_modify, 'rb') as fh:
        while True:
            # window starts off empty
            eprint(len(window), location_read)
            fh.seek(location_read)
            next_byte = fh.read(1)
            ic(next_byte)
            if next_byte is b'':
                break

            window.append(next_byte)
            location_read += 1
            # first byte in window, or EOF

            #ic(b''.join(window))

            if len(window) < len(match):
                # keep filling the window
                continue

            # if it's possible to do a match, see if there is one
            if len(window) == len(match):
                #ic(len(window))
                print('\n')
                eprint('match :', repr(match))
                eprint('window:', repr(b''.join(window)))
                #ic(window)
                # if there is a match, we know the whole window gets replaced, =>< the current window
                if b''.join(window) == match:
                    eprint("matched")
                    window = replacement
                    ic(window)
                    modified = True
                    temp_file.write(window)  # flush the replacement to disk
                    window = []  # start a new window, dont want to match on the replacement
                    continue
                else:
                    # here the window was full, but it did not match, so the window must be shifted by one byte, and the byte that fell off must be written
                    # the same thing needs to happen when the window was not full, so there is nothing to do down here
                    pass

            if (len(window) - 1) == len(match):     # window needs to move
                temp_file.write(window[0])
                window = window[1:]
                assert len(window) == window_size   # only time window_size is used

            assert len(window) <= len(match)

        temp_file_name = temp_file.name
        temp_file.close()
        if modified:
            shutil.copystat(file_to_modify, temp_file_name)
            shutil.move(temp_file_name, file_to_modify)
        else:
            os.unlink(temp_file_name)

        ic(modified)

def replace_text(file_to_modify: Path,
                 match,
                 replacement,
                 verbose: bool,
                 debug: bool,
                 ):

    if isinstance(match, bytes):
        replace_text_bytes(file_to_modify=file_to_modify,
                           match=match,
                           replacement=replacement,
                           verbose=verbose,
                           debug=debug,
                           )
    else:
        replace_text_line(file_to_modify=file_to_modify,
                          match=match,
                          replacement=replacement,
                          verbose=verbose,
                          debug=debug,
                          )

def get_thing(*,
              prompt: str,
              match: str,
              match_file: str,
              ask: bool,
              verbose: bool,
              debug: bool,
              ):

    result = None
    assert prompt in ['match', 'replacement']
    if not maxone([match, match_file, ask]):
        raise ValueError('--{0} --{0}-file and --ask-{0} are mutually exclusive'.format(prompt))
    if match:
        assert len(match) > 0
        result = match
    if match_file:
        match_file = Path(match_file)
        with open(match_file.as_posix(), 'rb') as fh:
            file_bytes = fh.read()
        assert len(file_bytes) > 0
        #result = file_bytes.decode('utf8')
        result = file_bytes
    if ask:
        match = input(prompt)
        assert len(match) > 0
        result = match

    if result:
        if verbose:
            ic(result)
        return result

    raise ValueError('one of --{0} --{0}-file or --ask-{0} is required'.format(prompt))

@click.command()
@click.argument("files", nargs=-1, required=False)
@click.option("--match", type=str)
@click.option("--replacement", type=str)
@click.option('--match-file', type=str)
@click.option('--replacement-file', type=str)
@click.option('--recursive', '-r', is_flag=True)
@click.option('--endswith', type=str)
@click.option('--recursive-dotfiles', '-d', is_flag=True)
@click.option('--verbose', is_flag=True)
@click.option('--debug', is_flag=True)
@click.option('--ask-match', is_flag=True, help="escape from shell escaping")
@click.option('--ask-replacement', is_flag=True, help="escape from shell escaping")
def cli(files,
        match,
        replacement,
        match_file,
        replacement_file,
        recursive,
        recursive_dotfiles,
        endswith,
        verbose,
        debug,
        ask_match,
        ask_replacement,
        ):

    match = get_thing(prompt='match',
                      match=match,
                      match_file=match_file,
                      ask=ask_match,
                      verbose=verbose,
                      debug=debug,)

    replacement = get_thing(prompt='replacement',
                            match=replacement,
                            match_file=replacement_file,
                            ask=ask_replacement,
                            verbose=verbose,
                            debug=debug,)

    if isinstance(match, bytes):
        assert isinstance(replacement, bytes)

    if not files:  # todo: enumerate_input
        for line in sys.stdin:
            print(line.replace(match, replacement), end='')

    files = list(files)

    for file_to_modify in files:
        if verbose:
            ic(file_to_modify)
        if endswith:
            if not file_to_modify.endswith(endswith):
                continue
        if os.path.isdir(file_to_modify):
            if not recursive:
                print("Warning: skipping folder:",
                      file_to_modify,
                      "specify --recursive to decend into it.", file=sys.stderr)
                continue

            for sub_file in all_files_iter(file_to_modify):
                if is_regular_file(sub_file):
                    if '.' in os.fsdecode(sub_file.parent):
                        if not recursive_dotfiles:
                            if verbose:
                                eprint("skipping:", sub_file, "due to dot '.' in parent")
                            continue
                    replace_text(file_to_modify=Path(sub_file),
                                 match=match,
                                 replacement=replacement,
                                 verbose=verbose,
                                 debug=debug,)

        else:
            if is_regular_file(file_to_modify):
                try:
                    replace_text(file_to_modify=Path(file_to_modify),
                                 match=match,
                                 replacement=replacement,
                                 verbose=verbose,
                                 debug=debug,)
                except UnicodeDecodeError:
                    pass
<|endoftext|>


<commit_before>"""
Given a string S and a string T, count the number of distinct subsequences of T in S.

A subsequence of a string is a new string which is formed from the original string by deleting some (can be none) of the characters without disturbing the relative positions of the remaining characters. (ie, "ACE" is a subsequence of "ABCDE" while "AEC" is not).

Here is an example:
S = "rabbbit", T = "rabbit"

Return 3.

Note:
The answer is three rabbit by removing the first, second, third 'b'
"""

class Solution:
    # @return an integer
    def numDistinct(self, S, T):
        N = len(S)
        M = len(T)
        dp = [ [0 for j in range(N+1)] for i in range(M+1)]
        for i in range(M+1):
            dp[i][0] = 0
        for j in range(N+1):
            dp[0][j] = 1
        for i in range(1, M+1):
            for j in range(1, N+1):
                if S[j-1] == T[i-1]:
                    dp[i][j] = dp[i][j-1] + dp[i-1][j-1]
                else:
                    dp[i][j] = dp[i][j-1]
        return dp[M][N]
    # Note
    # dp[i][j]表示S的前i个字符配上T的前j个字符的DS
    # [i][0] = 0, dp[0][j] = 1
    # dp[i][j] = dp[i][j-1] + dp[i-1][j-1] # if S[i-1] == T[j-1]
    #          = dp[i][j-1]                # if S[i-1] != T[j-1]
    # dp[M][N]
<commit_msg>Added distinct subsequences<commit_after>"""
Given a string S and a string T, count the number of distinct subsequences of T in S.

A subsequence of a string is a new string which is formed from the original string by deleting some (can be none) of the characters without disturbing the relative positions of the remaining characters. (ie, "ACE" is a subsequence of "ABCDE" while "AEC" is not).

Here is an example:
S = "rabbbit", T = "rabbit"

Return 3.

Note:
The answer is three rabbit by removing the first, second, third 'b'
"""

class Solution:
    # @return an integer
    def numDistinct(self, S, T):
        M = len(T)
        N = len(S)
        dp = [ [0 for j in range(N+1)] for i in range(M+1)]
        for i in range(M+1):
            dp[i][0] = 0
        for j in range(N+1):
            dp[0][j] = 1
        for i in range(1, M+1):
            for j in range(1, N+1):
                if S[j-1] == T[i-1]:
                    dp[i][j] = dp[i][j-1] + dp[i-1][j-1]
                else:
                    dp[i][j] = dp[i][j-1]
        return dp[M][N]
    # !!!!分清M,i和N,j分别对应T和S哪个
    # Note:
    # dp[i][j]表示S的前i个字符配上T的前j个字符的DS
    # [i][0] = 0, dp[0][j] = 1
    # dp[i][j] = dp[i][j-1] + dp[i-1][j-1] # if T[i-1] == S[j-1]
    #          = dp[i][j-1]                # if T[i-1] != S[j-1]
    # dp[M][N]
    # Need to draw this pic when solving this problem
    # 大概意思就是， 因为算的是S的子串和T匹配的方法， 所以一旦S[:j-1]和T[:i]有x种匹配方法时
    # S[:j]必定也至少和T[:i]有x种匹配方法，但尤其当S[j-1]==T[i-1]的时候，需要再加上S[:j-1]和T[:i-1]的匹配方法数
    #     r a b b b i t
    #   1 1 1 1 1 1 1 1
    # r 0 1 1 1 1 1 1 1
    # a 0 0 1 1 1 1 1 1
    # b 0 0 0 1 2 3 3 3
    # b 0 0 0 0 1 3 3 3
    # i 0 0 0 0 0 0 3 3
    # t 0 0 0 0 0 0 0 3
    # No matter T[i-1] ?= S[j-1],  dp[i][j] = dp[i][j-1]
    # But    if T[i-1] == S[j-1], we can add another one which is dp[i-1][j-1]
<|endoftext|>


<commit_before># Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

def GetFactoryProperties(api, factory_properties, build_properties):
  # TODO(iannucci): Pass the build repo info directly via build_properties
  repo_name = factory_properties.get('repo_name')
  steps = api.Steps(build_properties)

  spec = steps.gclient_common_spec(repo_name)

  git_steps = []
  if spec['solutions'][0]['url'].endswith('.git'):
    email = 'commit-bot@chromium.org'
    git_steps = [
      steps.git_step('config', 'user.email', email),
      steps.git_step('config', 'user.name', 'The Commit Bot'),
      steps.git_step('clean', '-xfq'),
    ]

  return {
    'checkout': 'gclient',
    'gclient_spec': spec,
    'steps': git_steps + [
      steps.apply_issue_step(),
      steps.step('presubmit', [
        api.depot_tools_path('presubmit_support.py'),
        '--root', api.checkout_path(),
        '--commit',
        '--verbose', '--verbose',
        '--issue', build_properties['issue'],
        '--patchset', build_properties['patchset'],
        '--skip_canned', 'CheckRietveldTryJobExecution',
        '--skip_canned', 'CheckTreeIsOpen',
        '--skip_canned', 'CheckBuildbotPendingBuilds',
        '--skip_canned', 'CheckOwners',
        '--rietveld_url', build_properties['rietveld'],
        '--rietveld_fetch'])
    ]
  }
<commit_msg>Use anonymous mode for presubmit_support on run_presubmit recipe.<commit_after># Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

def GetFactoryProperties(api, factory_properties, build_properties):
  # TODO(iannucci): Pass the build repo info directly via build_properties
  repo_name = factory_properties.get('repo_name')
  steps = api.Steps(build_properties)

  spec = steps.gclient_common_spec(repo_name)

  git_steps = []
  if spec['solutions'][0]['url'].endswith('.git'):
    email = 'commit-bot@chromium.org'
    git_steps = [
      steps.git_step('config', 'user.email', email),
      steps.git_step('config', 'user.name', 'The Commit Bot'),
      steps.git_step('clean', '-xfq'),
    ]

  return {
    'checkout': 'gclient',
    'gclient_spec': spec,
    'steps': git_steps + [
      steps.apply_issue_step(),
      steps.step('presubmit', [
        api.depot_tools_path('presubmit_support.py'),
        '--root', api.checkout_path(),
        '--commit',
        '--verbose', '--verbose',
        '--issue', build_properties['issue'],
        '--patchset', build_properties['patchset'],
        '--skip_canned', 'CheckRietveldTryJobExecution',
        '--skip_canned', 'CheckTreeIsOpen',
        '--skip_canned', 'CheckBuildbotPendingBuilds',
        '--skip_canned', 'CheckOwners',
        '--rietveld_url', build_properties['rietveld'],
        '--rietveld_email', '',  # activates anonymous mode
        '--rietveld_fetch'])
    ]
  }
<|endoftext|>


<commit_before>#
# blake.core 0.2.1
# Do cool things with your Markdown docs
# skazhy / sept-dec 2012
#

import copy
import os
import re
import unicodedata
import yaml

from datetime import datetime
from markdown import markdown

EXTENSIONS = [".md", ".markdown"]

class QueryDict(object):
    def __init__(self, d=None):
        if d is not None:
            self._dict = d
        else:
            self._dict = {}

    def __getitem__(self, key):
        return self._dict.get(key, None)

    def __setitem__(self, key, value):
        self._dict[key] = value

    def __delitem__(self, key):
        if key in self._dict:
            self._dict.pop(key)

    def __iter__(self):
        for key in self._dict:
            yield key

    def keys(self):
        return self._dict.keys()

class AttrList(object):
    # TODO: blake0.2.2 should add more functions to both of these
    #       classes
    def __init__(self):
        self._list = []

    def __getitem__(self, key):
        return self._list[key]

    def __setitem__(self, key, value):
        self._list[key] = value

    def __iter__(self):
        for item in self._list:
            yield item

    def __contains__(self, item):
        return item in self._list

    def __str__(self):
        return ",".join(self._list)

    def __unicode__(self):
        return ",".join(self._list)

    def append(self, item):
        self._list.append(item)

def islocal(path):
    if path[:7] == "http://" or path[:8] == "https://" or path[:2] == "//":
        return False
    return True

def _validate_path(path, filename=None, extensions=EXTENSIONS):
    """Checks if given path contains a valid Markdown document."""

    # If no filename is given, assume that it is given in path
    if filename is None:
        path, filename = os.path.split(path)

    name, extension = os.path.splitext(filename)
    extension = extension.lower()

    if not os.path.exists(path):
        return False

    # Ignore hidden files
    if not len(name) or name[0] == '.':
        return False

    #  If given file has a proper extension, return the full path
    for t in extensions:
        if extension == t:
            return os.path.join(path, filename)

    return False

def _relative_subdirectories(src, path):
    """ Gets the relative subdirectory part from 2 given paths."""
    return filter(lambda x: len(x) > 0, path.replace(src, "").split("/"))

def slugify(text, delim='-', escape_html=False):
    """Generates an ASCII-only slug."""

    if type(text) is list:
        text = delim.join(text)

    text = text.decode("UTF-8")
    text = unicodedata.normalize("NFKD", text)  # transform to normalforms
    text = text.encode("ascii", "ignore").decode("ascii")

    _punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+')

    if escape_html:
        text = re.sub('<[^<]+?>', '', text)

    result = []
    for word in _punct_re.split(text.lower()):
        if word:
            result.append(word)
    return delim.join(result)

class Blake(object):
    @property
    def slug(self):
        """Returns slug of the document."""
        if self._slug is not None:
            return self._slug
        slug = ""
        if "subdirectory" in self.head and self.head["subdirectory"]:
            subdir = self.head["subdirectory"]
            dir_slug = "-".join(map(lambda x: slugify(x), subdir))
            return "-".join([dir_slug, slugify(self.filename)])
        if self.filename is not None:
            return self.filename
        return None

    @slug.setter
    def slug(self, value):
        self._slug = slugify(value)

    @property
    def title(self):
        if self._title is not None:
            return self._title
        return self.filename

    @title.setter
    def title(self, value):
        self._title = value

class Document(Blake):
    def __init__(self, filename=None, parse=True, static_prefix=""):
        self.head = QueryDict({
            "full_path": filename,
            "subdirectory": []
        })
        self._slug = None
        self._title = None
        self._content = ""
        self.static_prefix = static_prefix

        if parse:
            self.create()

    def __hash__(self):
        # Unique identifier for 2 newly created docs will work?
        return self.slug.__hash__()

    def __eq__(self, other):
        return hash(self) == hash(other)

    def _path_split(self, index=0):
        if self.head["full_path"] is not None:
            path, filename = os.path.split(self.head["full_path"])
            filename, extension = os.path.splitext(filename)
            return [path, filename, extension][index]
        return None

    @property
    def filename(self):
        return self._path_split(1)

    @property
    def extension(self):
        return self._path_split(2)

    @property
    def images(self):
        pattern = '!\[.*\]\((.*)\)'
        seen = set()
        seen_add = seen.add
        return [x for x in [i for i in re.findall(pattern, self._content)]
                if x not in seen and not seen_add(x)]

    # Override the following methods to define custom
    # rendering and content generation process
    def render(self, renderable, context):
        md = markdown(renderable)
        for img in filter(lambda x: islocal(x), self.images):
            md = md.replace(img, self.static_prefix + img)
        return md

    def context(self):
        # Default context returns everything except content
        return self.to_dict(exclude=["content"])

    @property
    def content(self):
        return self.render(self._content, self.context())

    @content.setter
    def content(self, c):
        self._content = c

    def create(self, head=None):
        """Parses the raw document."""
        if self.head["full_path"] is not None:
            blakefile = open(self.head["full_path"], "r")
            cont = ""
            yaml_present = False
            blakefile.readline()   # This line should always contain "---"
            line = blakefile.readline()
            while line:
                if line[0] == "-":
                    yaml_present = True
                    break
                cont += line
                line = blakefile.readline()

            if yaml_present:
                # The following line is the bottleneck of #create(), should
                # investigate how to boost YAML parsing
                h = yaml.load(cont)
                for key in h:
                    if key == "title":
                        self._title = h["title"]
                    elif key == "tags":
                        if not isinstance(h["tags"], list):
                            h["tags"] = h["tags"].split(",")
                        self.head["tags"] = AttrList()
                        [self.head["tags"].append(t.strip()) for t in h["tags"]]
                    else:
                        self.head[key] = h[key]
                line = blakefile.readline()
                while line:
                    self._content += line
                    line = blakefile.readline()
            else:
                self._content = cont
            blakefile.close()
            self._content = self._content.decode("UTF-8")

        # Extra params in head argument override those found in yaml (if any)
        if head is not None:
            for key in head:
                self.head[key] = head[key]

        if "published" in self.head:
            try:
                self.head['published'].now()
            except AttributeError:
                self.head['published'] = datetime.now()

    def to_dict(self, include=[], exclude=[]):
        """ Return a dict representing attributes. """
        d = {}
        if exclude is None:
            exclude = []
        for key in self.head.keys():
            if key not in exclude:
                d[key] = self.head[key].__str__()

        if "slug" not in exclude:
            d["slug"] = self.slug
        if "filename" not in exclude:
            d["filename"] = self.filename
        if "title" not in exclude:
            d["title"] = self.title
        if "content" not in exclude:
            d["content"] = self.content

        # If include is present - leave only given keys
        # This is kinda dangerous -as is-
        if include:
            for key in d.keys():
                if key not in include:
                    d.pop(key)

        if "subdirectory" in d:
            d["subdirectory"] = "/".join(self.head["subdirectory"])

        return d

    def slugify(self, attr):
        """ Slugify a head attr."""
        if attr in self.head:
            return slugify(self.head[attr])
        return None

    def add_slug(self, *args, **kwargs):
        for arg in args:
            if arg in self.head:
                self.head["%s_slug" % arg] = slugify(self.head[arg])

    def dump(self):
        yield "---\n"
        exc = ["full_path", "subdirectory", "content", "filename", "slug"]
        dct = self.to_dict(exclude=exc)
        for key in dct:
            yield "%s: %s\n" % (key, dct[key])
        yield "---\n"
        yield self._content.encode("utf-8")

    def replace(self, old, new, count=-1):
        """ Merely a wrapper for str.replace. """
        self._content = self._content.replace(old, new, count)

class DocumentList(Blake):
    document = Document
    extensions = EXTENSIONS

    def __init__(self, src=None, static_prefix="", recursive=True):
        self._documents = []
        self._slug = None
        self.static_prefix = static_prefix
        self.subdirectory = []

        if src:
            self.add(src, recursive=recursive)

    def __getitem__(self, i):
        if isinstance(i, slice):
            a = copy.copy(self)
            a._documents = self._documents[i]
            return a
        else:
            return self._documents[i]

    def __iter__(self):
        for doc in self._documents:
            yield doc

    def __len__(self):
        return len(self._documents)

    # documentlist += document
    def __iadd__(self, doc):
        if isinstance(doc, Document):
            self._documents.append(doc)
        return self

    @property
    def documents(self):
        return self._documents

    @documents.setter
    def documents(self, docs):
        self._documents = docs

    def add(self, filename=None, parse=True, static_prefix="", recursive=True):
        # TODO: nested documentlists
        if not os.path.exists(filename):
            return False
        if os.path.isdir(filename):
            valid_documents(filename, parse=parse,
                            instance=self,
                            static_prefix=static_prefix,
                            recursive=recursive)
        else:
            self += self.document(filename=filename,
                                  parse=parse,
                                  static_prefix=static_prefix)
        return True

    def exclude(self, *args, **kwargs):
        new_kw = {}
        for key in kwargs:
            new_kw[key + "__ne"] = kwargs[key]
        return self.find(self, *args, **new_kw)

    def find(self, *args, **kwargs):
        """ Query the documentlist. Return a DL with fewer results."""
        if kwargs:
            a = copy.copy(self)
            while kwargs:
                key, value = kwargs.popitem()
                key_arr = key.split("__")
                # This should be improved to add more __ options
                if len(key_arr) == 2 and key_arr[1] == "has":
                    key = key_arr[0]
                    a.documents = filter(lambda x: x.head[key] and value in x.head[key], a)
                else:
                    if len(key_arr) == 2 and key_arr[1] == "ne":
                        key = key_arr[0]
                        comp = value.__ne__
                    else:
                        comp = value.__eq__

                    if key == "title":
                        a.documents = filter(lambda x: comp(x.title), a)
                    elif key == "filename":
                        a.documents = filter(lambda x: comp(x.filename), a)
                    elif key == "slug":
                        a.documents = filter(lambda x: comp(x.slug), a)
                    else:
                        a.documents = filter(lambda x: comp(x.head[key]), a)
            return a
        return self

    def to_list(self, include=[], exclude=[]):
        return map(lambda d: d.to_dict(include=include, exclude=exclude), self)

    def get(self, *args, **kwargs):
        """ Retrieve a single document. """
        doc = self.find(*args, **kwargs).documents
        if not len(doc):
            return None
        return doc[0]

    def distinct(self, key, sparse=True):
        """ Get distinct values of an attribute. """
        values = []
        if key == "title":
            for doc in self:
                values.append(doc.title)
        elif key == "filename":
            for doc in self:
                values.append(doc.filename)
        elif key == "slug":
            for doc in self:
                values.append(doc.slug)
        else:
            for doc in self:
                values.append(doc.head[key])
        s = set(values)
        if sparse and None in s:
            s.remove(None)
        return s

    def to_list(self, include=[], exclude=[]):
        """ Returns list of to_dict for each of the elements. """
        return map(lambda d: d.to_dict(include=include, exclude=exclude), self)

    def count():
        return len(self)

def valid_documents(src, parse=True, instance=None, static_prefix="", recursive=True):
    # This allows to populate the DL from customized instances
    if instance is not None:
        d = instance
    else:
        d = DocumentList()

    if recursive:
        for (path, dirs, files) in os.walk(src):
            for filename in files:
                doc_path = _validate_path(path, filename, d.extensions)
                if doc_path:
                    d.add(filename=doc_path, parse=parse, static_prefix=static_prefix)
                    d[-1].head["subdirectory"] = _relative_subdirectories(src, path)
    else:
        for filename in os.listdir(src):
            doc_path = _validate_path(src, filename)
            if doc_path:
                d.add(doc_path, parse, static_prefix=static_prefix)
    if instance is not None:
        return True
    return d
<commit_msg>More universal find() and distinct()<commit_after>#
# blake.core 0.2.1
# Do cool things with your Markdown docs
# skazhy / sept-dec 2012
#

import copy
import os
import re
import unicodedata
import yaml

from datetime import datetime
from markdown import markdown

EXTENSIONS = [".md", ".markdown"]

class QueryDict(object):
    def __init__(self, d=None):
        if d is not None:
            self._dict = d
        else:
            self._dict = {}

    def __getitem__(self, key):
        return self._dict.get(key, None)

    def __setitem__(self, key, value):
        self._dict[key] = value

    def __delitem__(self, key):
        if key in self._dict:
            self._dict.pop(key)

    def __iter__(self):
        for key in self._dict:
            yield key

    def keys(self):
        return self._dict.keys()

class AttrList(object):
    # TODO: blake0.2.2 should add more functions to both of these
    #       classes
    def __init__(self):
        self._list = []

    def __getitem__(self, key):
        return self._list[key]

    def __setitem__(self, key, value):
        self._list[key] = value

    def __iter__(self):
        for item in self._list:
            yield item

    def __contains__(self, item):
        return item in self._list

    def __str__(self):
        return ",".join(self._list)

    def __unicode__(self):
        return ",".join(self._list)

    def append(self, item):
        self._list.append(item)

def islocal(path):
    if path[:7] == "http://" or path[:8] == "https://" or path[:2] == "//":
        return False
    return True

def _validate_path(path, filename=None, extensions=EXTENSIONS):
    """Checks if given path contains a valid Markdown document."""

    # If no filename is given, assume that it is given in path
    if filename is None:
        path, filename = os.path.split(path)

    name, extension = os.path.splitext(filename)
    extension = extension.lower()

    if not os.path.exists(path):
        return False

    # Ignore hidden files
    if not len(name) or name[0] == '.':
        return False

    #  If given file has a proper extension, return the full path
    for t in extensions:
        if extension == t:
            return os.path.join(path, filename)

    return False

def _relative_subdirectories(src, path):
    """ Gets the relative subdirectory part from 2 given paths."""
    return filter(lambda x: len(x) > 0, path.replace(src, "").split("/"))

def slugify(text, delim='-', escape_html=False):
    """Generates an ASCII-only slug."""

    if type(text) is list:
        text = delim.join(text)

    text = text.decode("UTF-8")
    text = unicodedata.normalize("NFKD", text)  # transform to normalforms
    text = text.encode("ascii", "ignore").decode("ascii")

    _punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+')

    if escape_html:
        text = re.sub('<[^<]+?>', '', text)

    result = []
    for word in _punct_re.split(text.lower()):
        if word:
            result.append(word)
    return delim.join(result)

class Blake(object):
    @property
    def slug(self):
        """Returns slug of the document."""
        if self._slug is not None:
            return self._slug
        slug = ""
        if "subdirectory" in self.head and self.head["subdirectory"]:
            subdir = self.head["subdirectory"]
            dir_slug = "-".join(map(lambda x: slugify(x), subdir))
            return "-".join([dir_slug, slugify(self.filename)])
        if self.filename is not None:
            return self.filename
        return None

    @slug.setter
    def slug(self, value):
        self._slug = slugify(value)

    @property
    def title(self):
        if self._title is not None:
            return self._title
        return self.filename

    @title.setter
    def title(self, value):
        self._title = value

class Document(Blake):
    def __init__(self, filename=None, parse=True, static_prefix=""):
        self.head = QueryDict({
            "full_path": filename,
            "subdirectory": []
        })
        self._slug = None
        self._title = None
        self._content = ""
        self.static_prefix = static_prefix

        if parse:
            self.create()

    def __hash__(self):
        # Unique identifier for 2 newly created docs will work?
        return self.slug.__hash__()

    def __eq__(self, other):
        return hash(self) == hash(other)

    def _path_split(self, index=0):
        if self.head["full_path"] is not None:
            path, filename = os.path.split(self.head["full_path"])
            filename, extension = os.path.splitext(filename)
            return [path, filename, extension][index]
        return None

    @property
    def filename(self):
        return self._path_split(1)

    @property
    def extension(self):
        return self._path_split(2)

    @property
    def images(self):
        pattern = '!\[.*\]\((.*)\)'
        seen = set()
        seen_add = seen.add
        return [x for x in [i for i in re.findall(pattern, self._content)]
                if x not in seen and not seen_add(x)]

    # Override the following methods to define custom
    # rendering and content generation process
    def render(self, renderable, context):
        md = markdown(renderable)
        for img in filter(lambda x: islocal(x), self.images):
            md = md.replace(img, self.static_prefix + img)
        return md

    def context(self):
        # Default context returns everything except content
        return self.to_dict(exclude=["content"])

    @property
    def content(self):
        return self.render(self._content, self.context())

    @content.setter
    def content(self, c):
        self._content = c

    def create(self, head=None):
        """Parses the raw document."""
        if self.head["full_path"] is not None:
            blakefile = open(self.head["full_path"], "r")
            cont = ""
            yaml_present = False
            blakefile.readline()   # This line should always contain "---"
            line = blakefile.readline()
            while line:
                if line[0] == "-":
                    yaml_present = True
                    break
                cont += line
                line = blakefile.readline()

            if yaml_present:
                # The following line is the bottleneck of #create(), should
                # investigate how to boost YAML parsing
                h = yaml.load(cont)
                for key in h:
                    if key == "title":
                        self._title = h["title"]
                    elif key == "tags":
                        if not isinstance(h["tags"], list):
                            h["tags"] = h["tags"].split(",")
                        self.head["tags"] = AttrList()
                        [self.head["tags"].append(t.strip()) for t in h["tags"]]
                    else:
                        self.head[key] = h[key]
                line = blakefile.readline()
                while line:
                    self._content += line
                    line = blakefile.readline()
            else:
                self._content = cont
            blakefile.close()
            self._content = self._content.decode("UTF-8")

        # Extra params in head argument override those found in yaml (if any)
        if head is not None:
            for key in head:
                self.head[key] = head[key]

        if "published" in self.head:
            try:
                self.head['published'].now()
            except AttributeError:
                self.head['published'] = datetime.now()

    def to_dict(self, include=[], exclude=[]):
        """ Return a dict representing attributes. """
        d = {}
        if exclude is None:
            exclude = []
        for key in self.head.keys():
            if key not in exclude:
                d[key] = self.head[key].__str__()

        if "slug" not in exclude:
            d["slug"] = self.slug
        if "filename" not in exclude:
            d["filename"] = self.filename
        if "title" not in exclude:
            d["title"] = self.title
        if "content" not in exclude:
            d["content"] = self.content

        # If include is present - leave only given keys
        # This is kinda dangerous -as is-
        if include:
            for key in d.keys():
                if key not in include:
                    d.pop(key)

        if "subdirectory" in d:
            d["subdirectory"] = "/".join(self.head["subdirectory"])

        return d

    def slugify(self, attr):
        """ Slugify a head attr."""
        if attr in self.head:
            return slugify(self.head[attr])
        return None

    def add_slug(self, *args, **kwargs):
        for arg in args:
            if arg in self.head:
                self.head["%s_slug" % arg] = slugify(self.head[arg])

    def dump(self):
        yield "---\n"
        exc = ["full_path", "subdirectory", "content", "filename", "slug"]
        dct = self.to_dict(exclude=exc)
        for key in dct:
            yield "%s: %s\n" % (key, dct[key])
        yield "---\n"
        yield self._content.encode("utf-8")

    def replace(self, old, new, count=-1):
        """ Merely a wrapper for str.replace. """
        self._content = self._content.replace(old, new, count)

class DocumentList(Blake):
    document = Document
    extensions = EXTENSIONS
    properties = ("filename", "title", "slug", "bug")

    def __init__(self, src=None, static_prefix="", recursive=True):
        self._documents = []
        self._slug = None
        self.static_prefix = static_prefix
        self.subdirectory = []

        if src:
            self.add(src, recursive=recursive)

    def __getitem__(self, i):
        if isinstance(i, slice):
            a = copy.copy(self)
            a._documents = self._documents[i]
            return a
        else:
            return self._documents[i]

    def __iter__(self):
        for doc in self._documents:
            yield doc

    def __len__(self):
        return len(self._documents)

    # documentlist += document
    def __iadd__(self, doc):
        if isinstance(doc, Document):
            self._documents.append(doc)
        return self

    @property
    def documents(self):
        return self._documents

    @documents.setter
    def documents(self, docs):
        self._documents = docs

    def add(self, filename=None, parse=True, static_prefix="", recursive=True):
        # TODO: nested documentlists
        if not os.path.exists(filename):
            return False
        if os.path.isdir(filename):
            valid_documents(filename, parse=parse,
                            instance=self,
                            static_prefix=static_prefix,
                            recursive=recursive)
        else:
            self += self.document(filename=filename,
                                  parse=parse,
                                  static_prefix=static_prefix)
        return True

    def exclude(self, *args, **kwargs):
        new_kw = {}
        for key in kwargs:
            new_kw[key + "__ne"] = kwargs[key]
        return self.find(self, *args, **new_kw)

    def find(self, *args, **kwargs):
        """ Query the documentlist. Return a DL with fewer results."""
        if kwargs:
            a = copy.copy(self)
            while kwargs:
                key, value = kwargs.popitem()
                key_arr = key.split("__")
                # TODO: __has on self.properties attribute that is iterable
                # TODO: Improve to add more __ options, see how Django ORM
                #       implements __ option parsing
                if len(key_arr) == 2 and key_arr[1] == "has":
                    key = key_arr[0]
                    a.documents = filter(lambda x: x.head[key] and value in x.head[key], a)
                else:
                    if len(key_arr) == 2 and key_arr[1] == "ne":
                        key = key_arr[0]
                        comp = value.__ne__
                    else:
                        comp = value.__eq__

                    if key in self.properties:
                        a.documents = filter(lambda x: comp(getattr(x, key)), a)
                    else:
                        a.documents = filter(lambda x: comp(x.head[key]), a)
            return a
        return self

    def to_list(self, include=[], exclude=[]):
        return map(lambda d: d.to_dict(include=include, exclude=exclude), self)

    def get(self, *args, **kwargs):
        """ Retrieve a single document. """
        doc = self.find(*args, **kwargs).documents
        if not len(doc):
            return None
        return doc[0]

    def distinct(self, key, sparse=True):
        """ Get distinct values of an attribute. """
        values = []
        if key in self.properties:
            [values.append(getattr(doc, key, None)) for doc in self]
        else:
            [values.append(doc.head[key]) for doc in self]
        s = set(values)
        if sparse and None in s:
            s.remove(None)
        return s

    def to_list(self, include=[], exclude=[]):
        """ Returns list of to_dict for each of the elements. """
        return map(lambda d: d.to_dict(include=include, exclude=exclude), self)

    def count():
        return len(self)

def valid_documents(src, parse=True, instance=None, static_prefix="", recursive=True):
    # This allows to populate the DL from customized instances
    if instance is not None:
        d = instance
    else:
        d = DocumentList()

    if recursive:
        for (path, dirs, files) in os.walk(src):
            for filename in files:
                doc_path = _validate_path(path, filename, d.extensions)
                if doc_path:
                    d.add(filename=doc_path, parse=parse, static_prefix=static_prefix)
                    d[-1].head["subdirectory"] = _relative_subdirectories(src, path)
    else:
        for filename in os.listdir(src):
            doc_path = _validate_path(src, filename)
            if doc_path:
                d.add(doc_path, parse, static_prefix=static_prefix)
    if instance is not None:
        return True
    return d
<|endoftext|>