<commit_before># import modules
import os
import numpy as np
import requests
import csv
import random
import pandas as pd
from datetime import datetime, timedelta, date
from dateutil.rrule import rrule, DAILY

# set the path
path = '../'

#################################################################################################################
#                                        segment.csv                                                            #
#################################################################################################################

def select_trip_list(num_route=None, direction_id=0):
    """
    Generate the list of the trip id for the selected routes
    :param num_route: the number of the selected routes. If the num_route is None, then all the route id will be selected
    :param direction_id: the direction id can be 0 or 1
    :return: the list of the trip_id
    """
    # Read the GTFS data
    # data source: MTA, state island, Jan, 4, 2016
    trips = pd.read_csv(path + 'data/GTFS/gtfs/trips.txt')
    route_stop_dist = pd.read_csv('route_stop_dist.csv')

    # select a specific route and the corresponding trips
    route_list = list(route_stop_dist.route_id)
    non_dup_route_list = sorted(list(set(route_list)))
    if num_route is None:
        select_routes = non_dup_route_list
    else:
        select_routes = non_dup_route_list[:num_route]
    selected_trips_var = []
    for route in select_routes:
        selected_trips_var += list(trips[(trips.route_id == route) & (trips.direction_id == direction_id)].trip_id)
    return selected_trips_var

def filter_history_data(date_start, date_end, selected_trips_var):
    # type: (object, object, object) -> object
    """
    Filtering the historical data to remove the unselected trips
    :rtype: object
    :param date_start: start date for historical date, int, yyyymmdd, ex: 20160109
    :param date_end: end date for historical date. Similar to date_start. The date_start and the date_end are included.
    :param selected_trips_var: the list of the trip_id for the selected routes
    :return: dataframe for the filtered historical data
    """
    # List the historical file
    file_list_var = os.listdir(path + 'data/history/')
    history_list = []
    print "filtering historical data"
    for filename in file_list_var:
        if not filename.endswith('.csv'):
            continue
        if filename[9:17] < str(date_start) or filename[9:17] > str(date_end):
            continue
        print filename
        ptr_history = pd.read_csv(path + 'data/history/' + filename)
        tmp_history = ptr_history[
            (ptr_history.trip_id.isin(selected_trips_var)) & (ptr_history.dist_along_route != '\N') & (
                ptr_history.dist_along_route != 0)]
        history_list.append(tmp_history)
    result = pd.concat(history_list)
    return result

def add_weather_info(weather, date_var):
    """
    add the weather information from the file: weather.csv
    The weather are expressed as below:
    0: sunny
    1: rainy
    2: snowy
    :param weather: the dataframe for weather.csv file
    :param date_var: the date for querying the weather
    :return: return the weather value today.
    """
    ptr_weather = weather[weather.date == date_var]
    if ptr_weather.iloc[0].snow == 1:
        weather_today = 2
    elif ptr_weather.iloc[0].rain == 1:
        weather_today = 1
    else:
        weather_today = 0
    return weather_today

def generate_original_segment(full_history_var, weather, stop_times_var):
    """
    Generate the original segment data
    Algorithm:
    Split the full historical data according to the service date, trip_id with groupby function
    For name, item in splitted historical dataset:
        service date, trip_id = name
        Split the item according to the vehicle_id, keep the data with the larget length of list for the vehicle_id
        calcualte the travel duration within the segement of this segment df and save the result into list
    concatenate the list
    :param full_history_var: the historical data after filtering
    :param weather: the dataframe for the weather information
    :param stop_times_var: the dataframe from stop_times.txt
    :return: dataframe for the original segment
    """
    grouped = list(full_history_var.groupby(['service_date', 'trip_id']))
    print len(grouped)
    result_list = []
    for index in range(len(grouped)):
        name, single_history = grouped[index]
        if index % 1500 == 0:
            print index
        service_date, trip_id = name
        if service_date <= 20160103:
            continue
        grouped_vehicle_id = list(single_history.groupby(['vehicle_id']))
        majority_length = -1
        majority_vehicle = -1
        majority_history = single_history
        for vehicle_id, item in grouped_vehicle_id:
            if len(item) > majority_length:
                majority_length = len(item)
                majority_history = item
                majority_vehicle = vehicle_id
        stop_sequence = [str(item) for item in list(stop_times_var[stop_times_var.trip_id == trip_id].stop_id)]
        current_segment_df = generate_original_segment_single_history(majority_history, stop_sequence)
        if current_segment_df is None:
            continue
        current_weather = add_weather_info(weather, service_date)
        current_segment_df['weather'] = current_weather
        day_of_week = datetime.strptime(str(service_date), '%Y%m%d').weekday()
        current_segment_df['service_date'] = service_date
        current_segment_df['day_of_week'] = day_of_week
        current_segment_df['trip_id'] = trip_id
        current_segment_df['vehicle_id'] = majority_vehicle
        result_list.append(current_segment_df)
    if result_list != []:
        result = pd.concat(result_list)
    else:
        return None
    return result

"""
Algorithm:
Filter the historical data with the stop sequence here
for i = 1, len(history) - 1:
    use prev and the next to mark the record:
        prev = history[i - 1]
        next = history[i]
    calculate the distance for prev and next respectively:
        prev_distance = prev.dist_along_route - prev.dist_from_stop
        next_distance = next.dist_along_route - next.dist_from_stop
    if prev_distance == next_distance or prev_distance = 0, continue to next row
    calcualte the time duration between the two spot:
        prev_time = datetime.strptime(prev.timestamp, '%Y-%m-%dT%H:%M:%SZ')
        next_time = ...
        travel_duration = next_time - prev_time

"""

def generate_original_segment_single_history(history, stop_sequence):
    """
    Calculate the travel duration for a single historical data
    Algorithm:
    Filter the historical data with the stop sequence here
    arrival_time_list = []
    for i = 1, len(history) - 1:
        use prev and the next to mark the record:
            prev = history[i - 1]
            next = history[i]
        calculate the distance for prev and next respectively:
            prev_distance = prev.dist_along_route - prev.dist_from_stop
            next_distance = next.dist_along_route - next.dist_from_stop
        if prev_distance == next_distance or prev_distance = 0, continue to next row
        distance_ratio = prev.dist_from_stop / (next_distance - prev_distance)
        calcualte the time duration between the two spot:
            prev_time = datetime.strptime(prev.timestamp, '%Y-%m-%dT%H:%M:%SZ')
            next_time = ...
            travel_duration = next_time - prev_time
        current_arrival_duration = travel_duration * distance_ratio
        current_arrival_time = current_arrival_duration + prev_time
        arrival_time_list.append((prev.next_stop_id, current_arrival_time))
    result = pd.Dataframe
    for i in range(1, len(arrival_time_list)):
        prev = arrival_time_list[i - 1]
        next = arrival_time_list[i]
        segment_start, segment_end obtained
        travel_duration = next[1] - prev[1]
        timestamp = prev[1]
        service_date = history[0].service_date
        ...
        save the record to result

    :param history: single historical data
    :param stop_sequence: stop sequence for the corresponding trip id
    :return: the dataframe of the origianl segment dataset
    """
    history = history[history.next_stop_id.isin(stop_sequence)]
    if len(history) < 3:
        return None
    arrival_time_list = []
    i = 1
    while i < len(history):
        prev_record = history.iloc[i - 1]
        next_record = history.iloc[i]
        while i < len(history) and prev_record.next_stop_id == next_record.next_stop_id:
            i += 1
            if i == len(history):
                break
            next_record = history.iloc[i]
        if i == len(history):
            break
        # calculate the distance for prev and next respectively
        prev_distance = float(prev_record.dist_along_route) - float(prev_record.dist_from_stop)
        next_distance = float(next_record.dist_along_route) - float(next_record.dist_from_stop)
        if prev_distance == next_distance or prev_distance == 0:
            i += 1
            continue
        else:
            distance_ratio = float(prev_record.dist_from_stop) / (next_distance - prev_distance)
        # calcualte the time duration between the two spot
        prev_time = datetime.strptime(prev_record.timestamp, '%Y-%m-%dT%H:%M:%SZ')
        next_time = datetime.strptime(next_record.timestamp, '%Y-%m-%dT%H:%M:%SZ')
        travel_duration = next_time - prev_time
        travel_duration = travel_duration.total_seconds()
        # add it into the arrival time list
        current_arrival_duration = travel_duration * distance_ratio
        current_arrival_time = timedelta(0, current_arrival_duration) + prev_time
        arrival_time_list.append((prev_record.next_stop_id, current_arrival_time))
        i += 1
    result = pd.DataFrame(columns=['segment_start', 'segment_end', 'timestamp', 'travel_duration'])
    for i in range(1, len(arrival_time_list)):
        prev_record = arrival_time_list[i - 1]
        next_record = arrival_time_list[i]
        segment_start, segment_end = prev_record[0], next_record[0]
        timestamp = prev_record[1]
        travel_duration = next_record[1] - prev_record[1]
        travel_duration = travel_duration.total_seconds()
        result.loc[len(result)] = [segment_start, segment_end, timestamp, travel_duration]
    return result

file_list = os.listdir('./')
# export the segment data
print "export original_segment.csv file"
selected_trips = select_trip_list()
weather_df = pd.read_csv('weather.csv')
full_history = filter_history_data(20160104, 20160123, selected_trips)
stop_times = pd.read_csv(path + 'data/GTFS/gtfs/stop_times.txt')
segment_df = generate_original_segment(full_history, weather_df, stop_times)
segment_df.to_csv('original_segment.csv')
print "complete exporting the original_segement.csv file"
<commit_msg>fix the bug for original_segment.csv file<commit_after># import modules
import os
import numpy as np
import requests
import csv
import random
import pandas as pd
from datetime import datetime, timedelta, date
from dateutil.rrule import rrule, DAILY

# set the path
path = '../'

#################################################################################################################
#                                        segment.csv                                                            #
#################################################################################################################

def select_trip_list(num_route=None, direction_id=0):
    """
    Generate the list of the trip id for the selected routes
    :param num_route: the number of the selected routes. If the num_route is None, then all the route id will be selected
    :param direction_id: the direction id can be 0 or 1
    :return: the list of the trip_id
    """
    # Read the GTFS data
    # data source: MTA, state island, Jan, 4, 2016
    trips = pd.read_csv(path + 'data/GTFS/gtfs/trips.txt')
    route_stop_dist = pd.read_csv('route_stop_dist.csv')

    # select a specific route and the corresponding trips
    route_list = list(route_stop_dist.route_id)
    non_dup_route_list = sorted(list(set(route_list)))
    if num_route is None:
        select_routes = non_dup_route_list
    else:
        select_routes = non_dup_route_list[:num_route]
    selected_trips_var = []
    for route in select_routes:
        selected_trips_var += list(trips[(trips.route_id == route) & (trips.direction_id == direction_id)].trip_id)
    return selected_trips_var

def filter_history_data(date_start, date_end, selected_trips_var):
    # type: (object, object, object) -> object
    """
    Filtering the historical data to remove the unselected trips
    :rtype: object
    :param date_start: start date for historical date, int, yyyymmdd, ex: 20160109
    :param date_end: end date for historical date. Similar to date_start. The date_start and the date_end are included.
    :param selected_trips_var: the list of the trip_id for the selected routes
    :return: dataframe for the filtered historical data
    """
    # List the historical file
    file_list_var = os.listdir(path + 'data/history/')
    history_list = []
    print "filtering historical data"
    for filename in file_list_var:
        if not filename.endswith('.csv'):
            continue
        if filename[9:17] < str(date_start) or filename[9:17] > str(date_end):
            continue
        print filename
        ptr_history = pd.read_csv(path + 'data/history/' + filename)
        tmp_history = ptr_history[
            (ptr_history.trip_id.isin(selected_trips_var)) & (ptr_history.dist_along_route != '\N') & (
                ptr_history.dist_along_route != 0) & (ptr_history.progress == 0)]
        history_list.append(tmp_history)
    result = pd.concat(history_list)
    return result

def add_weather_info(weather, date_var):
    """
    add the weather information from the file: weather.csv
    The weather are expressed as below:
    0: sunny
    1: rainy
    2: snowy
    :param weather: the dataframe for weather.csv file
    :param date_var: the date for querying the weather
    :return: return the weather value today.
    """
    ptr_weather = weather[weather.date == date_var]
    if ptr_weather.iloc[0].snow == 1:
        weather_today = 2
    elif ptr_weather.iloc[0].rain == 1:
        weather_today = 1
    else:
        weather_today = 0
    return weather_today

def generate_original_segment(full_history_var, weather, stop_times_var):
    """
    Generate the original segment data
    Algorithm:
    Split the full historical data according to the service date, trip_id with groupby function
    For name, item in splitted historical dataset:
        service date, trip_id = name
        Split the item according to the vehicle_id, keep the data with the larget length of list for the vehicle_id
        calcualte the travel duration within the segement of this segment df and save the result into list
    concatenate the list
    :param full_history_var: the historical data after filtering
    :param weather: the dataframe for the weather information
    :param stop_times_var: the dataframe from stop_times.txt
    :return: dataframe for the original segment
    """
    test_trip = 'CA_A6-Weekday-SDon-039000_MISC_434'
    full_history_var = full_history_var[full_history_var.service_date == 20160104]
    full_history_var = full_history_var[full_history_var.trip_id == test_trip]
    grouped = list(full_history_var.groupby(['service_date', 'trip_id']))
    print len(grouped)
    result_list = []
    for index in range(len(grouped)):
        name, single_history = grouped[index]
        if index % 150 == 0:
            print index
        service_date, trip_id = name
        if service_date <= 20160103:
            continue
        grouped_vehicle_id = list(single_history.groupby(['vehicle_id']))
        majority_length = -1
        majority_vehicle = -1
        majority_history = single_history
        for vehicle_id, item in grouped_vehicle_id:
            if len(item) > majority_length:
                majority_length = len(item)
                majority_history = item
                majority_vehicle = vehicle_id
        stop_sequence = [str(item) for item in list(stop_times_var[stop_times_var.trip_id == trip_id].stop_id)]
        current_segment_df = generate_original_segment_single_history(majority_history, stop_sequence)
        if current_segment_df is None:
            continue
        current_weather = add_weather_info(weather, service_date)
        current_segment_df['weather'] = current_weather
        day_of_week = datetime.strptime(str(service_date), '%Y%m%d').weekday()
        current_segment_df['service_date'] = service_date
        current_segment_df['day_of_week'] = day_of_week
        current_segment_df['trip_id'] = trip_id
        current_segment_df['vehicle_id'] = majority_vehicle
        result_list.append(current_segment_df)
    if result_list != []:
        result = pd.concat(result_list)
    else:
        return None
    return result

"""
Algorithm:
Filter the historical data with the stop sequence here
for i = 1, len(history) - 1:
    use prev and the next to mark the record:
        prev = history[i - 1]
        next = history[i]
    calculate the distance for prev and next respectively:
        prev_distance = prev.dist_along_route - prev.dist_from_stop
        next_distance = next.dist_along_route - next.dist_from_stop
    if prev_distance == next_distance or prev_distance = 0, continue to next row
    calcualte the time duration between the two spot:
        prev_time = datetime.strptime(prev.timestamp, '%Y-%m-%dT%H:%M:%SZ')
        next_time = ...
        travel_duration = next_time - prev_time

"""

def generate_original_segment_single_history(history, stop_sequence):
    """
    Calculate the travel duration for a single historical data
    Algorithm:
    Filter the historical data with the stop sequence here
    arrival_time_list = []
    for i = 1, len(history) - 1:
        use prev and the next to mark the record:
            prev = history[i - 1]
            next = history[i]
        calculate the distance for prev and next respectively:
            prev_distance = prev.dist_along_route - prev.dist_from_stop
            next_distance = next.dist_along_route - next.dist_from_stop
        if prev_distance == next_distance or prev_distance = 0, continue to next row
        distance_ratio = prev.dist_from_stop / (next_distance - prev_distance)
        calcualte the time duration between the two spot:
            prev_time = datetime.strptime(prev.timestamp, '%Y-%m-%dT%H:%M:%SZ')
            next_time = ...
            travel_duration = next_time - prev_time
        current_arrival_duration = travel_duration * distance_ratio
        current_arrival_time = current_arrival_duration + prev_time
        arrival_time_list.append((prev.next_stop_id, current_arrival_time))
    result = pd.Dataframe
    for i in range(1, len(arrival_time_list)):
        prev = arrival_time_list[i - 1]
        next = arrival_time_list[i]
        segment_start, segment_end obtained
        travel_duration = next[1] - prev[1]
        timestamp = prev[1]
        service_date = history[0].service_date
        ...
        save the record to result

    :param history: single historical data
    :param stop_sequence: stop sequence for the corresponding trip id
    :return: the dataframe of the origianl segment dataset
    """
    history = history[history.next_stop_id.isin(stop_sequence)]
    if len(history) < 3:
        return None
    arrival_time_list = []
    i = 1
    while i < len(history):
        prev_record = history.iloc[i - 1]
        next_record = history.iloc[i]
        while i < len(history) and stop_sequence.index(prev_record.next_stop_id) >= stop_sequence.index(next_record.next_stop_id):
            i += 1
            if i == len(history):
                break
            if stop_sequence.index(prev_record.next_stop_id) == stop_sequence.index(next_record.next_stop_id):
                prev_record = next_record
            next_record = history.iloc[i]
        if i == len(history):
            break
        # calculate the distance for prev and next respectively
        prev_distance = float(prev_record.dist_along_route) - float(prev_record.dist_from_stop)
        next_distance = float(next_record.dist_along_route) - float(next_record.dist_from_stop)
        if prev_distance == next_distance or prev_distance == 0:
            i += 1
            continue
        else:
            distance_ratio = float(prev_record.dist_from_stop) / (next_distance - prev_distance)
        # calcualte the time duration between the two spot
        prev_time = datetime.strptime(prev_record.timestamp, '%Y-%m-%dT%H:%M:%SZ')
        next_time = datetime.strptime(next_record.timestamp, '%Y-%m-%dT%H:%M:%SZ')
        travel_duration = next_time - prev_time
        travel_duration = travel_duration.total_seconds()
        # add it into the arrival time list
        current_arrival_duration = travel_duration * distance_ratio
        current_arrival_time = timedelta(0, current_arrival_duration) + prev_time
        arrival_time_list.append((prev_record.next_stop_id, current_arrival_time))
        i += 1
    result = pd.DataFrame(columns=['segment_start', 'segment_end', 'timestamp', 'travel_duration'])
    for i in range(1, len(arrival_time_list)):
        prev_record = arrival_time_list[i - 1]
        next_record = arrival_time_list[i]
        segment_start, segment_end = prev_record[0], next_record[0]
        timestamp = prev_record[1]
        travel_duration = next_record[1] - prev_record[1]
        travel_duration = travel_duration.total_seconds()
        result.loc[len(result)] = [segment_start, segment_end, timestamp, travel_duration]
    return result

file_list = os.listdir('./')
# export the segment data
print "export original_segment.csv file"
selected_trips = select_trip_list()
weather_df = pd.read_csv('weather.csv')
full_history = filter_history_data(20160104, 20160123, selected_trips)
stop_times = pd.read_csv(path + 'data/GTFS/gtfs/stop_times.txt')
segment_df = generate_original_segment(full_history, weather_df, stop_times)
segment_df.to_csv('original_segment.csv')
print "complete exporting the original_segement.csv file"
<|endoftext|>


<commit_before>import operator
import logging

from django.db.models.query import Q
from django.forms.formsets import formset_factory
from django.core.urlresolvers import reverse
from django.utils.encoding import StrAndUnicode
from django.template import RequestContext
from django.template.loader import render_to_string
from django.db.models.sql.constants import LOOKUP_SEP

from appsearch.registry import search, SearchRegistry
from appsearch.forms import ModelSelectionForm, ConstraintForm, ConstraintFormset

log = logging.getLogger(__name__)

class Searcher(StrAndUnicode):
    """ Template helper, wrapping all the necessary components to render an appsearch page. """

    # Methods and fields not meant to be accessed from the template should start with an underscore
    # to let the template variable name resolution block access.

    # Customizable classes
    model_selection_form_class = ModelSelectionForm
    constraint_form_class = None
    constraint_formset_class = None

    # Instance data
    model_selection_form = None
    constraint_formset = None

    field_data_url = None
    operator_data_url = None

    results = None

    # Processing callback hooks
    _display_fields_callback = None
    _build_queryset_callback = None
    _process_results_callback = None

    ## Fallback items normally provided by the view
    context_object_name = 'search'

    # Default templates
    form_template_name = "appsearch/default_form.html"
    search_form_template_name = "appsearch/search_form.html"
    results_list_template_name = "appsearch/results_list.html"

    def __init__(self, request, url=None, querydict=None, registry=search, **kwargs):
        self.kwargs = kwargs
        self.request = request
        self.url = url or request.path
        self._determine_urls(kwargs)

        self._forms_ready = False
        self._set_up_forms(querydict or request.GET, registry)

        # Fallback items
        self.context_object_name = kwargs.get('context_object_name', self.context_object_name)
        self.form_template_name = kwargs.get('form_template_name', self.form_template_name)
        self.search_form_template_name = kwargs.get('search_form_template_name', self.search_form_template_name)
        self.results_list_template_name = kwargs.get('results_list_template_name', self.results_list_template_name)

        self._display_fields_callback = kwargs.get('display_fields_callback')
        self._build_queryset_callback = kwargs.get('build_queryset_callback')
        self._process_results_callback = kwargs.get('process_results_callback')

    # Rendering methods
    def __unicode__(self):
        return render_to_string(self.form_template_name, RequestContext(self.request, {
            self.context_object_name: self,
        }))

    def render_search_form(self):
        """ Renders only the template at ``search_form_template_name`` """
        return render_to_string(self.search_form_template_name, RequestContext(self.request, {
            self.context_object_name: self,
        }))

    def render_results_list(self):
        """ Renders only the template at ``results_list_template_name`` """
        return render_to_string(self.results_list_template_name, RequestContext(self.request, {
            self.context_object_name: self,
        }))

    def get_model_selection_form_class(self):
        """ Returns ``self.model_selection_form_class`` """
        return self.model_selection_form_class

    def get_constraint_form_class(self):
        """ Returns ``self.constraint_form_class`` """
        return self.constraint_form_class

    def get_constraint_formset_class(self):
        """ Returns ``self.constraint_formset_class`` """
        return self.constraint_formset_class

    @property
    def ready(self):
        """ Indicates if a search has been executed by the constructed state. """

        if self._forms_ready:
            self.model_config = self.model_selection_form.get_selected_configuration()
            self.model = self.model_config.model

        return self._forms_ready

    def _set_up_forms(self, querydict, registry, permission=None):
        ModelSelectionFormClass = self.get_model_selection_form_class()
        ConstraintFormClass = self.get_constraint_form_class()
        ConstraintFormsetClass = self.get_constraint_formset_class()

        ConstraintFormsetClass = formset_factory(ConstraintFormClass, formset=ConstraintFormsetClass)

        self.model_selection_form = ModelSelectionFormClass(registry, self.request.user, querydict,
                permission=permission)

        if self.model_selection_form.is_valid():
            model_configuration = self.model_selection_form.get_selected_configuration()
            self.constraint_formset = ConstraintFormsetClass(model_configuration, querydict)
            if self.constraint_formset.is_valid():
                self._forms_ready = True
        else:
            self.model_selection_form = ModelSelectionFormClass(registry, self.request.user,
                    permission=permission)
            self.constraint_formset = ConstraintFormsetClass(configuration=None)

    def _determine_urls(self, kwargs):
        # If a URL is not customized, this namespace will be used to search out the default URL
        url_namespace = kwargs.get('url_namespace')

        # Check custom data URLs
        field_url = kwargs.get('field_url')
        operator_url = kwargs.get('operator_url')

        if field_url is None:
            url_name = 'appsearch:constraint-fields'
            if url_namespace is not None:
                url_name = url_namespace + ':' + url_name
            field_url = reverse(url_name, kwargs=kwargs.get('field_url_kwargs', {}))
        if operator_url is None:
            url_name = 'appsearch:constraint-operators'
            if url_namespace is not None:
                url_name = url_namespace + ':' + url_name
            operator_url = reverse(url_name, kwargs=kwargs.get('operator_url_kwargs', {}))

        self.field_data_url = field_url
        self.operator_data_url = operator_url

    def _perform_search(self):
        """
        Generates the query using the validated constraint formset.  Also compiles a natural
        language string (``self.natural_string``) in the format of
        "N [Model]s with [field] [operator] '[term]'".

        """

        natural_string = []

        # 2-tuples of an operator and Q instance, sent through reduce() after it's built
        query_list = []

        for i, constraint_form in enumerate(self.constraint_formset):
            type_operator = constraint_form.cleaned_data['type']
            field_list = constraint_form.cleaned_data['field']
            constraint_operator = constraint_form.cleaned_data['operator']
            term = constraint_form.cleaned_data['term']
            end_term = constraint_form.cleaned_data['end_term']

            verbose_name = self.model_config._fields[field_list]

            # Build this field's query.
            # Search fields bound together in a tuple are considered OR conditions for a single
            # virtual field name.
            query = None
            for field in field_list:
                value = term

                # Prep an inverted lookup
                negative = constraint_operator.startswith('!')
                if negative:
                    constraint_operator = constraint_operator[1:]

                if constraint_operator == "isnull":
                    value = not negative
                    negative = False

                # Bake the queryset language string
                field_query = LOOKUP_SEP.join((field, constraint_operator))

                q = Q(**{
                    field_query: value,
                })

                log.debug("Querying %s [%d]: %s=%r", self.model.__name__, i, field_query, value)

                # Negate if necessary
                if negative:
                    q = ~q

                if query is None:
                    query = q
                else:
                    query |= q

            query_list.append((type_operator, query))

            # Do some natural processing
            if isinstance(value, (tuple, list)):
                value = ' - '.join(map(unicode, value))
            else:
                value = str(value)
            bits = [verbose_name, constraint_form['operator'].value(), value]
            if i != 0: # Skip the leading "and" on the first constraint
                bits.insert(0, constraint_form['type'].value())
            natural_string.append(bits)

        # The first query's "type" should be ignored for the sake of the reduce line below.
        query = reduce(lambda q1, (op, q2): op(q1, q2), query_list[1:], query_list[0][1])

        queryset = self.build_queryset(self.model, query)
        data_rows = self.process_results(queryset)

        self.results = {
            'count': len(queryset),
            'list': data_rows,
            'fields': self._get_display_fields(),
            'natural_string': "where " + ', '.join(map(' '.join, natural_string)),
        }

    def _get_display_fields(self, model, config):
        if self._display_fields_callback:
            return self._display_fields_callback(self, model, config)
        return config.get_display_fields()

    def get_select_related_fields(self, model, config):
        """ Returns a list of queryset language names to pass into ``.select_related()`` """
        display_fields = self._get_display_fields(model, config)
        related_names = set()
        for field_info in display_fields:
            if isinstance(field_info, (tuple, list)):
                _, field_name = field_info
            else:
                field_name = field_info

            related_path = field_name.rsplit(LOOKUP_SEP, 1)[0]

            if LOOKUP_SEP in related_path:
                related_names.add(related_path)

        return related_names

    def build_queryset(self, model, query, queryset=None):
        """
        Returns the queryset using ``query``.

        Default behavior inspects the display fields for any related items and requests their
        selection and adds ``.distinct()``.

        If ``base_queryset`` is provided, it will be used as the starting point for applying the
        ``query`` filter.  This is useful for subclasses that want to change the default manager
        being accessed for the initial queryset.  Passing up a ``queryset`` via super() will
        accomplish this, returning a fully built queryset with minimum hassle.

        If ``build_queryset`` callback was defined on the originating view, it will be called
        after the queryset is built and will be sent the searcher instance, model, config, initial
        ``Q`` query, and the derived queryset.  The callback should return the queryset in its final
        state for execution.

        """

        related_names = self.get_select_related_fields(model, self.model_config)

        if queryset is None:
            queryset = self.model_config.get_queryset(self.request.user)

        queryset = queryset.filter(query).select_related(*related_names).distinct()

        if self._build_queryset_callback:
            queryset = self._build_queryset_callback(self, model, self.model_config, query,
                    queryset)

        return queryset

    def process_results(self, queryset):
        """
        Should return a sequence of n-tuples, where ``n`` is the number of columns to be shown in
        the table.

        """

        if self._process_results_callback:
            return self._process_results_callback(self, self.model, self.model_config, queryset)

        return [self.model_config.get_object_data(obj) for obj in queryset]
<commit_msg>Fix error in default attributes<commit_after>import operator
import logging

from django.db.models.query import Q
from django.forms.formsets import formset_factory
from django.core.urlresolvers import reverse
from django.utils.encoding import StrAndUnicode
from django.template import RequestContext
from django.template.loader import render_to_string
from django.db.models.sql.constants import LOOKUP_SEP

from appsearch.registry import search, SearchRegistry
from appsearch.forms import ModelSelectionForm, ConstraintForm, ConstraintFormset

log = logging.getLogger(__name__)

class Searcher(StrAndUnicode):
    """ Template helper, wrapping all the necessary components to render an appsearch page. """

    # Methods and fields not meant to be accessed from the template should start with an underscore
    # to let the template variable name resolution block access.

    # Customizable classes
    model_selection_form_class = ModelSelectionForm
    constraint_form_class = ConstraintForm
    constraint_formset_class = ConstraintFormset

    # Instance data
    model_selection_form = None
    constraint_formset = None

    field_data_url = None
    operator_data_url = None

    results = None

    # Processing callback hooks
    _display_fields_callback = None
    _build_queryset_callback = None
    _process_results_callback = None

    ## Fallback items normally provided by the view
    context_object_name = 'search'

    # Default templates
    form_template_name = "appsearch/default_form.html"
    search_form_template_name = "appsearch/search_form.html"
    results_list_template_name = "appsearch/results_list.html"

    def __init__(self, request, url=None, querydict=None, registry=search, **kwargs):
        self.kwargs = kwargs
        self.request = request
        self.url = url or request.path
        self._determine_urls(kwargs)

        self._forms_ready = False
        self._set_up_forms(querydict or request.GET, registry)

        # Fallback items
        self.context_object_name = kwargs.get('context_object_name', self.context_object_name)
        self.form_template_name = kwargs.get('form_template_name', self.form_template_name)
        self.search_form_template_name = kwargs.get('search_form_template_name', self.search_form_template_name)
        self.results_list_template_name = kwargs.get('results_list_template_name', self.results_list_template_name)

        self._display_fields_callback = kwargs.get('display_fields_callback')
        self._build_queryset_callback = kwargs.get('build_queryset_callback')
        self._process_results_callback = kwargs.get('process_results_callback')

    # Rendering methods
    def __unicode__(self):
        return render_to_string(self.form_template_name, RequestContext(self.request, {
            self.context_object_name: self,
        }))

    def render_search_form(self):
        """ Renders only the template at ``search_form_template_name`` """
        return render_to_string(self.search_form_template_name, RequestContext(self.request, {
            self.context_object_name: self,
        }))

    def render_results_list(self):
        """ Renders only the template at ``results_list_template_name`` """
        return render_to_string(self.results_list_template_name, RequestContext(self.request, {
            self.context_object_name: self,
        }))

    def get_model_selection_form_class(self):
        """ Returns ``self.model_selection_form_class`` """
        return self.model_selection_form_class

    def get_constraint_form_class(self):
        """ Returns ``self.constraint_form_class`` """
        return self.constraint_form_class

    def get_constraint_formset_class(self):
        """ Returns ``self.constraint_formset_class`` """
        return self.constraint_formset_class

    @property
    def ready(self):
        """ Indicates if a search has been executed by the constructed state. """

        if self._forms_ready:
            self.model_config = self.model_selection_form.get_selected_configuration()
            self.model = self.model_config.model

        return self._forms_ready

    def _set_up_forms(self, querydict, registry, permission=None):
        ModelSelectionFormClass = self.get_model_selection_form_class()
        ConstraintFormClass = self.get_constraint_form_class()
        ConstraintFormsetClass = self.get_constraint_formset_class()

        ConstraintFormsetClass = formset_factory(ConstraintFormClass, formset=ConstraintFormsetClass)

        self.model_selection_form = ModelSelectionFormClass(registry, self.request.user, querydict,
                permission=permission)

        if self.model_selection_form.is_valid():
            model_configuration = self.model_selection_form.get_selected_configuration()
            self.constraint_formset = ConstraintFormsetClass(model_configuration, querydict)
            if self.constraint_formset.is_valid():
                self._forms_ready = True
        else:
            self.model_selection_form = ModelSelectionFormClass(registry, self.request.user,
                    permission=permission)
            self.constraint_formset = ConstraintFormsetClass(configuration=None)

    def _determine_urls(self, kwargs):
        # If a URL is not customized, this namespace will be used to search out the default URL
        url_namespace = kwargs.get('url_namespace')

        # Check custom data URLs
        field_url = kwargs.get('field_url')
        operator_url = kwargs.get('operator_url')

        if field_url is None:
            url_name = 'appsearch:constraint-fields'
            if url_namespace is not None:
                url_name = url_namespace + ':' + url_name
            field_url = reverse(url_name, kwargs=kwargs.get('field_url_kwargs', {}))
        if operator_url is None:
            url_name = 'appsearch:constraint-operators'
            if url_namespace is not None:
                url_name = url_namespace + ':' + url_name
            operator_url = reverse(url_name, kwargs=kwargs.get('operator_url_kwargs', {}))

        self.field_data_url = field_url
        self.operator_data_url = operator_url

    def _perform_search(self):
        """
        Generates the query using the validated constraint formset.  Also compiles a natural
        language string (``self.natural_string``) in the format of
        "N [Model]s with [field] [operator] '[term]'".

        """

        natural_string = []

        # 2-tuples of an operator and Q instance, sent through reduce() after it's built
        query_list = []

        for i, constraint_form in enumerate(self.constraint_formset):
            type_operator = constraint_form.cleaned_data['type']
            field_list = constraint_form.cleaned_data['field']
            constraint_operator = constraint_form.cleaned_data['operator']
            term = constraint_form.cleaned_data['term']
            end_term = constraint_form.cleaned_data['end_term']

            verbose_name = self.model_config._fields[field_list]

            # Build this field's query.
            # Search fields bound together in a tuple are considered OR conditions for a single
            # virtual field name.
            query = None
            for field in field_list:
                value = term

                # Prep an inverted lookup
                negative = constraint_operator.startswith('!')
                if negative:
                    constraint_operator = constraint_operator[1:]

                if constraint_operator == "isnull":
                    value = not negative
                    negative = False

                # Bake the queryset language string
                field_query = LOOKUP_SEP.join((field, constraint_operator))

                q = Q(**{
                    field_query: value,
                })

                log.debug("Querying %s [%d]: %s=%r", self.model.__name__, i, field_query, value)

                # Negate if necessary
                if negative:
                    q = ~q

                if query is None:
                    query = q
                else:
                    query |= q

            query_list.append((type_operator, query))

            # Do some natural processing
            if isinstance(value, (tuple, list)):
                value = ' - '.join(map(unicode, value))
            else:
                value = str(value)
            bits = [verbose_name, constraint_form['operator'].value(), value]
            if i != 0: # Skip the leading "and" on the first constraint
                bits.insert(0, constraint_form['type'].value())
            natural_string.append(bits)

        # The first query's "type" should be ignored for the sake of the reduce line below.
        query = reduce(lambda q1, (op, q2): op(q1, q2), query_list[1:], query_list[0][1])

        queryset = self.build_queryset(self.model, query)
        data_rows = self.process_results(queryset)

        self.results = {
            'count': len(queryset),
            'list': data_rows,
            'fields': self._get_display_fields(),
            'natural_string': "where " + ', '.join(map(' '.join, natural_string)),
        }

    def _get_display_fields(self, model, config):
        if self._display_fields_callback:
            return self._display_fields_callback(self, model, config)
        return config.get_display_fields()

    def get_select_related_fields(self, model, config):
        """ Returns a list of queryset language names to pass into ``.select_related()`` """
        display_fields = self._get_display_fields(model, config)
        related_names = set()
        for field_info in display_fields:
            if isinstance(field_info, (tuple, list)):
                _, field_name = field_info
            else:
                field_name = field_info

            related_path = field_name.rsplit(LOOKUP_SEP, 1)[0]

            if LOOKUP_SEP in related_path:
                related_names.add(related_path)

        return related_names

    def build_queryset(self, model, query, queryset=None):
        """
        Returns the queryset using ``query``.

        Default behavior inspects the display fields for any related items and requests their
        selection and adds ``.distinct()``.

        If ``base_queryset`` is provided, it will be used as the starting point for applying the
        ``query`` filter.  This is useful for subclasses that want to change the default manager
        being accessed for the initial queryset.  Passing up a ``queryset`` via super() will
        accomplish this, returning a fully built queryset with minimum hassle.

        If ``build_queryset`` callback was defined on the originating view, it will be called
        after the queryset is built and will be sent the searcher instance, model, config, initial
        ``Q`` query, and the derived queryset.  The callback should return the queryset in its final
        state for execution.

        """

        related_names = self.get_select_related_fields(model, self.model_config)

        if queryset is None:
            queryset = self.model_config.get_queryset(self.request.user)

        queryset = queryset.filter(query).select_related(*related_names).distinct()

        if self._build_queryset_callback:
            queryset = self._build_queryset_callback(self, model, self.model_config, query,
                    queryset)

        return queryset

    def process_results(self, queryset):
        """
        Should return a sequence of n-tuples, where ``n`` is the number of columns to be shown in
        the table.

        """

        if self._process_results_callback:
            return self._process_results_callback(self, self.model, self.model_config, queryset)

        return [self.model_config.get_object_data(obj) for obj in queryset]
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
from django.conf import settings
from django.conf.urls import (
    include,
    patterns,
    url,
)
from django.conf.urls.static import static
from django.contrib import admin
from django.contrib.sitemaps import GenericSitemap

from block.models import Page

admin.autodiscover()

info_dict = {
  'queryset': Page.objects.pages(),
  'date_field': 'modified',
}

sitemaps = {
    'block': GenericSitemap(info_dict, priority=0.5, changefreq='monthly'),
}

urlpatterns = patterns(
    '',
    url(regex=r'^sitemap\.xml$',
        view='django.contrib.sitemaps.views.sitemap',
        kwargs={'sitemaps': sitemaps},
    ),
    url(regex=r'^',
        view=include('login.urls')
        ),
    url(regex=r'^admin/',
        view=include(admin.site.urls)
        ),
    url(regex=r'^block/',
        view=include('block.urls')
        ),
    url(regex=r'^compose/',
        view=include('compose.urls.compose')
        ),
    url(regex=r'^dash/',
        view=include('dash.urls')
        ),
    # this url include should come last
    url(regex=r'^',
        view=include('compose.urls.cms')
        ),
)

urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
#   ^ helper function to return a URL pattern for serving files in debug mode.
# https://docs.djangoproject.com/en/1.5/howto/static-files/#serving-files-uploaded-by-a-user
<commit_msg>Change page_article_cards to page_article<commit_after># -*- encoding: utf-8 -*-
from django.conf import settings
from django.conf.urls import (
    include,
    patterns,
    url,
)
from django.conf.urls.static import static
from django.contrib import admin
from django.contrib.sitemaps import GenericSitemap

from block.models import Page

admin.autodiscover()

info_dict = {
  'queryset': Page.objects.pages(),
  'date_field': 'modified',
}

sitemaps = {
    'block': GenericSitemap(info_dict, priority=0.5, changefreq='monthly'),
}

urlpatterns = patterns(
    '',
    url(regex=r'^sitemap\.xml$',
        view='django.contrib.sitemaps.views.sitemap',
        kwargs={'sitemaps': sitemaps},
    ),
    url(regex=r'^',
        view=include('login.urls')
        ),
    url(regex=r'^admin/',
        view=include(admin.site.urls)
        ),
    url(regex=r'^block/',
        view=include('block.urls.block')
        ),
    url(regex=r'^compose/',
        view=include('compose.urls')
        ),
    url(regex=r'^dash/',
        view=include('dash.urls')
        ),
    # this url include should come last
    url(regex=r'^',
        view=include('block.urls.cms')
        ),
)

urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
#   ^ helper function to return a URL pattern for serving files in debug mode.
# https://docs.djangoproject.com/en/1.5/howto/static-files/#serving-files-uploaded-by-a-user
<|endoftext|>


<commit_before>import logging
import json
from operator import itemgetter
from collections import defaultdict

from django.db.models.query import Q
from django.forms.formsets import formset_factory
from django.core import exceptions
from django.core.urlresolvers import reverse
from django.template import RequestContext
from django.template.loader import render_to_string
from django.db.models.sql.constants import LOOKUP_SEP
from django.utils.safestring import mark_safe
from django.contrib import messages

from .registry import search, SearchRegistry
from .forms import ModelSelectionForm, ConstraintForm, ConstraintFormset
from .ormutils import resolve_orm_path

log = logging.getLogger(__name__)

class Searcher(object):
    """ Template helper, wrapping all the necessary components to render an appsearch page. """

    # Methods and fields not meant to be accessed from the template should start with an underscore
    # to let the template variable name resolution block access.

    # Customizable classes
    model_selection_form_class = ModelSelectionForm
    constraint_form_class = ConstraintForm
    constraint_formset_class = ConstraintFormset

    # Instance data
    model_selection_form = None
    constraint_formset = None

    results = None

    # Processing callback hooks
    _display_fields_callback = None
    _build_queryset_callback = None
    _process_results_callback = None

    ## Fallback items normally provided by the view
    context_object_name = 'search'

    # Default templates
    form_template_name = "appsearch/default_form.html"
    search_form_template_name = "appsearch/search_form.html"
    results_list_template_name = "appsearch/results_list.html"

    def __init__(self, request, url=None, querydict=None, registry=search, permission=None, **kwargs):
        self.kwargs = kwargs
        self.request = request
        self.url = url or request.path

        self._forms_ready = False
        self._set_up_forms(querydict or request.GET, registry, permission)
        self.permission = permission
        self.registry = registry

        # Fallback items
        self.context_object_name = kwargs.get('context_object_name', self.context_object_name)
        self.form_template_name = kwargs.get('form_template_name', self.form_template_name)
        self.search_form_template_name = kwargs.get('search_form_template_name', self.search_form_template_name)
        self.results_list_template_name = kwargs.get('results_list_template_name', self.results_list_template_name)

        self._display_fields_callback = kwargs.get('display_fields_callback')
        self._build_queryset_callback = kwargs.get('build_queryset_callback')
        self._process_results_callback = kwargs.get('process_results_callback')

    # Rendering methods
    def __unicode__(self):
        return render_to_string(self.form_template_name, RequestContext(self.request, {
            self.context_object_name: self,
        }))

    def render_search_form(self):
        """ Renders only the template at ``search_form_template_name`` """
        return render_to_string(self.search_form_template_name, RequestContext(self.request, {
            self.context_object_name: self,
        }))

    def render_results_list(self):
        """ Renders only the template at ``results_list_template_name`` """
        return render_to_string(self.results_list_template_name, RequestContext(self.request, {
            self.context_object_name: self,
        }))

    def render_constraint_fields(self, model):
        """ Renders into JSON the model's fields available for search queries. """

        configuration = self.registry.get_configuration(model)
        choices = configuration.get_searchable_field_choices(include_types=True)
        return json.dumps({'choices': choices})

    def get_constraint_field_operators(self, model, field=None, hash=None):
        """ Returns into JSON the model's field's valid search operators. """

        if field is None and hash is None:
            raise ValueError("Need one of 'field' or 'hash'.")

        configuration = self.registry.get_configuration(model)

        # Fetch the choices without the query orm language.  Only UI labels are used at this stage.
        choices = configuration.get_operator_choices(field=field, hash=hash, flat=True)
        return choices

    def render_all_constraint_choices(self):
        """
        Returns a mapping of all models to their field choices, and all models to field hashes to
        operators.

        """

        field_data = defaultdict(list)
        operator_data = defaultdict(dict)

        configurations = self.model_selection_form.configurations
        model_values = map(itemgetter(0), self.model_selection_form.fields['model'].choices)[1:]

        for model_value, config in zip(model_values, configurations):
            model = config.model
            field_choices = config.get_searchable_field_choices(include_types=False)
            for orm_hash, field_label in field_choices:
                operator_choices = self.get_constraint_field_operators(model, hash=orm_hash)
                field_data[model_value].append([orm_hash, field_label])
                operator_data[model_value][orm_hash] = operator_choices

        return mark_safe(json.dumps({
            'fields': field_data,
            'operators': operator_data,
        }))

    # Configuration methods
    def get_model_selection_form_class(self):
        """ Returns ``self.model_selection_form_class`` """
        return self.model_selection_form_class

    def get_constraint_form_class(self):
        """ Returns ``self.constraint_form_class`` """
        return self.constraint_form_class

    def get_constraint_formset_class(self):
        """ Returns ``self.constraint_formset_class`` """
        return self.constraint_formset_class

    @property
    def ready(self):
        """ Indicates if a search has been executed by the constructed state. """

        if self._forms_ready:
            self.model_config = self.model_selection_form.get_selected_configuration()
            self.model = self.model_config.model
        return self._forms_ready

    def _set_up_forms(self, querydict, registry, permission=None):
        ModelSelectionFormClass = self.get_model_selection_form_class()
        ConstraintFormClass = self.get_constraint_form_class()
        ConstraintFormsetClass = self.get_constraint_formset_class()
        ConstraintFormsetClass = formset_factory(ConstraintFormClass, formset=ConstraintFormsetClass)

        self.model_selection_form = ModelSelectionFormClass(registry, self.request.user, permission,
                querydict)

        if self.model_selection_form.is_valid():
            model_configuration = self.model_selection_form.get_selected_configuration()
            self.constraint_formset = ConstraintFormsetClass(model_configuration, querydict)
            if self.constraint_formset.is_valid():
                self._forms_ready = True
        else:
            self.model_selection_form = ModelSelectionFormClass(registry, self.request.user,
                    permission)
            self.constraint_formset = ConstraintFormsetClass(configuration=None)

    # def _get_field(self, model, querystring):
    #     """ Looks up the field on ``model`` described by the ORM-style ``querystring``. """
    #     original = model
    #     field = None
    #     querystring_list = querystring.split(LOOKUP_SEP)
    #     target_field_string = querystring_list[-1]
    #     for fieldname in querystring_list[:-1]:
    #         # Covers FK
    #         try:
    #             model = getattr(model, fieldname).field.rel.to
    #         except (exceptions.ObjectDoesNotExist, AttributeError) as e:
    #             try:
    #                 model = getattr(model(), fieldname).model
    #             except exceptions.ObjectDoesNotExist:
    #                 log.warning("Unable to figure out the target field given a model of {} and "
    #                             "querystring of {}".format(original, querystring))
    #                 break
    #     field = next((f for f in model._meta.fields if f.name==target_field_string), None)
    #     log.debug("Translated {} to field type of {}".format(querystring, repr(field)))
    #     return field

    def _perform_search(self):
        """
        Generates the query using the validated constraint formset.  Also compiles a natural
        language string (``self.natural_string``) in the format of
        "N [Model]s with [field] [operator] '[term]'".

        """

        natural_string = []

        # 2-tuples of an operator and Q instance, sent through reduce() after it's built
        query_list = []

        for i, constraint_form in enumerate(self.constraint_formset):
            type_operator = constraint_form.cleaned_data['type']
            field_list = constraint_form.cleaned_data['field']
            constraint_operator = constraint_form.cleaned_data['operator']
            term = constraint_form.cleaned_data['term']
            end_term = constraint_form.cleaned_data['end_term']

            verbose_name = self.model_config._fields[field_list]

            # Build this field's query.
            # Search fields bound together in a tuple are considered OR conditions for a single
            # virtual field name.
            query = None

            # Iterate multiple fields defined in a compound column
            for field in field_list:
                value = term

                # This covers the field choice swaparoo
                target_field = resolve_orm_path(self.model, field)
                if target_field and len(target_field.choices):
                    for key, val in dict(target_field.choices).items():
                        if str(val).lower() in str(value).lower():
                            value = key
                            break
                    else:
                        vals = [x[1] for x in target_field.choices]
                        msg = "Unable to identify a matching key from '{}' - Valid " \
                              "choices are: '{}'".format(value, "', '".join(vals))
                        value = target_field.choices[0][0]
                        messages.warning(self.request, msg)

                # Prep an inverted lookup
                negative = constraint_operator.startswith('!')
                if negative:
                    constraint_operator = constraint_operator[1:]

                if constraint_operator == "isnull":
                    value = not negative
                    negative = False

                # Bake the queryset language string
                field_query = LOOKUP_SEP.join((field, constraint_operator))

                q = Q(**{
                    field_query: value,
                })
                log.debug("Querying %s [%d]: %s%s=%r", self.model.__name__, i, field_query,
                          "!" if negative else "", value)

                # Negate if necessary
                if negative:
                    q = ~q

                if query is None:
                    query = q
                else:
                    query |= q

            query_list.append((type_operator, query))

            # Do some natural processing
            if isinstance(value, (tuple, list)):
                value = ' - '.join(map(unicode, value))
            else:
                value = str(value)
            bits = [verbose_name, constraint_form['operator'].value(), value]
            if i != 0: # Skip the leading "and" on the first constraint
                bits.insert(0, constraint_form['type'].value())
            natural_string.append(bits)

        # The first query's "type" should be ignored for the sake of the reduce line below.
        query = reduce(lambda q1, (op, q2): op(q1, q2), query_list[1:], query_list[0][1])

        queryset = self.build_queryset(self.model, query)
        data_rows = self.process_results(queryset)

        self.results = {
            'count': len(queryset),
            'list': data_rows,
            'fields': self._get_display_fields(self.model, self.model_config),
            'natural_string': "where " + ', '.join(map(' '.join, natural_string)),
        }

    def _get_display_fields(self, model, config):
        if self._display_fields_callback:
            return self._display_fields_callback(self, model, config)
        return config.get_display_fields()

    def get_select_related_fields(self, model, config):
        """ Returns a list of queryset language names to pass into ``.select_related()`` """
        display_fields = self._get_display_fields(model, config)
        related_names = set()
        for field_info in display_fields:
            if isinstance(field_info, (tuple, list)):
                _, field_name = field_info
            else:
                field_name = field_info

            related_path = field_name.rsplit(LOOKUP_SEP, 1)[0]

            if LOOKUP_SEP in related_path:
                related_names.add(related_path)

        return related_names

    def build_queryset(self, model, query, queryset=None):
        """
        Returns the queryset using ``query``.

        Default behavior inspects the display fields for any related items and requests their
        selection and adds ``.distinct()``.

        If ``base_queryset`` is provided, it will be used as the starting point for applying the
        ``query`` filter.  This is useful for subclasses that want to change the default manager
        being accessed for the initial queryset.  Passing up a ``queryset`` via super() will
        accomplish this, returning a fully built queryset with minimum hassle.

        If ``build_queryset`` callback was defined on the originating view, it will be called
        after the queryset is built and will be sent the searcher instance, model, config, initial
        ``Q`` query, and the derived queryset.  The callback should return the queryset in its final
        state for execution.

        """

        related_names = self.get_select_related_fields(model, self.model_config)

        if queryset is None:
            queryset = self.model_config.get_queryset(self.request, self.request.user)

        queryset = queryset.filter(query).select_related(*related_names).distinct()

        if self._build_queryset_callback:
            queryset = self._build_queryset_callback(self, model, self.model_config, query,
                    queryset)

        return queryset

    def process_results(self, queryset):
        """
        Should return a sequence of n-tuples, where ``n`` is the number of columns to be shown in
        the table.

        """

        if self._process_results_callback:
            return self._process_results_callback(self, self.model, self.model_config, queryset)

        return [self.model_config.get_object_data(obj) for obj in queryset]
<commit_msg>Fixes select_related derivation<commit_after>import logging
import json
from operator import itemgetter
from collections import defaultdict

from django.db.models.query import Q
from django.forms.formsets import formset_factory
from django.core import exceptions
from django.core.urlresolvers import reverse
from django.template import RequestContext
from django.template.loader import render_to_string
from django.db.models.sql.constants import LOOKUP_SEP
from django.utils.safestring import mark_safe
from django.contrib import messages

from .registry import search, SearchRegistry
from .forms import ModelSelectionForm, ConstraintForm, ConstraintFormset
from .ormutils import resolve_orm_path

log = logging.getLogger(__name__)

class Searcher(object):
    """ Template helper, wrapping all the necessary components to render an appsearch page. """

    # Methods and fields not meant to be accessed from the template should start with an underscore
    # to let the template variable name resolution block access.

    # Customizable classes
    model_selection_form_class = ModelSelectionForm
    constraint_form_class = ConstraintForm
    constraint_formset_class = ConstraintFormset

    # Instance data
    model_selection_form = None
    constraint_formset = None

    results = None

    # Processing callback hooks
    _display_fields_callback = None
    _build_queryset_callback = None
    _process_results_callback = None

    ## Fallback items normally provided by the view
    context_object_name = 'search'

    # Default templates
    form_template_name = "appsearch/default_form.html"
    search_form_template_name = "appsearch/search_form.html"
    results_list_template_name = "appsearch/results_list.html"

    def __init__(self, request, url=None, querydict=None, registry=search, permission=None, **kwargs):
        self.kwargs = kwargs
        self.request = request
        self.url = url or request.path

        self._forms_ready = False
        self._set_up_forms(querydict or request.GET, registry, permission)
        self.permission = permission
        self.registry = registry

        # Fallback items
        self.context_object_name = kwargs.get('context_object_name', self.context_object_name)
        self.form_template_name = kwargs.get('form_template_name', self.form_template_name)
        self.search_form_template_name = kwargs.get('search_form_template_name', self.search_form_template_name)
        self.results_list_template_name = kwargs.get('results_list_template_name', self.results_list_template_name)

        self._display_fields_callback = kwargs.get('display_fields_callback')
        self._build_queryset_callback = kwargs.get('build_queryset_callback')
        self._process_results_callback = kwargs.get('process_results_callback')

    # Rendering methods
    def __unicode__(self):
        return render_to_string(self.form_template_name, RequestContext(self.request, {
            self.context_object_name: self,
        }))

    def render_search_form(self):
        """ Renders only the template at ``search_form_template_name`` """
        return render_to_string(self.search_form_template_name, RequestContext(self.request, {
            self.context_object_name: self,
        }))

    def render_results_list(self):
        """ Renders only the template at ``results_list_template_name`` """
        return render_to_string(self.results_list_template_name, RequestContext(self.request, {
            self.context_object_name: self,
        }))

    def render_constraint_fields(self, model):
        """ Renders into JSON the model's fields available for search queries. """

        configuration = self.registry.get_configuration(model)
        choices = configuration.get_searchable_field_choices(include_types=True)
        return json.dumps({'choices': choices})

    def get_constraint_field_operators(self, model, field=None, hash=None):
        """ Returns into JSON the model's field's valid search operators. """

        if field is None and hash is None:
            raise ValueError("Need one of 'field' or 'hash'.")

        configuration = self.registry.get_configuration(model)

        # Fetch the choices without the query orm language.  Only UI labels are used at this stage.
        choices = configuration.get_operator_choices(field=field, hash=hash, flat=True)
        return choices

    def render_all_constraint_choices(self):
        """
        Returns a mapping of all models to their field choices, and all models to field hashes to
        operators.

        """

        field_data = defaultdict(list)
        operator_data = defaultdict(dict)

        configurations = self.model_selection_form.configurations
        model_values = map(itemgetter(0), self.model_selection_form.fields['model'].choices)[1:]

        for model_value, config in zip(model_values, configurations):
            model = config.model
            field_choices = config.get_searchable_field_choices(include_types=False)
            for orm_hash, field_label in field_choices:
                operator_choices = self.get_constraint_field_operators(model, hash=orm_hash)
                field_data[model_value].append([orm_hash, field_label])
                operator_data[model_value][orm_hash] = operator_choices

        return mark_safe(json.dumps({
            'fields': field_data,
            'operators': operator_data,
        }))

    # Configuration methods
    def get_model_selection_form_class(self):
        """ Returns ``self.model_selection_form_class`` """
        return self.model_selection_form_class

    def get_constraint_form_class(self):
        """ Returns ``self.constraint_form_class`` """
        return self.constraint_form_class

    def get_constraint_formset_class(self):
        """ Returns ``self.constraint_formset_class`` """
        return self.constraint_formset_class

    @property
    def ready(self):
        """ Indicates if a search has been executed by the constructed state. """

        if self._forms_ready:
            self.model_config = self.model_selection_form.get_selected_configuration()
            self.model = self.model_config.model
        return self._forms_ready

    def _set_up_forms(self, querydict, registry, permission=None):
        ModelSelectionFormClass = self.get_model_selection_form_class()
        ConstraintFormClass = self.get_constraint_form_class()
        ConstraintFormsetClass = self.get_constraint_formset_class()
        ConstraintFormsetClass = formset_factory(ConstraintFormClass, formset=ConstraintFormsetClass)

        self.model_selection_form = ModelSelectionFormClass(registry, self.request.user, permission,
                querydict)

        if self.model_selection_form.is_valid():
            model_configuration = self.model_selection_form.get_selected_configuration()
            self.constraint_formset = ConstraintFormsetClass(model_configuration, querydict)
            if self.constraint_formset.is_valid():
                self._forms_ready = True
        else:
            self.model_selection_form = ModelSelectionFormClass(registry, self.request.user,
                    permission)
            self.constraint_formset = ConstraintFormsetClass(configuration=None)

    # def _get_field(self, model, querystring):
    #     """ Looks up the field on ``model`` described by the ORM-style ``querystring``. """
    #     original = model
    #     field = None
    #     querystring_list = querystring.split(LOOKUP_SEP)
    #     target_field_string = querystring_list[-1]
    #     for fieldname in querystring_list[:-1]:
    #         # Covers FK
    #         try:
    #             model = getattr(model, fieldname).field.rel.to
    #         except (exceptions.ObjectDoesNotExist, AttributeError) as e:
    #             try:
    #                 model = getattr(model(), fieldname).model
    #             except exceptions.ObjectDoesNotExist:
    #                 log.warning("Unable to figure out the target field given a model of {} and "
    #                             "querystring of {}".format(original, querystring))
    #                 break
    #     field = next((f for f in model._meta.fields if f.name==target_field_string), None)
    #     log.debug("Translated {} to field type of {}".format(querystring, repr(field)))
    #     return field

    def _perform_search(self):
        """
        Generates the query using the validated constraint formset.  Also compiles a natural
        language string (``self.natural_string``) in the format of
        "N [Model]s with [field] [operator] '[term]'".

        """

        natural_string = []

        # 2-tuples of an operator and Q instance, sent through reduce() after it's built
        query_list = []

        for i, constraint_form in enumerate(self.constraint_formset):
            type_operator = constraint_form.cleaned_data['type']
            field_list = constraint_form.cleaned_data['field']
            constraint_operator = constraint_form.cleaned_data['operator']
            term = constraint_form.cleaned_data['term']
            end_term = constraint_form.cleaned_data['end_term']

            verbose_name = self.model_config._fields[field_list]

            # Build this field's query.
            # Search fields bound together in a tuple are considered OR conditions for a single
            # virtual field name.
            query = None

            # Iterate multiple fields defined in a compound column
            for field in field_list:
                value = term

                # This covers the field choice swaparoo
                target_field = resolve_orm_path(self.model, field)
                if target_field and len(target_field.choices):
                    for key, val in dict(target_field.choices).items():
                        if str(val).lower() in str(value).lower():
                            value = key
                            break
                    else:
                        vals = [x[1] for x in target_field.choices]
                        msg = "Unable to identify a matching key from '{}' - Valid " \
                              "choices are: '{}'".format(value, "', '".join(vals))
                        value = target_field.choices[0][0]
                        messages.warning(self.request, msg)

                # Prep an inverted lookup
                negative = constraint_operator.startswith('!')
                if negative:
                    constraint_operator = constraint_operator[1:]

                if constraint_operator == "isnull":
                    value = not negative
                    negative = False

                # Bake the queryset language string
                field_query = LOOKUP_SEP.join((field, constraint_operator))

                q = Q(**{
                    field_query: value,
                })
                log.debug("Querying %s [%d]: %s%s=%r", self.model.__name__, i, field_query,
                          "!" if negative else "", value)

                # Negate if necessary
                if negative:
                    q = ~q

                if query is None:
                    query = q
                else:
                    query |= q

            query_list.append((type_operator, query))

            # Do some natural processing
            if isinstance(value, (tuple, list)):
                value = ' - '.join(map(unicode, value))
            else:
                value = str(value)
            bits = [verbose_name, constraint_form['operator'].value(), value]
            if i != 0: # Skip the leading "and" on the first constraint
                bits.insert(0, constraint_form['type'].value())
            natural_string.append(bits)

        # The first query's "type" should be ignored for the sake of the reduce line below.
        query = reduce(lambda q1, (op, q2): op(q1, q2), query_list[1:], query_list[0][1])

        queryset = self.build_queryset(self.model, query)
        data_rows = self.process_results(queryset)

        self.results = {
            'count': len(queryset),
            'list': data_rows,
            'fields': self._get_display_fields(self.model, self.model_config),
            'natural_string': "where " + ', '.join(map(' '.join, natural_string)),
        }

    def _get_display_fields(self, model, config):
        if self._display_fields_callback:
            return self._display_fields_callback(self, model, config)
        return config.get_display_fields()

    def get_select_related_fields(self, model, config):
        """ Returns a list of queryset language names to pass into ``.select_related()`` """
        display_fields = config._display_fields
        related_names = set()
        for _, field_name, _ in display_fields:
            name_bits = field_name.rsplit(LOOKUP_SEP, 1)
            if len(name_bits) == 2:
                related_names.add(name_bits[0])

        return related_names

    def build_queryset(self, model, query, queryset=None):
        """
        Returns the queryset using ``query``.

        Default behavior inspects the display fields for any related items and requests their
        selection and adds ``.distinct()``.

        If ``base_queryset`` is provided, it will be used as the starting point for applying the
        ``query`` filter.  This is useful for subclasses that want to change the default manager
        being accessed for the initial queryset.  Passing up a ``queryset`` via super() will
        accomplish this, returning a fully built queryset with minimum hassle.

        If ``build_queryset`` callback was defined on the originating view, it will be called
        after the queryset is built and will be sent the searcher instance, model, config, initial
        ``Q`` query, and the derived queryset.  The callback should return the queryset in its final
        state for execution.

        """

        related_names = self.get_select_related_fields(model, self.model_config)

        if queryset is None:
            queryset = self.model_config.get_queryset(self.request, self.request.user)

        queryset = queryset.filter(query).select_related(*related_names).distinct()

        if self._build_queryset_callback:
            queryset = self._build_queryset_callback(self, model, self.model_config, query,
                    queryset)

        return queryset

    def process_results(self, queryset):
        """
        Should return a sequence of n-tuples, where ``n`` is the number of columns to be shown in
        the table.

        """

        if self._process_results_callback:
            return self._process_results_callback(self, self.model, self.model_config, queryset)

        return [self.model_config.get_object_data(obj) for obj in queryset]
<|endoftext|>


<commit_before>GEVENT = False

from regs_models import *

def run():
    import settings

    # grab a dictionary
    word_file = getattr(settings, 'WORD_FILE', '/usr/share/dict/words')
    english_words = set([word.strip() for word in open(word_file, 'r') if word and word[0] == word[0].lower()])

    from influenceexplorer import InfluenceExplorer
    api = InfluenceExplorer(settings.API_KEY, getattr(settings, 'AGGREGATES_API_BASE_URL', "http://transparencydata.com/api/1.0/"))

    entities = []
    for type in ['individual', 'organization', 'politician']:
        count = api.entities.count(type)
        for i in range(0, count, 10000):
            entities.extend(api.entities.list(i, i + 10000, type))

    from oxtail.matching.normalize import normalize_list
    for entity in entities:
        record = {
            'id': entity['id'],
            'td_type': entity['type'],
            'td_name': entity['name'],
            'aliases': normalize_list([entity['name']] + entity['aliases'], entity['type'])
        }
        record['filtered_aliases'] = [alias for alias in record['aliases'] if alias.lower() not in english_words]

        db_entity = Entity(**record)
        db_entity.save()

        print "Saved %s as %s" % (record['aliases'][0], record['id'])<commit_msg>Add ability to update existing entity lists preserving stats.<commit_after>GEVENT = False

from regs_models import *

from optparse import OptionParser
arg_parser = OptionParser()
arg_parser.add_option("-U", "--update", dest="update", action="store_true", default=False, help="Check if entities already existing before creating (slower)")

def run(options, args):
    import settings

    # if we're updating
    if options.update:
        print "Preparing to update existing entities; retrieving current entity list..."
        current = set((e.id for e in Entity.objects()))
        print "Entities retrieved."
    else:
        print "Constructing new entity list."

    # grab a dictionary
    word_file = getattr(settings, 'WORD_FILE', '/usr/share/dict/words')
    english_words = set([word.strip() for word in open(word_file, 'r') if word and word[0] == word[0].lower()])

    from influenceexplorer import InfluenceExplorer
    api = InfluenceExplorer(settings.API_KEY, getattr(settings, 'AGGREGATES_API_BASE_URL', "http://transparencydata.com/api/1.0/"))

    entities = []
    for type in ['individual', 'organization', 'politician']:
        count = api.entities.count(type)
        for i in range(0, count, 10000):
            entities.extend(api.entities.list(i, i + 10000, type))

    from oxtail.matching.normalize import normalize_list
    for entity in entities:
        record = {
            'id': entity['id'],
            'td_type': entity['type'],
            'td_name': entity['name'],
            'aliases': normalize_list([entity['name']] + entity['aliases'], entity['type'])
        }
        record['filtered_aliases'] = [alias for alias in record['aliases'] if alias.lower() not in english_words]

        if options.update and record['id'] in current:
            Entity.objects(id=record['id']) \
                  .update(safe_update=True, set__td_type=record['td_type'], set__td_name=record['td_name'], set__aliases=record['aliases'], set__filtered_aliases=record['filtered_aliases'])
            print "Updated %s as existing record %s" % (record['aliases'][0], record['id'])
        else:
            db_entity = Entity(**record)
            db_entity.save()
            print "Saved %s as new record %s" % (record['aliases'][0], record['id'])<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
Scripts related to age prediction model.
"""

import logging
import json
import os
import os.path as op
import sys

import pandas as pd

from jinja2 import Template
from jcvi.apps.base import OptionParser, ActionDispatcher, iglob

def main():

    actions = (
        ('compile', 'extract telomere length and ccn'),
        ('traits', 'make HTML page that reports eye and skin color'),
            )
    p = ActionDispatcher(actions)
    p.dispatch(globals())

traits_template = '''

        <title>ART traits</title>
        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/css/materialize.min.css">

        Sample ID
        Skin
        Eyes

    {% for sample in samples %}

        {{ sample.sample_id }}

    {% endfor %}

'''

def lab2rgb(L, A, B):
    # Borrowed from:
    # <https://github.com/antimatter15/rgb-lab/blob/master/color.js>
    y = (L + 16) / 116
    x = A / 500 + y
    z = y - B / 200

    x = 0.95047 * (x * x * x if (x * x * x > 0.008856) else (x - 16 / 116) / 7.787)
    y = 1.00000 * (y * y * y if (y * y * y > 0.008856) else (y - 16 / 116) / 7.787)
    z = 1.08883 * (z * z * z if (z * z * z > 0.008856) else (z - 16 / 116) / 7.787)

    r = x * 3.2406 + y * -1.5372 + z * -0.4986
    g = x * -0.9689 + y * 1.8758 + z * 0.0415
    b = x * 0.0557 + y * -0.2040 + z * 1.0570

    r = (1.055 * r ** (1 / 2.4) - 0.055) if (r > 0.0031308) else 12.92 * r
    g = (1.055 * g ** (1 / 2.4) - 0.055) if (g > 0.0031308) else 12.92 * g
    b = (1.055 * b ** (1 / 2.4) - 0.055) if (b > 0.0031308) else 12.92 * b

    return max(0, min(1, r)) * 255, max(0, min(1, g)) * 255, max(0, min(1, b)) * 255

def make_rgb(L, A, B):
    r, g, b = lab2rgb(L, A, B)
    r = int(round(r))
    g = int(round(g))
    b = int(round(b))
    return "rgb({}, {}, {})".format(r, g, b)

def traits(args):
    """
    %prog traits directory

    Make HTML page that reports eye and skin color.
    """
    p = OptionParser(traits.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    samples = []
    for folder in args:
        filename = iglob(folder, "*-traits.json")[0]
        js = json.load(open(filename))
        js["skin_rgb"] = make_rgb(
            js["traits"]["skin-color"]["L"],
            js["traits"]["skin-color"]["A"],
            js["traits"]["skin-color"]["B"])
        js["eye_rgb"] = make_rgb(
            js["traits"]["eye-color"]["L"],
            js["traits"]["eye-color"]["A"],
            js["traits"]["eye-color"]["B"])
        samples.append(js)

    template = Template(traits_template)
    fw = open("report.html", "w")
    print >> fw, template.render(samples=samples)
    logging.debug("Report written to `{}`".format(fw.name))
    fw.close()

def compile(args):
    """
    %prog compile directory

    Extract telomere length and ccn.
    """
    p = OptionParser(compile.__doc__)
    p.set_outfile(outfile="age.tsv")
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    dfs = []
    for folder in args:
        ofolder = os.listdir(folder)

        # telomeres
        subdir = [x for x in ofolder if x.startswith("telomeres")][0]
        subdir = op.join(folder, subdir)
        filename = op.join(subdir, "tel_lengths.txt")
        df = pd.read_csv(filename, sep="\t")
        d1 = df.ix[0].to_dict()

        # ccn
        subdir = [x for x in ofolder if x.startswith("ccn")][0]
        subdir = op.join(folder, subdir)
        filename = iglob(subdir, "*.ccn.json")[0]
        js = json.load(open(filename))
        d1.update(js)
        df = pd.DataFrame(d1, index=[0])
        dfs.append(df)

    df = pd.concat(dfs, ignore_index=True)
    df.to_csv(opts.outfile, sep="\t", index=False)

if __name__ == '__main__':
    main()
<commit_msg>[projects] Add rounded corners for box<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
Scripts related to age prediction model.
"""

import logging
import json
import os
import os.path as op
import sys

import pandas as pd

from jinja2 import Template
from jcvi.apps.base import OptionParser, ActionDispatcher, iglob

def main():

    actions = (
        ('compile', 'extract telomere length and ccn'),
        ('traits', 'make HTML page that reports eye and skin color'),
            )
    p = ActionDispatcher(actions)
    p.dispatch(globals())

traits_template = '''

        <title>ART traits</title>
        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/css/materialize.min.css">

        Sample ID
        Skin
        Eyes

    {% for sample in samples %}

        {{ sample.sample_id }}

    {% endfor %}

'''

def lab2rgb(L, A, B):
    # Borrowed from:
    # <https://github.com/antimatter15/rgb-lab/blob/master/color.js>
    y = (L + 16) / 116
    x = A / 500 + y
    z = y - B / 200

    x = 0.95047 * (x * x * x if (x * x * x > 0.008856) else (x - 16 / 116) / 7.787)
    y = 1.00000 * (y * y * y if (y * y * y > 0.008856) else (y - 16 / 116) / 7.787)
    z = 1.08883 * (z * z * z if (z * z * z > 0.008856) else (z - 16 / 116) / 7.787)

    r = x * 3.2406 + y * -1.5372 + z * -0.4986
    g = x * -0.9689 + y * 1.8758 + z * 0.0415
    b = x * 0.0557 + y * -0.2040 + z * 1.0570

    r = (1.055 * r ** (1 / 2.4) - 0.055) if (r > 0.0031308) else 12.92 * r
    g = (1.055 * g ** (1 / 2.4) - 0.055) if (g > 0.0031308) else 12.92 * g
    b = (1.055 * b ** (1 / 2.4) - 0.055) if (b > 0.0031308) else 12.92 * b

    return max(0, min(1, r)) * 255, max(0, min(1, g)) * 255, max(0, min(1, b)) * 255

def make_rgb(L, A, B):
    r, g, b = lab2rgb(L, A, B)
    r = int(round(r))
    g = int(round(g))
    b = int(round(b))
    return "rgb({}, {}, {})".format(r, g, b)

def traits(args):
    """
    %prog traits directory

    Make HTML page that reports eye and skin color.
    """
    p = OptionParser(traits.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    samples = []
    for folder in args:
        filename = iglob(folder, "*-traits.json")[0]
        js = json.load(open(filename))
        js["skin_rgb"] = make_rgb(
            js["traits"]["skin-color"]["L"],
            js["traits"]["skin-color"]["A"],
            js["traits"]["skin-color"]["B"])
        js["eye_rgb"] = make_rgb(
            js["traits"]["eye-color"]["L"],
            js["traits"]["eye-color"]["A"],
            js["traits"]["eye-color"]["B"])
        samples.append(js)

    template = Template(traits_template)
    fw = open("report.html", "w")
    print >> fw, template.render(samples=samples)
    logging.debug("Report written to `{}`".format(fw.name))
    fw.close()

def compile(args):
    """
    %prog compile directory

    Extract telomere length and ccn.
    """
    p = OptionParser(compile.__doc__)
    p.set_outfile(outfile="age.tsv")
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    dfs = []
    for folder in args:
        ofolder = os.listdir(folder)

        # telomeres
        subdir = [x for x in ofolder if x.startswith("telomeres")][0]
        subdir = op.join(folder, subdir)
        filename = op.join(subdir, "tel_lengths.txt")
        df = pd.read_csv(filename, sep="\t")
        d1 = df.ix[0].to_dict()

        # ccn
        subdir = [x for x in ofolder if x.startswith("ccn")][0]
        subdir = op.join(folder, subdir)
        filename = iglob(subdir, "*.ccn.json")[0]
        js = json.load(open(filename))
        d1.update(js)
        df = pd.DataFrame(d1, index=[0])
        dfs.append(df)

    df = pd.concat(dfs, ignore_index=True)
    df.to_csv(opts.outfile, sep="\t", index=False)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python2
# -*- coding: utf-8 -*-
#
# Copyright (C) 2015 Richard Hughes <richard@hughsie.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA

import xml.etree.ElementTree as ET

try:
    # Py2.7 and newer
    from xml.etree.ElementTree import ParseError as StdlibParseError
except ImportError:
    # Py2.6 and older
    from xml.parsers.expat import ExpatError as StdlibParseError

from appstream.errors import ParseError

def _join_lines(txt):
    """ Remove whitespace from XML input """
    txt = txt or ''  # Handle NoneType input values
    val = ''
    lines = txt.split('\n')
    for line in lines:
        stripped = line.strip()
        if len(stripped) == 0:
            continue
        val += stripped + ' '
    return val.strip()

def _parse_desc(node):
    """ A quick'n'dirty description parser """
    desc = ''
    for n in node:
        if n.tag == 'p':
            desc += '
' + _join_lines(n.text) + '
'
        elif n.tag == 'ol' or n.tag == 'ul':
            desc += '
'
            for c in n:
                if c.tag == 'li':
                    desc += '
' + _join_lines(c.text) + '
'
                else:
                    raise ParseError('Expected
 in <%s>, got <%s>' % (n.tag, c.tag))
            desc += '
'
        else:
            raise ParseError('Expected
,
,
 in <%s>, got <%s>' % (node.tag, n.tag))
    return desc

def validate_description(xml_data):
    """ Validate the description for validity """
    try:
        root = ET.fromstring('<document>' + xml_data + '</document>')
    except StdlibParseError as e:
        raise ParseError(str(e))
    return _parse_desc(root)

def _import_description_to_list_element(text):
    if len(text) < 5:
        return None
    if text.startswith('- '):
        return text[2:]
    if text[0].isdigit and text[1:3] == '. ':
        return text[3:]
    return None

def _import_description_sentence_case(text):
    return text[0].upper() + text[1:]

def import_description(text):
    """ Convert ASCII text to AppStream markup format """
    xml = ''
    is_in_ul = False
    for line in text.split('\n'):

        # don't include whitespace
        line = line.strip()
        if len(line) == 0:
            continue

        # detected as a list element?
        line_li = _import_description_to_list_element(line)
        if line_li:
            # first list element
            if not is_in_ul:
                xml += '
\n'
                is_in_ul = True
            xml += '
' + _import_description_sentence_case(line_li) + '
\n'
            continue

        # done with the list
        if is_in_ul:
            xml += '
\n'
            is_in_ul = False

        # regular paragraph
        xml += '
' + _import_description_sentence_case(line) + '
\n'

    # no trailing paragraph
    if is_in_ul:
        xml += '
\n'

    return xml
<commit_msg>trivial: Support parsing bare text in _parse_desc<commit_after>#!/usr/bin/python2
# -*- coding: utf-8 -*-
#
# Copyright (C) 2015 Richard Hughes <richard@hughsie.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA

import xml.etree.ElementTree as ET

try:
    # Py2.7 and newer
    from xml.etree.ElementTree import ParseError as StdlibParseError
except ImportError:
    # Py2.6 and older
    from xml.parsers.expat import ExpatError as StdlibParseError

from appstream.errors import ParseError

def _join_lines(txt):
    """ Remove whitespace from XML input """
    txt = txt or ''  # Handle NoneType input values
    val = ''
    lines = txt.split('\n')
    for line in lines:
        stripped = line.strip()
        if len(stripped) == 0:
            continue
        val += stripped + ' '
    return val.strip()

def _parse_desc(node):
    """ A quick'n'dirty description parser """
    desc = ''
    if len(node) == 0:
        return '
' + node.text + '
'
    for n in node:
        if n.tag == 'p':
            desc += '
' + _join_lines(n.text) + '
'
        elif n.tag == 'ol' or n.tag == 'ul':
            desc += '
'
            for c in n:
                if c.tag == 'li':
                    desc += '
' + _join_lines(c.text) + '
'
                else:
                    raise ParseError('Expected
 in <%s>, got <%s>' % (n.tag, c.tag))
            desc += '
'
        else:
            raise ParseError('Expected
,
,
 in <%s>, got <%s>' % (node.tag, n.tag))
    return desc

def validate_description(xml_data):
    """ Validate the description for validity """
    try:
        root = ET.fromstring('<document>' + xml_data + '</document>')
    except StdlibParseError as e:
        raise ParseError(str(e))
    return _parse_desc(root)

def _import_description_to_list_element(text):
    if len(text) < 5:
        return None
    if text.startswith('- '):
        return text[2:]
    if text[0].isdigit and text[1:3] == '. ':
        return text[3:]
    return None

def _import_description_sentence_case(text):
    return text[0].upper() + text[1:]

def import_description(text):
    """ Convert ASCII text to AppStream markup format """
    xml = ''
    is_in_ul = False
    for line in text.split('\n'):

        # don't include whitespace
        line = line.strip()
        if len(line) == 0:
            continue

        # detected as a list element?
        line_li = _import_description_to_list_element(line)
        if line_li:
            # first list element
            if not is_in_ul:
                xml += '
\n'
                is_in_ul = True
            xml += '
' + _import_description_sentence_case(line_li) + '
\n'
            continue

        # done with the list
        if is_in_ul:
            xml += '
\n'
            is_in_ul = False

        # regular paragraph
        xml += '
' + _import_description_sentence_case(line) + '
\n'

    # no trailing paragraph
    if is_in_ul:
        xml += '
\n'

    return xml
<|endoftext|>


<commit_before><commit_msg>Make the input_fn return the dataset directly.<commit_after><|endoftext|>