<commit_before>from math import log, isnan

import numpy as np
from bokeh.models import *
from bokeh.plotting import figure
from itertools import cycle

from hail.expr import aggregators
from hail.expr.expressions import *
from hail.expr.expressions import Expression
from hail.typecheck import *
import hail

palette = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']

@typecheck(data=oneof(hail.utils.struct.Struct, expr_float64), range=nullable(sized_tupleof(numeric, numeric)),
           bins=int, legend=nullable(str), title=nullable(str))
def histogram(data, range=None, bins=50, legend=None, title=None):
    """Create a histogram.

    Parameters
    ----------
    data : :class:`.Struct` or :class:`.Float64Expression`
        Sequence of data to plot.
    range : Tuple[float]
        Range of x values in the histogram.
    bins : int
        Number of bins in the histogram.
    legend : str
        Label of data on the x-axis.
    title : str
        Title of the histogram.

    Returns
    -------
    :class:`bokeh.plotting.figure.Figure`
    """
    if isinstance(data, Expression):
        if data._indices.source is not None:
            agg_f = data._aggregation_method()
            if range is not None:
                start = range[0]
                end = range[1]
            else:
                start, end = agg_f((aggregators.min(data), aggregators.max(data)))
            data = agg_f(aggregators.hist(data, start, end, bins))
        else:
            return ValueError('Invalid input')

    p = figure(title=title, x_axis_label=legend, y_axis_label='Frequency', background_fill_color='#EEEEEE')
    p.quad(
        bottom=0, top=data.bin_freq,
        left=data.bin_edges[:-1], right=data.bin_edges[1:],
        legend=legend, line_color='black')
    if data.n_larger > 0:
        p.quad(
            bottom=0, top=data.n_larger,
            left=data.bin_edges[-1], right=(data.bin_edges[-1] + (data.bin_edges[1] - data.bin_edges[0])),
            line_color='black', fill_color='green', legend='Large Outliers')
    if data.n_smaller > 0:
        p.quad(
            bottom=0, top=data.n_smaller,
            left=data.bin_edges[0] - (data.bin_edges[1] - data.bin_edges[0]), right=data.bin_edges[0],
            line_color='black', fill_color='red', legend='Small Outliers')
    return p

@typecheck(data=oneof(hail.utils.struct.Struct, expr_float64), range=nullable(sized_tupleof(numeric, numeric)),
           bins=int, legend=nullable(str), title=nullable(str), normalize=bool, log=bool)
def cumulative_histogram(data, range=None, bins=50, legend=None, title=None, normalize=True, log=False):
    """Create a cumulative histogram.

    Parameters
    ----------
    data : :class:`.Struct` or :class:`.Float64Expression`
        Sequence of data to plot.
    range : Tuple[float]
        Range of x values in the histogram.
    bins : int
        Number of bins in the histogram.
    legend : str
        Label of data on the x-axis.
    title : str
        Title of the histogram.
    normalize: bool
        Whether or not the cumulative data should be normalized.
    log: bool
        Whether or not the y-axis should be of type log.

    Returns
    -------
    :class:`bokeh.plotting.figure.Figure`
    """
    if isinstance(data, Expression):
        if data._indices.source is not None:
            agg_f = data._aggregation_method()
            if range is not None:
                start = range[0]
                end = range[1]
            else:
                start, end = agg_f((aggregators.min(data), aggregators.max(data)))
            data = agg_f(aggregators.hist(data, start, end, bins))
        else:
            return ValueError('Invalid input')

    cumulative_data = np.cumsum(data.bin_freq) + data.n_smaller
    np.append(cumulative_data, [cumulative_data[-1] + data.n_larger])
    num_data_points = max(cumulative_data)

    if normalize:
        cumulative_data = cumulative_data / num_data_points
    if title is not None:
        title = f'{title} ({num_data_points:,} data points)'
    if log:
        p = figure(title=title, x_axis_label=legend, y_axis_label='Frequency',
                   background_fill_color='#EEEEEE', y_axis_type='log')
    else:
        p = figure(title=title, x_axis_label=legend, y_axis_label='Frequency', background_fill_color='#EEEEEE')
    p.line(data.bin_edges[:-1], cumulative_data, line_color='#036564', line_width=3)
    return p

@typecheck(x=oneof(sequenceof(numeric), expr_float64), y=oneof(sequenceof(numeric), expr_float64),
           label=oneof(nullable(str), expr_str, sequenceof(str)), title=nullable(str),
           xlabel=nullable(str), ylabel=nullable(str), size=int, legend=bool,
           source_fields=nullable(dictof(str, sequenceof(anytype))), collect_all=nullable(bool), n_divisions=int)
def scatter(x, y, label=None, title=None, xlabel=None, ylabel=None, size=4, legend=True,
            collect_all=False, n_divisions=500, source_fields=None):
    """Create a scatterplot.

    Parameters
    ----------
    x : List[float] or :class:`.Float64Expression`
        List of x-values to be plotted.
    y : List[float] or :class:`.Float64Expression`
        List of y-values to be plotted.
    label : List[str] or :class:`.StringExpression`
        List of labels for x and y values, used to assign each point a label (e.g. population)
    title : str
        Title of the scatterplot.
    xlabel : str
        X-axis label.
    ylabel : str
        Y-axis label.
    size : int
        Size of markers in screen space units.
    legend : bool
        Whether or not to show the legend in the resulting figure.
    collect_all : bool
        Whether to collect all values or downsample before plotting.
        This parameter will be ignored if x and y are Python objects.
    n_divisions : int
        Factor by which to downsample (default value = 500). A lower input results in fewer output datapoints.
    source_fields : Dict[str, List[Any]]
        Extra fields for the ColumnDataSource of the plot.

    Returns
    -------
    :class:`bokeh.plotting.figure.Figure`
    """
    if isinstance(x, Expression) and isinstance(y, Expression):
        agg_f = x._aggregation_method()
        if isinstance(label, Expression):
            if collect_all:
                res = hail.tuple([x, y, label]).collect()
                label = [point[2] for point in res]
            else:
                res = agg_f(aggregators.downsample(x, y, label=label, n_divisions=n_divisions))
                label = [point[2][0] for point in res]

            x = [point[0] for point in res]
            y = [point[1] for point in res]
        else:
            if collect_all:
                res = hail.tuple([x, y]).collect()
            else:
                res = agg_f(aggregators.downsample(x, y, n_divisions=n_divisions))

            x = [point[0] for point in res]
            y = [point[1] for point in res]
    elif isinstance(x, Expression) or isinstance(y, Expression):
        raise TypeError('Invalid input: x and y must both be either Expressions or Python Lists.')
    else:
        if isinstance(label, Expression):
            label = label.collect()

    p = figure(title=title, x_axis_label=xlabel, y_axis_label=ylabel, background_fill_color='#EEEEEE')
    if label is not None:
        fields = dict(x=x, y=y, label=label)
        if source_fields is not None:
            for key, values in source_fields.items():
                fields[key] = values

        source = ColumnDataSource(fields)

        if legend:
            leg = 'label'
        else:
            leg = None

        factors = list(set(label))
        if len(factors) > len(palette):
            color_gen = cycle(palette)
            colors = []
            for i in range(0, len(factors)):
                colors.append(next(color_gen))
        else:
            colors = palette[0:len(factors)]

        color_mapper = CategoricalColorMapper(factors=factors, palette=colors)
        p.circle('x', 'y', alpha=0.5, source=source, size=size,
                 color={'field': 'label', 'transform': color_mapper}, legend=leg)
    else:
        p.circle(x, y, alpha=0.5, size=size)
    return p

@typecheck(pvals=oneof(sequenceof(numeric), expr_float64), collect_all=bool, n_divisions=int)
def qq(pvals, collect_all=False, n_divisions=500):
    """Create a Quantile-Quantile plot. (https://en.wikipedia.org/wiki/Q-Q_plot)

    Parameters
    ----------
    pvals : List[float] or :class:`.Float64Expression`
        P-values to be plotted.
    collect_all : bool
        Whether to collect all values or downsample before plotting.
        This parameter will be ignored if pvals is a Python object.
    n_divisions : int
        Factor by which to downsample (default value = 500). A lower input results in fewer output datapoints.

    Returns
    -------
    :class:`bokeh.plotting.figure.Figure`
    """
    if isinstance(pvals, Expression):
        if pvals._indices.source is not None:
            if collect_all:
                pvals = pvals.collect()
                spvals = sorted(filter(lambda x: x and not(isnan(x)), pvals))
                exp = [-log(float(i) / len(spvals), 10) for i in np.arange(1, len(spvals) + 1, 1)]
                obs = [-log(p, 10) for p in spvals]
            else:
                ht = pvals._indices.source.select_rows(pval=pvals).rows()
                n = ht.count()
                ht = ht.order_by('pval').add_index()
                ht = ht.annotate(expected_p=(ht.idx + 1) / n)
                pvals = ht.aggregate(
                    aggregators.downsample(-hail.log10(ht.expected_p), -hail.log10(ht.pval), n_divisions=n_divisions))
                exp = [point[0] for point in pvals if not isnan(point[1])]
                obs = [point[1] for point in pvals if not isnan(point[1])]
        else:
            return ValueError('Invalid input')
    else:
        spvals = sorted(filter(lambda x: x and not(isnan(x)), pvals))
        exp = [-log(float(i) / len(spvals), 10) for i in np.arange(1, len(spvals) + 1, 1)]
        obs = [-log(p, 10) for p in spvals]

    p = figure(
        title='Q-Q Plot',
        x_axis_label='Expected p-value (-log10 scale)',
        y_axis_label='Observed p-value (-log10 scale)')
    p.scatter(x=exp, y=obs, color='black')
    bound = max(max(exp), max(obs)) * 1.1
    p.line([0, bound], [0, bound], color='red')
    return p

@typecheck(pvals=expr_float64, locus=nullable(expr_locus()), title=nullable(str),
           size=int, hover_fields=nullable(dictof(str, expr_any)), collect_all=bool, n_divisions=int)
def manhattan(pvals, locus=None, title=None, size=4, hover_fields=None, collect_all=False, n_divisions=500):
    """Create a Manhattan plot. (https://en.wikipedia.org/wiki/Manhattan_plot)

    Parameters
    ----------
    pvals : :class:`.Float64Expression`
        P-values to be plotted.
    locus : :class:`.LocusExpression`
        Locus values to be plotted.
    title : str
        Title of the plot.
    size : int
        Size of markers in screen space units.
    hover_fields : Dict[str, :class:`.Expression`]
        Dictionary of field names and values to be shown in the HoverTool of the plot.
    collect_all : bool
        Whether to collect all values or downsample before plotting.
    n_divisions : int
        Factor by which to downsample (default value = 500). A lower input results in fewer output datapoints.

    Returns
    -------
    :class:`bokeh.plotting.figure.Figure`
    """
    def get_contig_index(x, starts):
        left = 0
        right = len(starts) - 1
        while left <= right:
            mid = (left + right) // 2
            if x < starts[mid]:
                if x >= starts[mid - 1]:
                    return mid - 1
                right = mid
            elif x >= starts[mid+1]:
                left = mid + 1
            else:
                return mid

    pvals = -hail.log10(pvals)

    if locus is None:
        locus = pvals._indices.source.locus

    if hover_fields is None:
        hover_fields = {}

    hover_fields['locus'] = hail.str(locus)

    if collect_all:
        res = hail.tuple([locus.global_position(), pvals, hail.struct(**hover_fields)]).collect()
        hf_struct = [point[2] for point in res]
        for key in hover_fields:
            hover_fields[key] = [item[key] for item in hf_struct]
    else:
        agg_f = pvals._aggregation_method()
        res = agg_f(aggregators.downsample(locus.global_position(), pvals,
                                           label=hail.array([hail.str(x) for x in hover_fields.values()]),
                                           n_divisions=n_divisions))
        fields = [point[2] for point in res]
        for idx, key in enumerate(list(hover_fields.keys())):
            hover_fields[key] = [field[idx] for field in fields]

    x = [point[0] for point in res]
    y = [point[1] for point in res]

    ref = locus.dtype.reference_genome

    total_pos = 0
    start_points = []
    for i in range(0, len(ref.contigs)):
        start_points.append(total_pos)
        total_pos += ref.lengths.get(ref.contigs[i])
    start_points.append(total_pos)  # end point of all contigs

    observed_contigs = set()
    label = []
    for element in x:
        contig_index = get_contig_index(element, start_points)
        label.append(str(contig_index % 2))
        observed_contigs.add(ref.contigs[contig_index])

    labels = ref.contigs.copy()
    num_deleted = 0
    mid_points = []
    for i in range(0, len(ref.contigs)):
        if ref.contigs[i] in observed_contigs:
            length = ref.lengths.get(ref.contigs[i])
            mid = start_points[i] + length / 2
            if mid % 1 == 0:
                mid += 0.5
            mid_points.append(mid)
        else:
            del labels[i - num_deleted]
            num_deleted += 1

    p = scatter(x, y, label=label, title=title, xlabel='Chromosome', ylabel='P-value (-log10 scale)',
                size=size, legend=False, source_fields=hover_fields)

    p.xaxis.ticker = mid_points
    p.xaxis.major_label_overrides = dict(zip(mid_points, labels))
    p.width = 1000

    tooltips = [(key, "@{}".format(key)) for key in hover_fields]
    tooltips.append(tuple(('p-value', "$y")))
    p.add_tools(HoverTool(
        tooltips=tooltips
    ))

    return p
<commit_msg>Change histogram legend (#4143)<commit_after>from math import log, isnan

import numpy as np
from bokeh.models import *
from bokeh.plotting import figure
from itertools import cycle

from hail.expr import aggregators
from hail.expr.expressions import *
from hail.expr.expressions import Expression
from hail.typecheck import *
import hail

palette = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']

@typecheck(data=oneof(hail.utils.struct.Struct, expr_float64), range=nullable(sized_tupleof(numeric, numeric)),
           bins=int, legend=nullable(str), title=nullable(str))
def histogram(data, range=None, bins=50, legend=None, title=None):
    """Create a histogram.

    Parameters
    ----------
    data : :class:`.Struct` or :class:`.Float64Expression`
        Sequence of data to plot.
    range : Tuple[float]
        Range of x values in the histogram.
    bins : int
        Number of bins in the histogram.
    legend : str
        Label of data on the x-axis.
    title : str
        Title of the histogram.

    Returns
    -------
    :class:`bokeh.plotting.figure.Figure`
    """
    if isinstance(data, Expression):
        if data._indices.source is not None:
            agg_f = data._aggregation_method()
            if range is not None:
                start = range[0]
                end = range[1]
            else:
                start, end = agg_f((aggregators.min(data), aggregators.max(data)))
            data = agg_f(aggregators.hist(data, start, end, bins))
        else:
            return ValueError('Invalid input')

    p = figure(title=title, x_axis_label=legend, y_axis_label='Frequency', background_fill_color='#EEEEEE')
    p.quad(
        bottom=0, top=data.bin_freq,
        left=data.bin_edges[:-1], right=data.bin_edges[1:],
        legend=legend, line_color='black')
    if data.n_larger > 0:
        p.quad(
            bottom=0, top=data.n_larger,
            left=data.bin_edges[-1], right=(data.bin_edges[-1] + (data.bin_edges[1] - data.bin_edges[0])),
            line_color='black', fill_color='green', legend='Outliers Above')
    if data.n_smaller > 0:
        p.quad(
            bottom=0, top=data.n_smaller,
            left=data.bin_edges[0] - (data.bin_edges[1] - data.bin_edges[0]), right=data.bin_edges[0],
            line_color='black', fill_color='red', legend='Outliers Below')
    return p

@typecheck(data=oneof(hail.utils.struct.Struct, expr_float64), range=nullable(sized_tupleof(numeric, numeric)),
           bins=int, legend=nullable(str), title=nullable(str), normalize=bool, log=bool)
def cumulative_histogram(data, range=None, bins=50, legend=None, title=None, normalize=True, log=False):
    """Create a cumulative histogram.

    Parameters
    ----------
    data : :class:`.Struct` or :class:`.Float64Expression`
        Sequence of data to plot.
    range : Tuple[float]
        Range of x values in the histogram.
    bins : int
        Number of bins in the histogram.
    legend : str
        Label of data on the x-axis.
    title : str
        Title of the histogram.
    normalize: bool
        Whether or not the cumulative data should be normalized.
    log: bool
        Whether or not the y-axis should be of type log.

    Returns
    -------
    :class:`bokeh.plotting.figure.Figure`
    """
    if isinstance(data, Expression):
        if data._indices.source is not None:
            agg_f = data._aggregation_method()
            if range is not None:
                start = range[0]
                end = range[1]
            else:
                start, end = agg_f((aggregators.min(data), aggregators.max(data)))
            data = agg_f(aggregators.hist(data, start, end, bins))
        else:
            return ValueError('Invalid input')

    cumulative_data = np.cumsum(data.bin_freq) + data.n_smaller
    np.append(cumulative_data, [cumulative_data[-1] + data.n_larger])
    num_data_points = max(cumulative_data)

    if normalize:
        cumulative_data = cumulative_data / num_data_points
    if title is not None:
        title = f'{title} ({num_data_points:,} data points)'
    if log:
        p = figure(title=title, x_axis_label=legend, y_axis_label='Frequency',
                   background_fill_color='#EEEEEE', y_axis_type='log')
    else:
        p = figure(title=title, x_axis_label=legend, y_axis_label='Frequency', background_fill_color='#EEEEEE')
    p.line(data.bin_edges[:-1], cumulative_data, line_color='#036564', line_width=3)
    return p

@typecheck(x=oneof(sequenceof(numeric), expr_float64), y=oneof(sequenceof(numeric), expr_float64),
           label=oneof(nullable(str), expr_str, sequenceof(str)), title=nullable(str),
           xlabel=nullable(str), ylabel=nullable(str), size=int, legend=bool,
           source_fields=nullable(dictof(str, sequenceof(anytype))), collect_all=nullable(bool), n_divisions=int)
def scatter(x, y, label=None, title=None, xlabel=None, ylabel=None, size=4, legend=True,
            collect_all=False, n_divisions=500, source_fields=None):
    """Create a scatterplot.

    Parameters
    ----------
    x : List[float] or :class:`.Float64Expression`
        List of x-values to be plotted.
    y : List[float] or :class:`.Float64Expression`
        List of y-values to be plotted.
    label : List[str] or :class:`.StringExpression`
        List of labels for x and y values, used to assign each point a label (e.g. population)
    title : str
        Title of the scatterplot.
    xlabel : str
        X-axis label.
    ylabel : str
        Y-axis label.
    size : int
        Size of markers in screen space units.
    legend : bool
        Whether or not to show the legend in the resulting figure.
    collect_all : bool
        Whether to collect all values or downsample before plotting.
        This parameter will be ignored if x and y are Python objects.
    n_divisions : int
        Factor by which to downsample (default value = 500). A lower input results in fewer output datapoints.
    source_fields : Dict[str, List[Any]]
        Extra fields for the ColumnDataSource of the plot.

    Returns
    -------
    :class:`bokeh.plotting.figure.Figure`
    """
    if isinstance(x, Expression) and isinstance(y, Expression):
        agg_f = x._aggregation_method()
        if isinstance(label, Expression):
            if collect_all:
                res = hail.tuple([x, y, label]).collect()
                label = [point[2] for point in res]
            else:
                res = agg_f(aggregators.downsample(x, y, label=label, n_divisions=n_divisions))
                label = [point[2][0] for point in res]

            x = [point[0] for point in res]
            y = [point[1] for point in res]
        else:
            if collect_all:
                res = hail.tuple([x, y]).collect()
            else:
                res = agg_f(aggregators.downsample(x, y, n_divisions=n_divisions))

            x = [point[0] for point in res]
            y = [point[1] for point in res]
    elif isinstance(x, Expression) or isinstance(y, Expression):
        raise TypeError('Invalid input: x and y must both be either Expressions or Python Lists.')
    else:
        if isinstance(label, Expression):
            label = label.collect()

    p = figure(title=title, x_axis_label=xlabel, y_axis_label=ylabel, background_fill_color='#EEEEEE')
    if label is not None:
        fields = dict(x=x, y=y, label=label)
        if source_fields is not None:
            for key, values in source_fields.items():
                fields[key] = values

        source = ColumnDataSource(fields)

        if legend:
            leg = 'label'
        else:
            leg = None

        factors = list(set(label))
        if len(factors) > len(palette):
            color_gen = cycle(palette)
            colors = []
            for i in range(0, len(factors)):
                colors.append(next(color_gen))
        else:
            colors = palette[0:len(factors)]

        color_mapper = CategoricalColorMapper(factors=factors, palette=colors)
        p.circle('x', 'y', alpha=0.5, source=source, size=size,
                 color={'field': 'label', 'transform': color_mapper}, legend=leg)
    else:
        p.circle(x, y, alpha=0.5, size=size)
    return p

@typecheck(pvals=oneof(sequenceof(numeric), expr_float64), collect_all=bool, n_divisions=int)
def qq(pvals, collect_all=False, n_divisions=500):
    """Create a Quantile-Quantile plot. (https://en.wikipedia.org/wiki/Q-Q_plot)

    Parameters
    ----------
    pvals : List[float] or :class:`.Float64Expression`
        P-values to be plotted.
    collect_all : bool
        Whether to collect all values or downsample before plotting.
        This parameter will be ignored if pvals is a Python object.
    n_divisions : int
        Factor by which to downsample (default value = 500). A lower input results in fewer output datapoints.

    Returns
    -------
    :class:`bokeh.plotting.figure.Figure`
    """
    if isinstance(pvals, Expression):
        if pvals._indices.source is not None:
            if collect_all:
                pvals = pvals.collect()
                spvals = sorted(filter(lambda x: x and not(isnan(x)), pvals))
                exp = [-log(float(i) / len(spvals), 10) for i in np.arange(1, len(spvals) + 1, 1)]
                obs = [-log(p, 10) for p in spvals]
            else:
                ht = pvals._indices.source.select_rows(pval=pvals).rows()
                n = ht.count()
                ht = ht.order_by('pval').add_index()
                ht = ht.annotate(expected_p=(ht.idx + 1) / n)
                pvals = ht.aggregate(
                    aggregators.downsample(-hail.log10(ht.expected_p), -hail.log10(ht.pval), n_divisions=n_divisions))
                exp = [point[0] for point in pvals if not isnan(point[1])]
                obs = [point[1] for point in pvals if not isnan(point[1])]
        else:
            return ValueError('Invalid input')
    else:
        spvals = sorted(filter(lambda x: x and not(isnan(x)), pvals))
        exp = [-log(float(i) / len(spvals), 10) for i in np.arange(1, len(spvals) + 1, 1)]
        obs = [-log(p, 10) for p in spvals]

    p = figure(
        title='Q-Q Plot',
        x_axis_label='Expected p-value (-log10 scale)',
        y_axis_label='Observed p-value (-log10 scale)')
    p.scatter(x=exp, y=obs, color='black')
    bound = max(max(exp), max(obs)) * 1.1
    p.line([0, bound], [0, bound], color='red')
    return p

@typecheck(pvals=expr_float64, locus=nullable(expr_locus()), title=nullable(str),
           size=int, hover_fields=nullable(dictof(str, expr_any)), collect_all=bool, n_divisions=int)
def manhattan(pvals, locus=None, title=None, size=4, hover_fields=None, collect_all=False, n_divisions=500):
    """Create a Manhattan plot. (https://en.wikipedia.org/wiki/Manhattan_plot)

    Parameters
    ----------
    pvals : :class:`.Float64Expression`
        P-values to be plotted.
    locus : :class:`.LocusExpression`
        Locus values to be plotted.
    title : str
        Title of the plot.
    size : int
        Size of markers in screen space units.
    hover_fields : Dict[str, :class:`.Expression`]
        Dictionary of field names and values to be shown in the HoverTool of the plot.
    collect_all : bool
        Whether to collect all values or downsample before plotting.
    n_divisions : int
        Factor by which to downsample (default value = 500). A lower input results in fewer output datapoints.

    Returns
    -------
    :class:`bokeh.plotting.figure.Figure`
    """
    def get_contig_index(x, starts):
        left = 0
        right = len(starts) - 1
        while left <= right:
            mid = (left + right) // 2
            if x < starts[mid]:
                if x >= starts[mid - 1]:
                    return mid - 1
                right = mid
            elif x >= starts[mid+1]:
                left = mid + 1
            else:
                return mid

    pvals = -hail.log10(pvals)

    if locus is None:
        locus = pvals._indices.source.locus

    if hover_fields is None:
        hover_fields = {}

    hover_fields['locus'] = hail.str(locus)

    if collect_all:
        res = hail.tuple([locus.global_position(), pvals, hail.struct(**hover_fields)]).collect()
        hf_struct = [point[2] for point in res]
        for key in hover_fields:
            hover_fields[key] = [item[key] for item in hf_struct]
    else:
        agg_f = pvals._aggregation_method()
        res = agg_f(aggregators.downsample(locus.global_position(), pvals,
                                           label=hail.array([hail.str(x) for x in hover_fields.values()]),
                                           n_divisions=n_divisions))
        fields = [point[2] for point in res]
        for idx, key in enumerate(list(hover_fields.keys())):
            hover_fields[key] = [field[idx] for field in fields]

    x = [point[0] for point in res]
    y = [point[1] for point in res]

    ref = locus.dtype.reference_genome

    total_pos = 0
    start_points = []
    for i in range(0, len(ref.contigs)):
        start_points.append(total_pos)
        total_pos += ref.lengths.get(ref.contigs[i])
    start_points.append(total_pos)  # end point of all contigs

    observed_contigs = set()
    label = []
    for element in x:
        contig_index = get_contig_index(element, start_points)
        label.append(str(contig_index % 2))
        observed_contigs.add(ref.contigs[contig_index])

    labels = ref.contigs.copy()
    num_deleted = 0
    mid_points = []
    for i in range(0, len(ref.contigs)):
        if ref.contigs[i] in observed_contigs:
            length = ref.lengths.get(ref.contigs[i])
            mid = start_points[i] + length / 2
            if mid % 1 == 0:
                mid += 0.5
            mid_points.append(mid)
        else:
            del labels[i - num_deleted]
            num_deleted += 1

    p = scatter(x, y, label=label, title=title, xlabel='Chromosome', ylabel='P-value (-log10 scale)',
                size=size, legend=False, source_fields=hover_fields)

    p.xaxis.ticker = mid_points
    p.xaxis.major_label_overrides = dict(zip(mid_points, labels))
    p.width = 1000

    tooltips = [(key, "@{}".format(key)) for key in hover_fields]
    tooltips.append(tuple(('p-value', "$y")))
    p.add_tools(HoverTool(
        tooltips=tooltips
    ))

    return p
<|endoftext|>


<commit_before><commit_msg>Add TestCase to mockie __init__<commit_after>from .test_case import TestCase
<|endoftext|>


<commit_before>"""
Fitbit data processing.

Copyright (C) 2016 PersonalGenomes.org

This software is shared under the "MIT License" license (aka "Expat License"),
see LICENSE.TXT for full license text.

May be used on the command line from this project's base directory, e.g.

   foreman run python -m sources.fitbit <access token> files/

...where <access token> is the private token the Fitbit API has created that
grants permission to access a user's data. (Keep it safe!) This will assemble
a data set for the user in that directory:

   files/fitbit-storyline-data.tar.gz
"""

from __future__ import unicode_literals

import json
import os
import sys
import tempfile
import time

import arrow
import requests

from data_retrieval.files import mv_tempfile_to_output
from models import CacheItem
from utilities import get_fresh_token

if __name__ == '__main__':
    from utilities import init_db

    db = init_db()
else:
    from models import db  # noqa

fitbit_urls = [
    # general user data
    {'name': 'profile', 'url': '/-/profile.json', 'period': None},

    {'name': 'devices', 'url': '/-/devices.json', 'period': None},

    {'name': 'activities-overview',
     'url': '/{user_id}/activities.json',
     'period': None},

    # interday timeline data
    {'name': 'heart',
     'url': '/{user_id}/activities/heart/date/{date}/1m.json',
     'period': 'month'},
    {'name': 'tracker-activity-calories',
     'url': '/{user_id}/activities/tracker/activityCalories/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-calories',
     'url': '/{user_id}/activities/tracker/calories/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-distance',
     'url': '/{user_id}/activities/tracker/distance/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-elevation',
     'url': '/{user_id}/activities/tracker/elevation/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-floors',
     'url': '/{user_id}/activities/tracker/floors/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-minutes-fairly-active',
     'url': '/{user_id}/activities/tracker/minutesFairlyActive/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-minutes-lightly-active',
     'url': '/{user_id}/activities/tracker/minutesLightlyActive/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-minutes-sedentary',
     'url': '/{user_id}/activities/tracker/minutesSedentary/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-minutes-very-active',
     'url': '/{user_id}/activities/tracker/minutesVeryActive/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-steps',
     'url': '/{user_id}/activities/tracker/steps/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'weight-log',
     'url': '/{user_id}/body/log/weight/date/{date}/1m.json',
     'period': 'month'},
    {'name': 'weight',
     'url': '/{user_id}/body/weight/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'sleep-awakenings',
     'url': '/{user_id}/sleep/awakeningsCount/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'sleep-efficiency',
     'url': '/{user_id}/sleep/efficiency/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'sleep-minutes-after-wakeup',
     'url': '/{user_id}/sleep/minutesAfterWakeup/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'sleep-minutes',
     'url': '/{user_id}/sleep/minutesAsleep/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'awake-minutes',
     'url': '/{user_id}/sleep/minutesAwake/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'minutes-to-sleep',
     'url': '/{user_id}/sleep/minutesToFallAsleep/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'sleep-start-time',
     'url': '/{user_id}/sleep/startTime/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'time-in-bed',
     'url': '/{user_id}/sleep/timeInBed/date/{date}/1y.json',
     'period': 'year'},

    # intraday timeline data
    {'name': 'intraday-heart',
     'url': '/-/activities/heart/date/{date}/1d/1sec.json',
     'period': 'day'},
    {'name': 'intraday-steps',
     'url': '/-/activities/tracker/steps/date/{date}/1d/1min.json',
     'period': 'day'},
]

class RateCapException(Exception):
    """
    An exception that is raised if the Fitbit API tells us we've reached the
    request rate cap.
    """

    pass

def fitbit_query(access_token, path, parameters=None):
    """
    Query Fitbit API and return result.
    """
    if not parameters:
        parameters = {}

    headers = {
        'Authorization': 'Bearer %s' % access_token,
        # Required for American units (miles, pounds)
        'Accept-Language': 'en_US',
    }

    path = path.format(**parameters)
    data_url = 'https://api.fitbit.com/1/user{}'.format(path)
    data_key = '{}{}'.format(data_url, access_token)

    cached_response = (CacheItem.query
                       .filter_by(key=data_key)
                       .order_by(CacheItem.request_time.desc())
                       .first())

    if cached_response:
        return cached_response.response

    data_response = requests.get(data_url, headers=headers)

    if 'errors' in data_response:
        raise Exception(data_response['errors'])

    # If a rate cap is encountered, return a result reporting this.
    if data_response.status_code == 429:
        raise RateCapException()

    query_result = data_response.json()

    db.session.add(CacheItem(data_key, query_result))
    db.session.commit()

    return query_result

def get_fitbit_data(access_token):
    """
    Iterate to get all items for a given access_token and path, return result.

    Result is a dict with the following keys:
        'all_data': data from all items, or None if rate cap hit.
        'rate_cap_encountered': None, or True if rate cap hit.
    """
    fitbit_data = {}
    user_id = None

    try:
        for url in [url for url in fitbit_urls if url['period'] is None]:
            if not user_id and 'profile' in fitbit_data:
                user_id = fitbit_data['profile']['user']['encodedId']

            query_result = fitbit_query(access_token=access_token,
                                        path=url['url'],
                                        parameters={'user_id': user_id})

            fitbit_data[url['name']] = query_result

        for url in [url for url in fitbit_urls if url['period'] == 'year']:
            member_since = fitbit_data['profile']['user']['memberSince']
            start_year = arrow.get(member_since, 'YYYY-MM-DD').year
            current_year = arrow.get().year

            for year in xrange(start_year, current_year + 1):
                print 'retrieving {}: {}'.format(url['name'], year)

                fitbit_data[url['name']] = []

                query_result = fitbit_query(
                    access_token=access_token,
                    path=url['url'],
                    parameters={
                        'user_id': user_id,
                        'date': '{}-01-01'.format(year),
                    })

                fitbit_data[url['name']].append(query_result)

        for url in [url for url in fitbit_urls if url['period'] == 'month']:
            member_since = fitbit_data['profile']['user']['memberSince']

            start_date = arrow.get(member_since, 'YYYY-MM-DD')
            today_date = arrow.get()

            dates = []

            for year in xrange(start_date.year, today_date.year + 1):
                start_month = 1
                end_month = 12

                if year == start_date.year == today_date.year:
                    start_month = start_date.month
                    end_month = today_date.month
                elif year == start_date.year:
                    start_month = start_date.month
                elif year == today_date.year:
                    end_month = today_date.month

                dates += [(year, month) for month
                          in range(start_month, end_month + 1)]

            for year, month in dates:
                print 'retrieving {}: {}, {}'.format(url['name'], year, month)

                fitbit_data[url['name']] = []

                query_result = fitbit_query(
                    access_token=access_token,
                    path=url['url'],
                    parameters={
                        'user_id': user_id,
                        'date': '{}-{:02d}-01'.format(year, month),
                    })

                fitbit_data[url['name']].append(query_result)

        # TODO: implement these once we're approved for Fitbit intraday access
        for url in [url for url in fitbit_urls if url['period'] == 'day']:
            pass
    except RateCapException:
        return {
            'all_data': fitbit_data,
            'rate_cap_encountered': True,
        }

    return {
        'all_data': fitbit_data,
        'rate_cap_encountered': False,
    }

def create_datafiles(user_id, task_id=None, update_url=None, **kwargs):
    """
    Create Open Humans Dataset from Fitbit API data

    Required arguments:
        user_id: Open Humans user ID

    Optional arguments:
        filedir: Local filepath, folder in which to place the resulting file.
        s3_bucket_name: S3 bucket to write resulting file.
        s3_key_dir: S3 key "directory" to write resulting file. The full S3 key
                    name will add a filename to the end of s3_key_dir.

    Either 'filedir' (and no S3 arguments), or both S3 arguments (and no
    'filedir') must be specified.
    """
    tempdir = tempfile.mkdtemp()
    filename = 'fitbit-data.json'
    filepath = os.path.join(tempdir, filename)
    data_files = []

    temp_files = [{
        'temp_filename': filename,
        'tempdir': tempdir,
        'metadata': {
            'description': ('Fitbit activity, health, and fitness data.'),
            'tags': ['weight', 'Fitbit', 'steps', 'activity'],
        }
    }]

    access_token = get_fresh_token(user_id, provider='fitbit')

    fitbit_data = get_fitbit_data(access_token)

    if fitbit_data['rate_cap_encountered']:
        # If this is previously called and we got no new data this round,
        # double the wait period for resubmission.
        if 'return_status' in kwargs and not fitbit_data['all_data']:
            countdown = 2 * kwargs['return_status']['countdown']
        else:
            countdown = 60

        return {'countdown': countdown}

    user_data = fitbit_data['all_data']

    with open(filepath, 'w') as f:
        json.dump(user_data, f, indent=2)

    print 'Finished creating Fitbit dataset locally.'

    for file_info in temp_files:
        print 'File info: {}'.format(str(file_info))

        filename = file_info['temp_filename']
        file_tempdir = file_info['tempdir']

        output_path = mv_tempfile_to_output(
            os.path.join(file_tempdir, filename), filename, **kwargs)

        if 's3_key_dir' in kwargs and 's3_bucket_name' in kwargs:
            data_files.append({
                's3_key': output_path,
                'metadata': file_info['metadata'],
            })

    os.rmdir(tempdir)

    print 'Finished moving Fitbit data to permanent storage.'

    if not (task_id and update_url):
        return

    task_data = {'task_id': task_id,
                 's3_keys': [df['s3_key'] for df in data_files],
                 'data_files': data_files}

    status_msg = ('Updating main site ({}) with completed files for task_id={}'
                  ' with task_data:\n{}'.format(
                      update_url, task_id, json.dumps(task_data)))

    print status_msg

    requests.post(update_url, json={'task_data': task_data})

def cli_get_data():
    if len(sys.argv) != 3:
        print 'Please specify a token and directory.'

        sys.exit(1)

    while True:
        result = create_datafiles(*sys.argv[1:-1], filedir=sys.argv[-1])

        if result:
            countdown = result['countdown']

            print('Rate cap hit. Pausing {}s before resuming...'.format(
                countdown))

            time.sleep(countdown)
        else:
            break

if __name__ == '__main__':
    cli_get_data()
<commit_msg>change cache key to user ID<commit_after>"""
Fitbit data processing.

Copyright (C) 2016 PersonalGenomes.org

This software is shared under the "MIT License" license (aka "Expat License"),
see LICENSE.TXT for full license text.

May be used on the command line from this project's base directory, e.g.

   foreman run python -m sources.fitbit <access token> files/

...where <access token> is the private token the Fitbit API has created that
grants permission to access a user's data. (Keep it safe!) This will assemble
a data set for the user in that directory:

   files/fitbit-storyline-data.tar.gz
"""

from __future__ import unicode_literals

import json
import os
import sys
import tempfile
import time

import arrow
import requests

from data_retrieval.files import mv_tempfile_to_output
from models import CacheItem
from utilities import get_fresh_token

if __name__ == '__main__':
    from utilities import init_db

    db = init_db()
else:
    from models import db  # noqa

fitbit_urls = [
    # general user data
    {'name': 'profile', 'url': '/-/profile.json', 'period': None},

    {'name': 'devices', 'url': '/-/devices.json', 'period': None},

    {'name': 'activities-overview',
     'url': '/{user_id}/activities.json',
     'period': None},

    # interday timeline data
    {'name': 'heart',
     'url': '/{user_id}/activities/heart/date/{date}/1m.json',
     'period': 'month'},
    {'name': 'tracker-activity-calories',
     'url': '/{user_id}/activities/tracker/activityCalories/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-calories',
     'url': '/{user_id}/activities/tracker/calories/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-distance',
     'url': '/{user_id}/activities/tracker/distance/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-elevation',
     'url': '/{user_id}/activities/tracker/elevation/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-floors',
     'url': '/{user_id}/activities/tracker/floors/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-minutes-fairly-active',
     'url': '/{user_id}/activities/tracker/minutesFairlyActive/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-minutes-lightly-active',
     'url': '/{user_id}/activities/tracker/minutesLightlyActive/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-minutes-sedentary',
     'url': '/{user_id}/activities/tracker/minutesSedentary/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-minutes-very-active',
     'url': '/{user_id}/activities/tracker/minutesVeryActive/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'tracker-steps',
     'url': '/{user_id}/activities/tracker/steps/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'weight-log',
     'url': '/{user_id}/body/log/weight/date/{date}/1m.json',
     'period': 'month'},
    {'name': 'weight',
     'url': '/{user_id}/body/weight/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'sleep-awakenings',
     'url': '/{user_id}/sleep/awakeningsCount/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'sleep-efficiency',
     'url': '/{user_id}/sleep/efficiency/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'sleep-minutes-after-wakeup',
     'url': '/{user_id}/sleep/minutesAfterWakeup/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'sleep-minutes',
     'url': '/{user_id}/sleep/minutesAsleep/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'awake-minutes',
     'url': '/{user_id}/sleep/minutesAwake/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'minutes-to-sleep',
     'url': '/{user_id}/sleep/minutesToFallAsleep/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'sleep-start-time',
     'url': '/{user_id}/sleep/startTime/date/{date}/1y.json',
     'period': 'year'},
    {'name': 'time-in-bed',
     'url': '/{user_id}/sleep/timeInBed/date/{date}/1y.json',
     'period': 'year'},

    # intraday timeline data
    {'name': 'intraday-heart',
     'url': '/-/activities/heart/date/{date}/1d/1sec.json',
     'period': 'day'},
    {'name': 'intraday-steps',
     'url': '/-/activities/tracker/steps/date/{date}/1d/1min.json',
     'period': 'day'},
]

class RateCapException(Exception):
    """
    An exception that is raised if the Fitbit API tells us we've reached the
    request rate cap.
    """

    pass

def fitbit_query(access_token, path, open_humans_id, parameters=None):
    """
    Query Fitbit API and return result.
    """
    if not parameters:
        parameters = {}

    headers = {
        'Authorization': 'Bearer %s' % access_token,
        # Required for American units (miles, pounds)
        'Accept-Language': 'en_US',
    }

    path = path.format(**parameters)
    data_url = 'https://api.fitbit.com/1/user{}'.format(path)
    data_key = '{}-{}'.format(data_url, open_humans_id)

    cached_response = (CacheItem.query
                       .filter_by(key=data_key)
                       .order_by(CacheItem.request_time.desc())
                       .first())

    if cached_response:
        return cached_response.response

    data_response = requests.get(data_url, headers=headers)

    if 'errors' in data_response:
        raise Exception(data_response['errors'])

    # If a rate cap is encountered, return a result reporting this.
    if data_response.status_code == 429:
        raise RateCapException()

    query_result = data_response.json()

    db.session.add(CacheItem(data_key, query_result))
    db.session.commit()

    return query_result

def get_fitbit_data(access_token, open_humans_id):
    """
    Iterate to get all items for a given access_token and path, return result.

    Result is a dict with the following keys:
        'all_data': data from all items, or None if rate cap hit.
        'rate_cap_encountered': None, or True if rate cap hit.
    """
    fitbit_data = {}
    user_id = None

    try:
        for url in [url for url in fitbit_urls if url['period'] is None]:
            if not user_id and 'profile' in fitbit_data:
                user_id = fitbit_data['profile']['user']['encodedId']

            query_result = fitbit_query(access_token=access_token,
                                        path=url['url'],
                                        parameters={'user_id': user_id},
                                        open_humans_id=open_humans_id)

            fitbit_data[url['name']] = query_result

        for url in [url for url in fitbit_urls if url['period'] == 'year']:
            member_since = fitbit_data['profile']['user']['memberSince']
            start_year = arrow.get(member_since, 'YYYY-MM-DD').year
            current_year = arrow.get().year

            for year in xrange(start_year, current_year + 1):
                print 'retrieving {}: {}'.format(url['name'], year)

                fitbit_data[url['name']] = []

                query_result = fitbit_query(
                    access_token=access_token,
                    path=url['url'],
                    parameters={
                        'user_id': user_id,
                        'date': '{}-01-01'.format(year),
                    },
                    open_humans_id=open_humans_id)

                fitbit_data[url['name']].append(query_result)

        for url in [url for url in fitbit_urls if url['period'] == 'month']:
            member_since = fitbit_data['profile']['user']['memberSince']

            start_date = arrow.get(member_since, 'YYYY-MM-DD')
            today_date = arrow.get()

            dates = []

            for year in xrange(start_date.year, today_date.year + 1):
                start_month = 1
                end_month = 12

                if year == start_date.year == today_date.year:
                    start_month = start_date.month
                    end_month = today_date.month
                elif year == start_date.year:
                    start_month = start_date.month
                elif year == today_date.year:
                    end_month = today_date.month

                dates += [(year, month) for month
                          in range(start_month, end_month + 1)]

            for year, month in dates:
                print 'retrieving {}: {}, {}'.format(url['name'], year, month)

                fitbit_data[url['name']] = []

                query_result = fitbit_query(
                    access_token=access_token,
                    path=url['url'],
                    parameters={
                        'user_id': user_id,
                        'date': '{}-{:02d}-01'.format(year, month),
                    },
                    open_humans_id=open_humans_id)

                fitbit_data[url['name']].append(query_result)

        # TODO: implement these once we're approved for Fitbit intraday access
        for url in [url for url in fitbit_urls if url['period'] == 'day']:
            pass
    except RateCapException:
        return {
            'all_data': fitbit_data,
            'rate_cap_encountered': True,
        }

    return {
        'all_data': fitbit_data,
        'rate_cap_encountered': False,
    }

def create_datafiles(user_id, task_id=None, update_url=None, **kwargs):
    """
    Create Open Humans Dataset from Fitbit API data

    Required arguments:
        user_id: Open Humans user ID

    Optional arguments:
        filedir: Local filepath, folder in which to place the resulting file.
        s3_bucket_name: S3 bucket to write resulting file.
        s3_key_dir: S3 key "directory" to write resulting file. The full S3 key
                    name will add a filename to the end of s3_key_dir.

    Either 'filedir' (and no S3 arguments), or both S3 arguments (and no
    'filedir') must be specified.
    """
    tempdir = tempfile.mkdtemp()
    filename = 'fitbit-data.json'
    filepath = os.path.join(tempdir, filename)
    data_files = []

    temp_files = [{
        'temp_filename': filename,
        'tempdir': tempdir,
        'metadata': {
            'description': ('Fitbit activity, health, and fitness data.'),
            'tags': ['weight', 'Fitbit', 'steps', 'activity'],
        }
    }]

    access_token = get_fresh_token(user_id, provider='fitbit')

    fitbit_data = get_fitbit_data(access_token, user_id)

    if fitbit_data['rate_cap_encountered']:
        return {'countdown': 60}

    user_data = fitbit_data['all_data']

    with open(filepath, 'w') as f:
        json.dump(user_data, f, indent=2)

    print 'Finished creating Fitbit dataset locally.'

    for file_info in temp_files:
        print 'File info: {}'.format(str(file_info))

        filename = file_info['temp_filename']
        file_tempdir = file_info['tempdir']

        output_path = mv_tempfile_to_output(
            os.path.join(file_tempdir, filename), filename, **kwargs)

        if 's3_key_dir' in kwargs and 's3_bucket_name' in kwargs:
            data_files.append({
                's3_key': output_path,
                'metadata': file_info['metadata'],
            })

    os.rmdir(tempdir)

    print 'Finished moving Fitbit data to permanent storage.'

    if not (task_id and update_url):
        return

    task_data = {'task_id': task_id,
                 's3_keys': [df['s3_key'] for df in data_files],
                 'data_files': data_files}

    status_msg = ('Updating main site ({}) with completed files for task_id={}'
                  ' with task_data:\n{}'.format(
                      update_url, task_id, json.dumps(task_data)))

    print status_msg

    requests.post(update_url, json={'task_data': task_data})

def cli_get_data():
    if len(sys.argv) != 3:
        print 'Please specify a token and directory.'

        sys.exit(1)

    while True:
        result = create_datafiles(*sys.argv[1:-1], filedir=sys.argv[-1])

        if result:
            countdown = result['countdown']

            print('Rate cap hit. Pausing {}s before resuming...'.format(
                countdown))

            time.sleep(countdown)
        else:
            break

if __name__ == '__main__':
    cli_get_data()
<|endoftext|>


<commit_before>"""
Sparse matrix
=============

"""
postpone_import = 1
<commit_msg>Added some documentation for the sparse matrix package, with a simple  usage example in doctest format.<commit_after>"""
Sparse matrix
=============

Scipy 2D sparse matrix module.

Original code by Travis Oliphant.
Modified and extended by Ed Schofield and Robert Cimrman.

There are four available sparse matrix types:
    (1) csc_matrix: Compressed Sparse Column format
    (2) csr_matrix: Compressed Sparse Row format
    (3) lil_matrix: List of Lists format
    (4) dok_matrix: Dictionary of Keys format

To construct a matrix efficiently, use either lil_matrix (recommended) or
dok_matrix.  The lil_matrix class supports basic slicing and fancy
indexing with a similar syntax to NumPy arrays.

To perform manipulations such as multiplication or inversion, first
convert the matrix to either CSC or CSR format.  lil_matrix format is
row-based, so conversion to CSR very most efficient.

Example:
    Construct a 10x1000 lil_matrix and add some values to it:
    >>> from scipy import sparse, linsolve
    >>> from numpy import rand, linalg
    >>> A = sparse.lil_matrix((1000, 1000))
    >>> A[0, :100] = rand(100)
    >>> A[1, 100:200] = A[0, :100]
    >>> A.setdiag(rand(1000))

    Now convert it to CSR format and solve (A A^T) x = b for x:
    >>> A = A.tocsr()
    >>> b = rand(1000)
    >>> x = linsolve.spsolve(A * A.T, b)

    Convert it to a dense matrix and solve, and check that the result
    is the same:
    >>> A_ = A.todense()
    >>> x_ = linalg.solve(A_ * A_.T, b)
    >>> err = linalg.norm(x-x_)

    Now we can print the error norm with:
        print "Norm error =", err
    It should be small :)

"""

postpone_import = 1
<|endoftext|>


<commit_before>'''
Created on Sep 26, 2014

@author: tim
'''
import tweepy
from tweepy import OAuthHandler
import time
import itertools
import os
import cPickle as pickle
import sys
import re
import traceback
os.environ['http_proxy']= ''
import logging
import os

t0 = time.time()

DEBUG = True

config_name = 'debug.txt'

# determine if application is a script file or frozen exe
if getattr(sys, 'frozen', False):
    application_path = os.path.dirname(sys.executable)
elif __file__:
    application_path = os.path.dirname(__file__)

config_path = os.path.join(application_path, config_name)

logging.basicConfig(filename=config_path, format='%(message)s', level=logging.DEBUG)

def log(s):
    if DEBUG: logging.log(logging.DEBUG, s)

ckeys = []
csecrets = []
atokens = []
asecrets = []

ckeys.append('Go2BKKaGREBEioh1BEeRhA')
csecrets.append('lSsiczxot5t1AKKhcYzlGHNXL66B8s7Wpj1QB1oM3U')
atokens.append('872274950-ihHI1EEfLd20059n8eLBFOqvJMCMCfwNqXR9ObTh')
asecrets.append('X1ic8qTu5LcjpKFckibFH7S8sNZ7ut9rr7rc2Jt4')

ckeys.append('leXgZOVbVhG5BvdnvM58BgAmy')
csecrets.append('LAOr4rkUHTxYgdPiMeTxjOxWURgb9bWdrQog1DyRM1JQhcNokY')
atokens.append('872274950-hl11IsipRPTL1MvPrsqGapJPtjj56F5E81dlGIGf')
asecrets.append('kte8SvpQBPDVezSTAtgPHZ2zhcp9SqkiV4G11m8TQ0BhG')

ckeys.append('napyrVTJtsycJ9sTQOdQw')
csecrets.append('PJ9Gr0k17L9XrhSJgmEwJ5Pkb2L8BimMyjlpcerpc')
atokens.append('872274950-mAhGAEs7TQArb8SJjb5CRsOd3GBKpqTWqxniPXZa')
asecrets.append('AgPW8xj8PFDZcJVDK3aMWpCZw2FNV5t57jzHVZc1XQSdL')

ckeys.append('oHmPy6BDn6QPISA8O4YmSFcSn')
csecrets.append('rPWtdq811nMm1Ta4TsJXs6L2j0y5BP9kFEQ6wLZJ3zmi5ypCEt')
atokens.append('872274950-FrgfRor6nmfOn9YvIDihcIPpWxjsxL1kdBRXgDj5')
asecrets.append('s3sKetUiHS7LHvBPUWFjoGASyOQy7SzFnkm6FSxiw0Sx0')

#debug!
ckeys.append('h554bvyP67an9o98A0uZl8D5i')
csecrets.append('5hvRoKrEf1LKrCIqZU0htZpzZ6yAOm8SwV3WFFetDSvb6Tf2Sl')
atokens.append('872274950-jK8Q4sZlBo8PDJwbs6frOe7CStnygnoSn0GRP7P1')
asecrets.append('lFUmOkBEtI8vOkeIeQ2bSlo9ye5A28QpQxGfo6pAjGZu0')

ckeys.append('jrwNGHc9FsCSJHk2Z0McxYgXM')
csecrets.append('VVhEWzhgjkSRuiPs4p1DfsYIB3b38shXXonc9WXFFz70KWF32Z')
atokens.append('872274950-Xe9QzakUbYdQktLxlMgYvtt4L2ulHgy4lLPZyr2q')
asecrets.append('F2EcF3S56GEfDQJnCAiSdvTzD2Zp3SB98cFJmfjCv1jyy')

ckeys.append('jrwNGHc9FsCSJHk2Z0McxYgXM')
csecrets.append('VVhEWzhgjkSRuiPs4p1DfsYIB3b38shXXonc9WXFFz70KWF32Z')
atokens.append('872274950-Xe9QzakUbYdQktLxlMgYvtt4L2ulHgy4lLPZyr2q')
asecrets.append('F2EcF3S56GEfDQJnCAiSdvTzD2Zp3SB98cFJmfjCv1jyy')

ckeys.append('pKCZtQlg3ts27JLUTzQJK0qhT')
csecrets.append('Ojr04SvTURTrVarVUQkE9b3Q5v7N6wBKLhK48T4g5sJfSmsruI')
atokens.append('872274950-Eok1pRVwol8fGSFXfyFc7f6aZTOuGNz3e0bBMq7A')
asecrets.append('cJCrBRZs9sQkJ4sxeIEJZLsR266MwxMKFj0mALJCbvjqo')

ckeys.append('e7Q1xcHIms2kMSpJzF0uVWWxU')
csecrets.append('xTZe6shkMD2cGFg42udOkU24cOnZCJq2wFP4wtOMKHmSIBSY5x')
atokens.append('872274950-tY13UAP8kiNXVaWjn0dLrjC13JhFaj9eWeVljqzu')
asecrets.append('yJ7y9PJ3nFdw5bsQnePH5F4o05SX6fpyfMIhG6Yb6m2Tr')

ckeys.append('b5MSBg9l1THMjzVEJxGKAax7m')
csecrets.append('EzLcQRZj6p1B5z2Ox5nzv1kDwL861ZCtmK4p02km1PyNY8Fidc')
atokens.append('872274950-NUxhs2SY04oEievqDujTCEghgfsD91d2easr82KH')
asecrets.append('i9hxb96b1wIbFDHbOuZxO4oxNv9jSVlZEDooaiWRES1tr')

ckeys.append('1LSVbEQDZvv1yhq8cgnYHLYr0')
csecrets.append('VZGlznquz32k4Pjz5cGbaCZ2j0TUuuNUcDltpLMVIGTIhpDAGq')
atokens.append('872274950-buBF1nz43sgsVh9hwcsv8RoX1M8xurzgZ4bdjDyD')
asecrets.append('d84pljcCcRO7L3C2IImuPjD37LMAfYTZ7GwRAU5ubGHcU')

ckeys.append('roBbWjs34pNbc21qgzKto9mvl')
csecrets.append('MrC4atAsJSB4tIvrLZ267fPR9yvZRWkZP3gBEWnS8POzSoov0p')
atokens.append('872274950-no8Hdl2ikM20WTdT1Wa7T9WTciYGDA9UBZG2MyE5')
asecrets.append('083G8Ux5fy59YtDgXYjrTr4ZJpmFWcr50Eid0gdAhzuI0')

ckeys.append('fZH9P7N0CXu5aLLWLlSI8qmjP')
csecrets.append('Pfs0ARchnKYOaTDL8I7deOQYasioj8GPeiu8ulHqhbQbsvYGzy')
atokens.append('872274950-LhxYgTpUtLEsVAn6TAMPWAYhwEIuZEOrsFa5593I')
asecrets.append('L8aoQPkVSc7EEEgBay7Jo9qbIqonISG3bpQILcx2xYjeO')

ckeys.append('Yzn6fparxxRqL9M5hPDFSrpS8')
csecrets.append('lL8fFSYJrkeri12nPV14oN5WcK2Vy0LeDmAGuXIWWz2b5QHqlF')
atokens.append('872274950-hcrxze1FxWYFx2gZtp4TgFutUaPUZGFKdXFnkWGo')
asecrets.append('4NbbDdxFWDycPEsqXrL0QGZ92VlbBkijlJubxsy1xQhTN')

ckeys.append('3MlUKIKfo7uJ2pbfx1Oi43oSx')
csecrets.append('0o733A8OvTmsCo1Vi4iJenxtAj3zVlwB9FvCsw9hBVsdByZCyE')
atokens.append('872274950-8nbgDCcVDEkMFBLJdiUD6Vf22KM6XHVSGSsKplbK')
asecrets.append('i78B5SAy88xKab6bwO4SAf8ejyLCKlVzLK9AJJte7sK8z')

ckeys.append('jzZJjV4WaQTO0UfXhlXldqDAD')
csecrets.append('Bpac2gVzeg2JwiUuTyZcnAJ3o41o8i7iqr2NYu9px08QEVmRBG')
atokens.append('872274950-nqebPIFbyi7BgWmbT5UFiKnnLB1PNGzolXpdyed3')
asecrets.append('F0rSbgdbLv9yNNUzcLCIIWNiQaBNGB77mt9uvIABkAp37')

ckeys.append('2VPJEUQ9j1U1RBifjLzoORHPz')
csecrets.append('kTSS1DsYp5awkjCWUndEoKileEDmgLouHXixAWItCtPRaaafES')
atokens.append('872274950-OYftTzpKgcvuGaq4wqW9ZCQ9D6KIciiNef3OwmfR')
asecrets.append('oBy36n6rz8G7a6sZUOzNbeWcWCzPKFu4MuNmyZUyPRfkE')

current_api = 0

base_path = sys.argv[0].replace('hashtag_crawler.py','')
data_de = []

print base_path

def cast_status_to_dict(tweepy_status):
    ret = {}
    ret['text'] = tweepy_status.text
    ret['follower_count'] = tweepy_status.author.followers_count
    ret['friends_count'] = tweepy_status.author.friends_count
    ret['retweet_count'] = tweepy_status.retweet_count
    return ret

apis = []
for i in range(len(ckeys)):
    auth = OAuthHandler(ckeys[i], csecrets[i])
    auth.set_access_token(atokens[i], asecrets[i])
    apis.append(tweepy.API(auth))

def process_hashtag(query):
    global current_api
    global processed_hashtags
    global data_de
    processed_hashtags[query] = True
    items = tweepy.Cursor(apis[current_api].search, q=hashtag + query,lang='de').items(max_tweets)
    i = 0
    last_error = 0
    while True:
        try:
            i += 1
            status = next(items)
            if status.text not in unique_tweets:
                unique_tweets[status.text] = True
                data_de.append(cast_status_to_dict(status))
                matches = hashtag_rex.findall(status.text)
                for match in matches:
                    match = match.lower()
                    if match not in processed_hashtags and match not in next_wave:
                        next_wave.append(match)
        except tweepy.TweepError:
            log(traceback.format_exc())
            if i > 0 and last_error + 2 > i:
                time.sleep(60)
                log('Recurrent rate error. Going to sleep now...')
            last_error = i
            current_api += 1
            log('changed to next api key: {0}'.format(current_api))
            if current_api >= len(ckeys): current_api = 0
            continue
        except StopIteration:
            break

def process_current_wave():
    global current_wave
    i = 0
    while len(current_wave) > 0:
        if i % 10 == 0 and i > 0:
            log('Dumping data...')
            pickle.dump(processed_hashtags, open(base_path + 'hashtags.p','wb'))
            pickle.dump(data_de, open(base_path + 'data_de2.p','wb'))
            log('Dumping wave data...')
            pickle.dump(current_wave, open(base_path + 'current_wave.p','wb'))
            pickle.dump(next_wave, open(base_path + 'next_wave.p','wb'))
        process_hashtag(current_wave[0])
        processed_hashtags[current_wave[0]] = True
        log('Successfully processed hashtag: {0}'.format(current_wave[0]))
        del current_wave[0]
        log('Dumping wave data...')
        pickle.dump(current_wave, open(base_path + 'current_wave.p','wb'))
        pickle.dump(next_wave, open(base_path + 'next_wave.p','wb'))
        i+=1

    log('Dumping data...')
    pickle.dump(processed_hashtags, open(base_path + 'hashtags.p','wb'))
    pickle.dump(data_de, open(base_path + 'data_de2.p','wb'))
    log('Current tweet count: {0}'.format(len(data_de)))

    t1 = time.time()

    total_min = (t1-t0)/60.
    log('Current tweet rate: {0} tweets per minute'.format(len(data_de)/total_min))

processed_hashtags = {}

next_wave = []
current_wave = ['royals','gamergate','aufschrei','tinder','fml','ebola']

hashtag_rex = re.compile('(?<=^|(?<=[^a-zA-Z0-9-_\.]))#([A-Za-z]+[A-Za-z0-9]+)')

max_tweets = 1000
data_en = []
unique_tweets = {}
hashtag = '%23'

process_current_wave()
while len(next_wave) > 0:
    current_wave = next_wave
    next_wave = []
    process_current_wave()

<commit_msg>More logging.<commit_after>'''
Created on Sep 26, 2014

@author: tim
'''
import tweepy
from tweepy import OAuthHandler
import time
import itertools
import os
import cPickle as pickle
import sys
import re
import traceback
os.environ['http_proxy']= ''
import logging
import os

t0 = time.time()

DEBUG = True

config_name = 'debug.txt'

# determine if application is a script file or frozen exe
if getattr(sys, 'frozen', False):
    application_path = os.path.dirname(sys.executable)
elif __file__:
    application_path = os.path.dirname(__file__)

config_path = os.path.join(application_path, config_name)

logging.basicConfig(filename=config_path, format='%(message)s', level=logging.DEBUG)

def log(s):
    if DEBUG: print s
    logging.log(logging.DEBUG, s)

ckeys = []
csecrets = []
atokens = []
asecrets = []

ckeys.append('Go2BKKaGREBEioh1BEeRhA')
csecrets.append('lSsiczxot5t1AKKhcYzlGHNXL66B8s7Wpj1QB1oM3U')
atokens.append('872274950-ihHI1EEfLd20059n8eLBFOqvJMCMCfwNqXR9ObTh')
asecrets.append('X1ic8qTu5LcjpKFckibFH7S8sNZ7ut9rr7rc2Jt4')

ckeys.append('leXgZOVbVhG5BvdnvM58BgAmy')
csecrets.append('LAOr4rkUHTxYgdPiMeTxjOxWURgb9bWdrQog1DyRM1JQhcNokY')
atokens.append('872274950-hl11IsipRPTL1MvPrsqGapJPtjj56F5E81dlGIGf')
asecrets.append('kte8SvpQBPDVezSTAtgPHZ2zhcp9SqkiV4G11m8TQ0BhG')

ckeys.append('napyrVTJtsycJ9sTQOdQw')
csecrets.append('PJ9Gr0k17L9XrhSJgmEwJ5Pkb2L8BimMyjlpcerpc')
atokens.append('872274950-mAhGAEs7TQArb8SJjb5CRsOd3GBKpqTWqxniPXZa')
asecrets.append('AgPW8xj8PFDZcJVDK3aMWpCZw2FNV5t57jzHVZc1XQSdL')

ckeys.append('oHmPy6BDn6QPISA8O4YmSFcSn')
csecrets.append('rPWtdq811nMm1Ta4TsJXs6L2j0y5BP9kFEQ6wLZJ3zmi5ypCEt')
atokens.append('872274950-FrgfRor6nmfOn9YvIDihcIPpWxjsxL1kdBRXgDj5')
asecrets.append('s3sKetUiHS7LHvBPUWFjoGASyOQy7SzFnkm6FSxiw0Sx0')

#debug!
ckeys.append('h554bvyP67an9o98A0uZl8D5i')
csecrets.append('5hvRoKrEf1LKrCIqZU0htZpzZ6yAOm8SwV3WFFetDSvb6Tf2Sl')
atokens.append('872274950-jK8Q4sZlBo8PDJwbs6frOe7CStnygnoSn0GRP7P1')
asecrets.append('lFUmOkBEtI8vOkeIeQ2bSlo9ye5A28QpQxGfo6pAjGZu0')

ckeys.append('jrwNGHc9FsCSJHk2Z0McxYgXM')
csecrets.append('VVhEWzhgjkSRuiPs4p1DfsYIB3b38shXXonc9WXFFz70KWF32Z')
atokens.append('872274950-Xe9QzakUbYdQktLxlMgYvtt4L2ulHgy4lLPZyr2q')
asecrets.append('F2EcF3S56GEfDQJnCAiSdvTzD2Zp3SB98cFJmfjCv1jyy')

ckeys.append('jrwNGHc9FsCSJHk2Z0McxYgXM')
csecrets.append('VVhEWzhgjkSRuiPs4p1DfsYIB3b38shXXonc9WXFFz70KWF32Z')
atokens.append('872274950-Xe9QzakUbYdQktLxlMgYvtt4L2ulHgy4lLPZyr2q')
asecrets.append('F2EcF3S56GEfDQJnCAiSdvTzD2Zp3SB98cFJmfjCv1jyy')

ckeys.append('pKCZtQlg3ts27JLUTzQJK0qhT')
csecrets.append('Ojr04SvTURTrVarVUQkE9b3Q5v7N6wBKLhK48T4g5sJfSmsruI')
atokens.append('872274950-Eok1pRVwol8fGSFXfyFc7f6aZTOuGNz3e0bBMq7A')
asecrets.append('cJCrBRZs9sQkJ4sxeIEJZLsR266MwxMKFj0mALJCbvjqo')

ckeys.append('e7Q1xcHIms2kMSpJzF0uVWWxU')
csecrets.append('xTZe6shkMD2cGFg42udOkU24cOnZCJq2wFP4wtOMKHmSIBSY5x')
atokens.append('872274950-tY13UAP8kiNXVaWjn0dLrjC13JhFaj9eWeVljqzu')
asecrets.append('yJ7y9PJ3nFdw5bsQnePH5F4o05SX6fpyfMIhG6Yb6m2Tr')

ckeys.append('b5MSBg9l1THMjzVEJxGKAax7m')
csecrets.append('EzLcQRZj6p1B5z2Ox5nzv1kDwL861ZCtmK4p02km1PyNY8Fidc')
atokens.append('872274950-NUxhs2SY04oEievqDujTCEghgfsD91d2easr82KH')
asecrets.append('i9hxb96b1wIbFDHbOuZxO4oxNv9jSVlZEDooaiWRES1tr')

ckeys.append('1LSVbEQDZvv1yhq8cgnYHLYr0')
csecrets.append('VZGlznquz32k4Pjz5cGbaCZ2j0TUuuNUcDltpLMVIGTIhpDAGq')
atokens.append('872274950-buBF1nz43sgsVh9hwcsv8RoX1M8xurzgZ4bdjDyD')
asecrets.append('d84pljcCcRO7L3C2IImuPjD37LMAfYTZ7GwRAU5ubGHcU')

ckeys.append('roBbWjs34pNbc21qgzKto9mvl')
csecrets.append('MrC4atAsJSB4tIvrLZ267fPR9yvZRWkZP3gBEWnS8POzSoov0p')
atokens.append('872274950-no8Hdl2ikM20WTdT1Wa7T9WTciYGDA9UBZG2MyE5')
asecrets.append('083G8Ux5fy59YtDgXYjrTr4ZJpmFWcr50Eid0gdAhzuI0')

ckeys.append('fZH9P7N0CXu5aLLWLlSI8qmjP')
csecrets.append('Pfs0ARchnKYOaTDL8I7deOQYasioj8GPeiu8ulHqhbQbsvYGzy')
atokens.append('872274950-LhxYgTpUtLEsVAn6TAMPWAYhwEIuZEOrsFa5593I')
asecrets.append('L8aoQPkVSc7EEEgBay7Jo9qbIqonISG3bpQILcx2xYjeO')

ckeys.append('Yzn6fparxxRqL9M5hPDFSrpS8')
csecrets.append('lL8fFSYJrkeri12nPV14oN5WcK2Vy0LeDmAGuXIWWz2b5QHqlF')
atokens.append('872274950-hcrxze1FxWYFx2gZtp4TgFutUaPUZGFKdXFnkWGo')
asecrets.append('4NbbDdxFWDycPEsqXrL0QGZ92VlbBkijlJubxsy1xQhTN')

ckeys.append('3MlUKIKfo7uJ2pbfx1Oi43oSx')
csecrets.append('0o733A8OvTmsCo1Vi4iJenxtAj3zVlwB9FvCsw9hBVsdByZCyE')
atokens.append('872274950-8nbgDCcVDEkMFBLJdiUD6Vf22KM6XHVSGSsKplbK')
asecrets.append('i78B5SAy88xKab6bwO4SAf8ejyLCKlVzLK9AJJte7sK8z')

ckeys.append('jzZJjV4WaQTO0UfXhlXldqDAD')
csecrets.append('Bpac2gVzeg2JwiUuTyZcnAJ3o41o8i7iqr2NYu9px08QEVmRBG')
atokens.append('872274950-nqebPIFbyi7BgWmbT5UFiKnnLB1PNGzolXpdyed3')
asecrets.append('F0rSbgdbLv9yNNUzcLCIIWNiQaBNGB77mt9uvIABkAp37')

ckeys.append('2VPJEUQ9j1U1RBifjLzoORHPz')
csecrets.append('kTSS1DsYp5awkjCWUndEoKileEDmgLouHXixAWItCtPRaaafES')
atokens.append('872274950-OYftTzpKgcvuGaq4wqW9ZCQ9D6KIciiNef3OwmfR')
asecrets.append('oBy36n6rz8G7a6sZUOzNbeWcWCzPKFu4MuNmyZUyPRfkE')

current_api = 0

base_path = sys.argv[0].replace('hashtag_crawler.py','')
data_de = []

log(base_path)

def cast_status_to_dict(tweepy_status):
    ret = {}
    ret['text'] = tweepy_status.text
    ret['follower_count'] = tweepy_status.author.followers_count
    ret['friends_count'] = tweepy_status.author.friends_count
    ret['retweet_count'] = tweepy_status.retweet_count
    return ret

apis = []
for i in range(len(ckeys)):
    auth = OAuthHandler(ckeys[i], csecrets[i])
    auth.set_access_token(atokens[i], asecrets[i])
    apis.append(tweepy.API(auth))

def process_hashtag(query):
    global current_api
    global processed_hashtags
    global data_de
    processed_hashtags[query] = True
    items = tweepy.Cursor(apis[current_api].search, q=hashtag + query,lang='de').items(max_tweets)
    i = 0
    last_error = 0
    while True:
        try:
            i += 1
            status = next(items)
            if status.text not in unique_tweets:
                unique_tweets[status.text] = True
                data_de.append(cast_status_to_dict(status))
                matches = hashtag_rex.findall(status.text)
                for match in matches:
                    match = match.lower()
                    if match not in processed_hashtags and match not in next_wave:
                        next_wave.append(match)
        except tweepy.TweepError:
            log(traceback.format_exc())
            if i > 0 and last_error + 2 > i:
                time.sleep(60)
                log('Recurrent rate error. Going to sleep now...')
            last_error = i
            current_api += 1
            log('changed to next api key: {0}'.format(current_api))
            if current_api >= len(ckeys): current_api = 0
            continue
        except StopIteration:
            break

def process_current_wave():
    global current_wave
    i = 0
    while len(current_wave) > 0:
        log('Dumping data to {0}'.format(base_path + 'data_de2.p'))
        pickle.dump(processed_hashtags, open(base_path + 'hashtags.p','wb'))
        pickle.dump(data_de, open(base_path + 'data_de2.p','wb'))

        process_hashtag(current_wave[0])
        processed_hashtags[current_wave[0]] = True
        log('Successfully processed hashtag: {0}'.format(current_wave[0]))
        del current_wave[0]
        log('Dumping wave data...')
        pickle.dump(current_wave, open(base_path + 'current_wave.p','wb'))
        pickle.dump(next_wave, open(base_path + 'next_wave.p','wb'))
        i+=1

    log('Dumping data...')
    pickle.dump(processed_hashtags, open(base_path + 'hashtags.p','wb'))
    pickle.dump(data_de, open(base_path + 'data_de2.p','wb'))
    log('Current tweet count: {0}'.format(len(data_de)))

    t1 = time.time()

    total_min = (t1-t0)/60.
    log('Current tweet rate: {0} tweets per minute'.format(len(data_de)/total_min))

processed_hashtags = {}

next_wave = []
current_wave = ['royals','gamergate','aufschrei','tinder','fml','ebola']

hashtag_rex = re.compile('(?<=^|(?<=[^a-zA-Z0-9-_\.]))#([A-Za-z]+[A-Za-z0-9]+)')

max_tweets = 1000
data_en = []
unique_tweets = {}
hashtag = '%23'

process_current_wave()
while len(next_wave) > 0:
    current_wave = next_wave
    next_wave = []
    process_current_wave()

<|endoftext|>


<commit_before>from __future__ import unicode_literals

import contextlib
import os
import sys

import pre_commit.constants as C
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import Var
from pre_commit.languages import helpers
from pre_commit.languages.python import bin_dir
from pre_commit.util import clean_path_on_failure
from pre_commit.util import cmd_output

ENVIRONMENT_DIR = 'node_env'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy

def _envdir(prefix, version):
    directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
    return prefix.path(directory)

def get_env_patch(venv):  # pragma: windows no cover
    lib_dir = 'lib'
    if sys.platform == 'cygwin':  # pragma: no cover
        _, win_venv, _ = cmd_output('cygpath', '-w', venv)
        install_prefix = r'{}\bin'.format(win_venv.strip())
    elif sys.platform == 'win32':  # pragma: no cover
        install_prefix = bin_dir(venv)
        lib_dir = 'Scripts'
    else:  # pragma: windows no cover
        install_prefix = venv
    return (
        ('NODE_VIRTUAL_ENV', venv),
        ('NPM_CONFIG_PREFIX', install_prefix),
        ('npm_config_prefix', install_prefix),
        ('NODE_PATH', os.path.join(venv, lib_dir, 'node_modules')),
        ('PATH', (bin_dir(venv), os.pathsep, Var('PATH'))),
    )

@contextlib.contextmanager
def in_env(prefix, language_version):  # pragma: windows no cover
    with envcontext(get_env_patch(_envdir(prefix, language_version))):
        yield

def install_environment(
        prefix, version, additional_dependencies,
):  # pragma: windows no cover
    additional_dependencies = tuple(additional_dependencies)
    assert prefix.exists('package.json')
    envdir = _envdir(prefix, version)

    # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx?f=255&MSPPError=-2147217396#maxpath
    if sys.platform == 'win32':  # pragma: no cover
        envdir = '\\\\?\\' + os.path.normpath(envdir)
    with clean_path_on_failure(envdir):
        cmd = [
            sys.executable, '-mnodeenv', '--prebuilt', '--clean-src', envdir,
        ]
        if version != C.DEFAULT:
            cmd.extend(['-n', version])
        cmd_output(*cmd)

        with in_env(prefix, version):
            # https://npm.community/t/npm-install-g-git-vs-git-clone-cd-npm-install-g/5449
            # install as if we installed from git
            helpers.run_setup_cmd(prefix, ('npm', 'install'))
            helpers.run_setup_cmd(
                prefix,
                ('npm', 'install', '-g', '.') + additional_dependencies,
            )

def run_hook(hook, file_args):  # pragma: windows no cover
    with in_env(hook.prefix, hook.language_version):
        return helpers.run_xargs(hook, helpers.to_cmd(hook), file_args)
<commit_msg>Simplify if statement slightly<commit_after>from __future__ import unicode_literals

import contextlib
import os
import sys

import pre_commit.constants as C
from pre_commit.envcontext import envcontext
from pre_commit.envcontext import Var
from pre_commit.languages import helpers
from pre_commit.languages.python import bin_dir
from pre_commit.util import clean_path_on_failure
from pre_commit.util import cmd_output

ENVIRONMENT_DIR = 'node_env'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy

def _envdir(prefix, version):
    directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
    return prefix.path(directory)

def get_env_patch(venv):  # pragma: windows no cover
    if sys.platform == 'cygwin':  # pragma: no cover
        _, win_venv, _ = cmd_output('cygpath', '-w', venv)
        install_prefix = r'{}\bin'.format(win_venv.strip())
        lib_dir = 'lib'
    elif sys.platform == 'win32':  # pragma: no cover
        install_prefix = bin_dir(venv)
        lib_dir = 'Scripts'
    else:  # pragma: windows no cover
        install_prefix = venv
        lib_dir = 'lib'
    return (
        ('NODE_VIRTUAL_ENV', venv),
        ('NPM_CONFIG_PREFIX', install_prefix),
        ('npm_config_prefix', install_prefix),
        ('NODE_PATH', os.path.join(venv, lib_dir, 'node_modules')),
        ('PATH', (bin_dir(venv), os.pathsep, Var('PATH'))),
    )

@contextlib.contextmanager
def in_env(prefix, language_version):  # pragma: windows no cover
    with envcontext(get_env_patch(_envdir(prefix, language_version))):
        yield

def install_environment(
        prefix, version, additional_dependencies,
):  # pragma: windows no cover
    additional_dependencies = tuple(additional_dependencies)
    assert prefix.exists('package.json')
    envdir = _envdir(prefix, version)

    # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx?f=255&MSPPError=-2147217396#maxpath
    if sys.platform == 'win32':  # pragma: no cover
        envdir = '\\\\?\\' + os.path.normpath(envdir)
    with clean_path_on_failure(envdir):
        cmd = [
            sys.executable, '-mnodeenv', '--prebuilt', '--clean-src', envdir,
        ]
        if version != C.DEFAULT:
            cmd.extend(['-n', version])
        cmd_output(*cmd)

        with in_env(prefix, version):
            # https://npm.community/t/npm-install-g-git-vs-git-clone-cd-npm-install-g/5449
            # install as if we installed from git
            helpers.run_setup_cmd(prefix, ('npm', 'install'))
            helpers.run_setup_cmd(
                prefix,
                ('npm', 'install', '-g', '.') + additional_dependencies,
            )

def run_hook(hook, file_args):  # pragma: windows no cover
    with in_env(hook.prefix, hook.language_version):
        return helpers.run_xargs(hook, helpers.to_cmd(hook), file_args)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import (
    print_function,
    division,
    absolute_import,
    unicode_literals,
)
try:
    from inspect import getfullargspec as getargspec  # PY3
except ImportError:
    from inspect import getargspec  # PY2
from copy import deepcopy
from fontTools.misc.py23 import unichr
from fontTools import ttLib
from fontTools import subset
from fontTools.feaLib.builder import addOpenTypeFeatures
import logging

logger = logging.getLogger(__name__)

def makeOfficialGlyphOrder(font, glyphOrder=None):
    """ Make the final glyph order for 'font'.

    If glyphOrder is None, try getting the font.glyphOrder list.
    If not explicit glyphOrder is defined, sort glyphs alphabetically.

    If ".notdef" glyph is present in the font, force this to always be
    the first glyph (at index 0).
    """
    if glyphOrder is None:
        glyphOrder = getattr(font, "glyphOrder", ())
    names = set(font.keys())
    order = []
    if ".notdef" in names:
        names.remove(".notdef")
        order.append(".notdef")
    for name in glyphOrder:
        if name not in names:
            continue
        names.remove(name)
        order.append(name)
    order.extend(sorted(names))
    return order

def copyGlyphSet(font, layerName=None):
    if layerName is not None:
        layer = font.layers[layerName]
    else:
        layer = font.layers.defaultLayer

    if not len(layer):
        return {}

    # defcon.Glyph doesn't take a name argument, ufoLib2 requires one...
    g = next(iter(layer))
    cls = g.__class__
    if "name" in getargspec(cls.__init__).args:

        def newGlyph(name):
            return cls(name=name)

    else:

        def newGlyph(name):
            g = cls()
            g.name = name
            return g

    # copy everything except unused attributes: 'guidelines', 'note', 'image'
    glyphSet = {}
    for glyph in layer:
        copy = newGlyph(glyph.name)
        copy.width = glyph.width
        copy.height = glyph.height
        copy.unicodes = list(glyph.unicodes)
        copy.anchors = [dict(a) for a in glyph.anchors]
        copy.lib = deepcopy(glyph.lib)
        pointPen = copy.getPointPen()
        glyph.drawPoints(pointPen)
        glyphSet[glyph.name] = copy
    return glyphSet

def makeUnicodeToGlyphNameMapping(font, glyphOrder=None):
    """ Make a unicode: glyph name mapping for this glyph set (dict or Font).

    Raises InvalidFontData exception if multiple glyphs are mapped to the
    same unicode codepoint.
    """
    if glyphOrder is None:
        glyphOrder = makeOfficialGlyphOrder(font)
    mapping = {}
    for glyphName in glyphOrder:
        glyph = font[glyphName]
        unicodes = glyph.unicodes
        for uni in unicodes:
            if uni not in mapping:
                mapping[uni] = glyphName
            else:
                from ufo2ft.errors import InvalidFontData

                InvalidFontData(
                    "cannot map '%s' to U+%04X; already mapped to '%s'"
                    % (glyphName, uni, mapping[uni])
                )
    return mapping

def compileGSUB(featureFile, glyphOrder):
    """ Compile and return a GSUB table from `featureFile` (feaLib
    FeatureFile), using the given `glyphOrder` (list of glyph names).
    """
    font = ttLib.TTFont()
    font.setGlyphOrder(glyphOrder)
    addOpenTypeFeatures(font, featureFile, tables={"GSUB"})
    return font.get("GSUB")

def closeGlyphsOverGSUB(gsub, glyphs):
    """ Use the FontTools subsetter to perform a closure over the GSUB table
    given the initial `glyphs` (set of glyph names, str). Update the set
    in-place adding all the glyph names that can be reached via GSUB
    substitutions from this initial set.
    """
    subsetter = subset.Subsetter()
    subsetter.glyphs = glyphs
    gsub.closure_glyphs(subsetter)

def classifyGlyphs(unicodeFunc, cmap, gsub=None):
    """ 'unicodeFunc' is a callable that takes a Unicode codepoint and
    returns a string denoting some Unicode property associated with the
    given character (or None if a character is considered 'neutral').
    'cmap' is a dictionary mapping Unicode codepoints to glyph names.
    'gsub' is an (optional) fonttools GSUB table object, used to find all
    the glyphs that are "reachable" via substitutions from the initial
    sets of glyphs defined in the cmap.

    Returns a dictionary of glyph sets associated with the given Unicode
    properties.
    """
    glyphSets = {}
    neutralGlyphs = set()
    for uv, glyphName in cmap.items():
        key = unicodeFunc(uv)
        if key is None:
            neutralGlyphs.add(glyphName)
        else:
            glyphSets.setdefault(key, set()).add(glyphName)

    if gsub is not None:
        if neutralGlyphs:
            closeGlyphsOverGSUB(gsub, neutralGlyphs)

        for glyphs in glyphSets.values():
            s = glyphs | neutralGlyphs
            closeGlyphsOverGSUB(gsub, s)
            glyphs.update(s - neutralGlyphs)

    return glyphSets

def calcCodePageRanges(unicodes):
    """ Given a set of Unicode codepoints (integers), calculate the
    corresponding OS/2 CodePage range bits.
    This is a direct translation of FontForge implementation:
    https://github.com/fontforge/fontforge/blob/7b2c074/fontforge/tottf.c#L3158
    """
    codepageRanges = set()

    chars = [unichr(u) for u in unicodes]

    hasAscii = set(range(0x20, 0x7E)).issubset(unicodes)
    hasLineart = "┤" in chars

    for char in chars:
        if char == "Þ" and hasAscii:
            codepageRanges.add(0)           # Latin 1
        elif char == "Ľ" and hasAscii:
            codepageRanges.add(1)           # Latin 2: Eastern Europe
            if hasLineart:
                codepageRanges.add(58)      # Latin 2
        elif char == "Б":
            codepageRanges.add(2)           # Cyrillic
            if "Ѕ" in chars and hasLineart:
                codepageRanges.add(57)      # IBM Cyrillic
            if "╜" in chars and hasLineart:
                codepageRanges.add(49)      # MS-DOS Russian
        elif char == "Ά":
            codepageRanges.add(3)           # Greek
            if hasLineart and "½" in chars:
                codepageRanges.add(48)      # IBM Greek
            if hasLineart and "√" in chars:
                codepageRanges.add(60)      # Greek, former 437 G
        elif char == "İ" and hasAscii:
            codepageRanges.add(4)           # Turkish
            if hasLineart:
                codepageRanges.add(56)      # IBM turkish
        elif char == "א":
            codepageRanges.add(5)           # Hebrew
            if hasLineart and "√" in chars:
                codepageRanges.add(53)      # Hebrew
        elif char == "ر":
            codepageRanges.add(6)           # Arabic
            if "√" in chars:
                codepageRanges.add(51)      # Arabic
            if hasLineart:
                codepageRanges.add(61)      # Arabic; ASMO 708
        elif char == "ŗ" and hasAscii:
            codepageRanges.add(7)           # Windows Baltic
            if hasLineart:
                codepageRanges.add(59)      # MS-DOS Baltic
        elif char == "₫" and hasAscii:
            codepageRanges.add(8)           # Vietnamese
        elif char == "ๅ":
            codepageRanges.add(16)          # Thai
        elif char == "エ":
            codepageRanges.add(17)          # JIS/Japan
        elif char == "ㄅ":
            codepageRanges.add(18)          # Chinese: Simplified chars
        elif char == "ㄱ":
            codepageRanges.add(19)          # Korean wansung
        elif char == "央":
            codepageRanges.add(20)          # Chinese: Traditional chars
        elif char == "곴":
            codepageRanges.add(21)          # Korean Johab
        elif char == "♥" and hasAscii:
            codepageRanges.add(30)          # OEM Character Set
        # TODO: Symbol bit has a special meaning (check the spec), we need
        # to confirm if this is wanted by default.
        # elif unichr(0xF000) <= char <= unichr(0xF0FF):
        #    codepageRanges.add(31)          # Symbol Character Set
        elif char == "þ" and hasAscii and hasLineart:
            codepageRanges.add(54)          # MS-DOS Icelandic
        elif char == "╚" and hasAscii:
            codepageRanges.add(62)          # WE/Latin 1
            codepageRanges.add(63)          # US
        elif hasAscii and hasLineart and "√" in chars:
            if char == "Å":
                codepageRanges.add(50)      # MS-DOS Nordic
            elif char == "é":
                codepageRanges.add(52)      # MS-DOS Canadian French
            elif char == "õ":
                codepageRanges.add(55)      # MS-DOS Portuguese

    if hasAscii and "‰" in chars and "∑" in chars:
        codepageRanges.add(29)              # Macintosh Character Set (US Roman)

    return codepageRanges
<commit_msg>util: add unicodeInScripts function<commit_after># -*- coding: utf-8 -*-
from __future__ import (
    print_function,
    division,
    absolute_import,
    unicode_literals,
)
try:
    from inspect import getfullargspec as getargspec  # PY3
except ImportError:
    from inspect import getargspec  # PY2
from copy import deepcopy
from fontTools.misc.py23 import unichr
from fontTools import ttLib
from fontTools import subset
from fontTools import unicodedata
from fontTools.feaLib.builder import addOpenTypeFeatures
import logging

logger = logging.getLogger(__name__)

def makeOfficialGlyphOrder(font, glyphOrder=None):
    """ Make the final glyph order for 'font'.

    If glyphOrder is None, try getting the font.glyphOrder list.
    If not explicit glyphOrder is defined, sort glyphs alphabetically.

    If ".notdef" glyph is present in the font, force this to always be
    the first glyph (at index 0).
    """
    if glyphOrder is None:
        glyphOrder = getattr(font, "glyphOrder", ())
    names = set(font.keys())
    order = []
    if ".notdef" in names:
        names.remove(".notdef")
        order.append(".notdef")
    for name in glyphOrder:
        if name not in names:
            continue
        names.remove(name)
        order.append(name)
    order.extend(sorted(names))
    return order

def copyGlyphSet(font, layerName=None):
    if layerName is not None:
        layer = font.layers[layerName]
    else:
        layer = font.layers.defaultLayer

    if not len(layer):
        return {}

    # defcon.Glyph doesn't take a name argument, ufoLib2 requires one...
    g = next(iter(layer))
    cls = g.__class__
    if "name" in getargspec(cls.__init__).args:

        def newGlyph(name):
            return cls(name=name)

    else:

        def newGlyph(name):
            g = cls()
            g.name = name
            return g

    # copy everything except unused attributes: 'guidelines', 'note', 'image'
    glyphSet = {}
    for glyph in layer:
        copy = newGlyph(glyph.name)
        copy.width = glyph.width
        copy.height = glyph.height
        copy.unicodes = list(glyph.unicodes)
        copy.anchors = [dict(a) for a in glyph.anchors]
        copy.lib = deepcopy(glyph.lib)
        pointPen = copy.getPointPen()
        glyph.drawPoints(pointPen)
        glyphSet[glyph.name] = copy
    return glyphSet

def makeUnicodeToGlyphNameMapping(font, glyphOrder=None):
    """ Make a unicode: glyph name mapping for this glyph set (dict or Font).

    Raises InvalidFontData exception if multiple glyphs are mapped to the
    same unicode codepoint.
    """
    if glyphOrder is None:
        glyphOrder = makeOfficialGlyphOrder(font)
    mapping = {}
    for glyphName in glyphOrder:
        glyph = font[glyphName]
        unicodes = glyph.unicodes
        for uni in unicodes:
            if uni not in mapping:
                mapping[uni] = glyphName
            else:
                from ufo2ft.errors import InvalidFontData

                InvalidFontData(
                    "cannot map '%s' to U+%04X; already mapped to '%s'"
                    % (glyphName, uni, mapping[uni])
                )
    return mapping

def compileGSUB(featureFile, glyphOrder):
    """ Compile and return a GSUB table from `featureFile` (feaLib
    FeatureFile), using the given `glyphOrder` (list of glyph names).
    """
    font = ttLib.TTFont()
    font.setGlyphOrder(glyphOrder)
    addOpenTypeFeatures(font, featureFile, tables={"GSUB"})
    return font.get("GSUB")

def closeGlyphsOverGSUB(gsub, glyphs):
    """ Use the FontTools subsetter to perform a closure over the GSUB table
    given the initial `glyphs` (set of glyph names, str). Update the set
    in-place adding all the glyph names that can be reached via GSUB
    substitutions from this initial set.
    """
    subsetter = subset.Subsetter()
    subsetter.glyphs = glyphs
    gsub.closure_glyphs(subsetter)

def classifyGlyphs(unicodeFunc, cmap, gsub=None):
    """ 'unicodeFunc' is a callable that takes a Unicode codepoint and
    returns a string denoting some Unicode property associated with the
    given character (or None if a character is considered 'neutral').
    'cmap' is a dictionary mapping Unicode codepoints to glyph names.
    'gsub' is an (optional) fonttools GSUB table object, used to find all
    the glyphs that are "reachable" via substitutions from the initial
    sets of glyphs defined in the cmap.

    Returns a dictionary of glyph sets associated with the given Unicode
    properties.
    """
    glyphSets = {}
    neutralGlyphs = set()
    for uv, glyphName in cmap.items():
        key = unicodeFunc(uv)
        if key is None:
            neutralGlyphs.add(glyphName)
        else:
            glyphSets.setdefault(key, set()).add(glyphName)

    if gsub is not None:
        if neutralGlyphs:
            closeGlyphsOverGSUB(gsub, neutralGlyphs)

        for glyphs in glyphSets.values():
            s = glyphs | neutralGlyphs
            closeGlyphsOverGSUB(gsub, s)
            glyphs.update(s - neutralGlyphs)

    return glyphSets

def unicodeInScripts(uv, scripts):
    """ Check UnicodeData's ScriptExtension property for unicode codepoint
    'uv' and return True if it intersects with the set of 'scripts' provided,
    False if it does not intersect.
    Return None for 'Common' script ('Zyyy').
    """
    sx = unicodedata.script_extension(unichr(uv))
    if "Zyyy" in sx:
        return None
    return not sx.isdisjoint(scripts)

def calcCodePageRanges(unicodes):
    """ Given a set of Unicode codepoints (integers), calculate the
    corresponding OS/2 CodePage range bits.
    This is a direct translation of FontForge implementation:
    https://github.com/fontforge/fontforge/blob/7b2c074/fontforge/tottf.c#L3158
    """
    codepageRanges = set()

    chars = [unichr(u) for u in unicodes]

    hasAscii = set(range(0x20, 0x7E)).issubset(unicodes)
    hasLineart = "┤" in chars

    for char in chars:
        if char == "Þ" and hasAscii:
            codepageRanges.add(0)           # Latin 1
        elif char == "Ľ" and hasAscii:
            codepageRanges.add(1)           # Latin 2: Eastern Europe
            if hasLineart:
                codepageRanges.add(58)      # Latin 2
        elif char == "Б":
            codepageRanges.add(2)           # Cyrillic
            if "Ѕ" in chars and hasLineart:
                codepageRanges.add(57)      # IBM Cyrillic
            if "╜" in chars and hasLineart:
                codepageRanges.add(49)      # MS-DOS Russian
        elif char == "Ά":
            codepageRanges.add(3)           # Greek
            if hasLineart and "½" in chars:
                codepageRanges.add(48)      # IBM Greek
            if hasLineart and "√" in chars:
                codepageRanges.add(60)      # Greek, former 437 G
        elif char == "İ" and hasAscii:
            codepageRanges.add(4)           # Turkish
            if hasLineart:
                codepageRanges.add(56)      # IBM turkish
        elif char == "א":
            codepageRanges.add(5)           # Hebrew
            if hasLineart and "√" in chars:
                codepageRanges.add(53)      # Hebrew
        elif char == "ر":
            codepageRanges.add(6)           # Arabic
            if "√" in chars:
                codepageRanges.add(51)      # Arabic
            if hasLineart:
                codepageRanges.add(61)      # Arabic; ASMO 708
        elif char == "ŗ" and hasAscii:
            codepageRanges.add(7)           # Windows Baltic
            if hasLineart:
                codepageRanges.add(59)      # MS-DOS Baltic
        elif char == "₫" and hasAscii:
            codepageRanges.add(8)           # Vietnamese
        elif char == "ๅ":
            codepageRanges.add(16)          # Thai
        elif char == "エ":
            codepageRanges.add(17)          # JIS/Japan
        elif char == "ㄅ":
            codepageRanges.add(18)          # Chinese: Simplified chars
        elif char == "ㄱ":
            codepageRanges.add(19)          # Korean wansung
        elif char == "央":
            codepageRanges.add(20)          # Chinese: Traditional chars
        elif char == "곴":
            codepageRanges.add(21)          # Korean Johab
        elif char == "♥" and hasAscii:
            codepageRanges.add(30)          # OEM Character Set
        # TODO: Symbol bit has a special meaning (check the spec), we need
        # to confirm if this is wanted by default.
        # elif unichr(0xF000) <= char <= unichr(0xF0FF):
        #    codepageRanges.add(31)          # Symbol Character Set
        elif char == "þ" and hasAscii and hasLineart:
            codepageRanges.add(54)          # MS-DOS Icelandic
        elif char == "╚" and hasAscii:
            codepageRanges.add(62)          # WE/Latin 1
            codepageRanges.add(63)          # US
        elif hasAscii and hasLineart and "√" in chars:
            if char == "Å":
                codepageRanges.add(50)      # MS-DOS Nordic
            elif char == "é":
                codepageRanges.add(52)      # MS-DOS Canadian French
            elif char == "õ":
                codepageRanges.add(55)      # MS-DOS Portuguese

    if hasAscii and "‰" in chars and "∑" in chars:
        codepageRanges.add(29)              # Macintosh Character Set (US Roman)

    return codepageRanges
<|endoftext|>