<commit_before>import base64
import threading
from urllib.parse import parse_qs

import pandas as pd

from tornado.ioloop import IOLoop
from bokeh.application import Application
from bokeh.application.handlers import FunctionHandler
from bokeh.layouts import widgetbox, row
from bokeh.models import ColumnDataSource, RangeSlider, CustomJS
from bokeh.models.widgets.inputs import Select
from bokeh.plotting import figure
from bokeh.server.server import Server
from bokeh.models import HoverTool

def update_querystring(window=None, cb_obj=None):
    '''
    This callback should be converted to JS! It updates
    the url querystring when chart options change.
    '''
    params = []
    for c in cb_obj.document.roots()[0].children[0].children:
        value = c.value if c.value else c.range
        params.append(
            window.encodeURIComponent(c.name) + '=' +
            window.encodeURIComponent(value)
        )
    query = '?' + '&'.join(params)
    window.history.pushState({}, '', query)

def load_db():
    df = pd.read_csv('test.csv')
    df.ano = df.ano.astype(str)
    return df

def hide_widget(widget):
    if not widget.css_classes:
        widget.css_classes = []
    if 'hidden' not in widget.css_classes:
        widget.css_classes.append('hidden')

def show_widget(widget):
    if widget.css_classes and 'hidden' in widget.css_classes:
        widget.css_classes.remove('hidden')

class Dashboard(object):
    '''
    This class represents a dashboard and should be instanciated
    for each user.
    The `generate` static method is used by the Bokeh server to
    instanciate dashboards.
    '''

    # TODO: maybe remove this?
    df = load_db()

    @staticmethod
    def generate(doc):
        '''
        Generate a new dashboard. This function should be passed
        to the Bokeh FunctionHandler.
        '''
        return Dashboard(Dashboard.df.copy(), doc)

    def __init__(self, df, doc):
        '''
        Called when a user connects. This dashboard should be used
        while the connection remain, and is used by only one user.
        '''

        state = doc.session_context.request.arguments.get('state')
        # import IPython;IPython.embed()
        if state:
            state = parse_qs(base64.urlsafe_b64decode(state[0]).decode())
        else:
            state = {}

        self.chart_types = {
            'linha': self.plot_lines,
            'barras horizontais': self.plot_hbar,
            'barras verticais': self.plot_vbar,
            'círculos': self.plot_circles,
        }
        charts_names = list(self.chart_types)
        none_value = 'Nenhum'

        self.df = df
        self.doc = doc
        self.columns = sorted(df.columns)
        self.discrete = [x for x in self.columns if self.df[x].dtype == object]
        # continuous = [x for x in columns if x not in discrete]
        # discrete.append(continuous.pop(continuous.index('ano')))
        self.filter_options = [none_value]+self.columns

        gui_data = {
            'x_sel': {
                'name': 'x',
                'title': 'Eixo X',
                'value': 'ano',
                'options': self.columns,
                'class': Select,
            },
            'y_sel': {
                'name': 'y',
                'title': 'Eixo Y',
                'value': 'pop_masc',
                'options': self.columns,
                'class': Select,
            },
            'chart_type_sel': {
                'name': 'type',
                'title': 'Tipo de Gráfico',
                'value': charts_names[2],
                'options': charts_names,
                'class': Select,
            },
            'filter_sel': {
                'name': 'filter',
                'title': 'Filtro',
                'value': self.filter_options[0],
                'options': self.filter_options,
                'class': Select,
            },
            'filter_value_sel': {
                'name': 'f_value',
                'title': 'Valor',
                'value': none_value,
                'options': [none_value],
                'class': Select,
            },
            # TODO: this widget is bugged, but it seems they are improving it.
            'filter_range_sel': {
                'name': 'f_range',
                'title': 'Intervalo',
                'start': 0,
                'end': 10,
                'step': 1,
                'range': (0, 0),
                'class': RangeSlider,
            },
        }
        # TODO: validate state? Dangerous?
        if state:
            for gui, data in gui_data.items():
                value = state.get(data['name'])
                if value:
                    value = value[0]
                    if data.get('value'):
                        data['value'] = value
                    elif data.get('range'):
                        print(value)
                        data['range'] = tuple(float(i)
                                              for i in value.split(','))

        # Create gui inputs
        for k, v in gui_data.items():
            v = v.copy()
            class_ = v.pop('class')
            setattr(self, k, class_(**v))

        controls = [getattr(self, k) for k, v in gui_data.items()]

        # Bind callbacks
        js_update_querytstring = CustomJS.from_py_func(update_querystring)
        for control in controls:
            if hasattr(control, 'value'):
                control.on_change('value', self.update)
            elif hasattr(control, 'range'):
                control.on_change('range', self.update)
            control.callback = js_update_querytstring

        # TODO: seems to have no effect
        self.filter_range_sel.callback_policy = 'mouseup'

        controls = widgetbox(controls, width=200)
        self.layout = row(controls, self.create_figure())
        doc.add_root(self.layout)

    def update(self, attr, old, new):
        '''
        Called when chart options are modified.
        '''
        self.layout.children[1] = self.create_figure(attr, old, new)

    # TODO: move helpers (don't really need self) to a new file?
    def plot_lines(self, fig, x, y, source):
        fig.line(x, y, source=source)
        # plot.line(x=xs, width=0.5, bottom=0, top=ys)

    def plot_vbar(self, fig, x, y, source):
        fig.vbar(x, 0.5, y, source=source)
        # plot.vbar(x=xs, width=0.5, bottom=0, top=ys)

    def plot_hbar(self, fig, x, y, source):
        fig.hbar(y=y, height=0.5, right=x, left=0, source=source)

    def plot_circles(self, fig, x, y, source):
        fig.circle(x, y, source=source)
        # plot.circle(x=xs, y=ys)
    # -----------------------------------------

    def handle_filtering(self, df):
        '''
        Update filter selectors widgets and use their values
        to filter `df`.
        '''
        frs = self.filter_range_sel
        fvs = self.filter_value_sel
        if self.filter_sel.value != self.filter_options[0]:
            filter_column = df[self.filter_sel.value]
            if self.filter_sel.value in self.discrete:
                # Discrete filter

                # update selector
                options = sorted(list(filter_column.unique()))
                fvs.options = options
                if fvs.value not in options:
                    fvs.value = options[0]

                show_widget(fvs)
                hide_widget(frs)

                # filter df
                df = df[filter_column == fvs.value]

            else:
                # Continuous filter

                min_val = filter_column.min()
                max_val = filter_column.max()

                # update slider
                frs.start = min_val
                frs.end = max_val
                if frs.range[0] < min_val or frs.range[1] > max_val:
                    frs.range = (min_val, max_val)
                frs.step = (max_val-min_val)/10

                show_widget(frs)
                hide_widget(fvs)

                # filter df
                df = df[filter_column >= frs.range[0]]
                df = df[filter_column <= frs.range[1]]
        else:
            # No filter
            hide_widget(fvs)
            hide_widget(frs)

        return df

    def create_figure(self, attr=None, old=None, new=None):
        '''
        Creates the chart.
        '''
        print(attr, old, new)
        df = self.handle_filtering(self.df)
        df = df.groupby(self.x_sel.value).sum()
        source = ColumnDataSource(data=df)

        xs = self.df[self.x_sel.value].values
        ys = self.df[self.y_sel.value].values
        x_title = self.x_sel.value.title()
        y_title = self.y_sel.value.title()

        kw = dict()
        if self.x_sel.value in self.discrete:
            kw['x_range'] = sorted(set(xs))
        if self.y_sel.value in self.discrete:
            kw['y_range'] = sorted(set(ys))
        kw['title'] = "%s vs %s" % (x_title, y_title)

        fig = figure(plot_height=600, plot_width=800,
                     tools='pan,box_zoom,reset,save', **kw)
        fig.xaxis.axis_label = x_title
        fig.yaxis.axis_label = y_title

        if self.x_sel.value in self.discrete:
            fig.xaxis.major_label_orientation = pd.np.pi / 4

        self.chart_types[self.chart_type_sel.value](
            fig, self.x_sel.value, self.y_sel.value, source)

        hover = HoverTool(tooltips=[
            ("index", "$index"),
            (self.x_sel.value, '@'+self.x_sel.value),
            (self.y_sel.value, '@'+self.y_sel.value),
        ])
        fig.add_tools(hover)

        return fig

class BackgroundBokeh(threading.Thread):

    def run(self):
        bokeh_app = Application(FunctionHandler(Dashboard.generate))
        io_loop = IOLoop.current()
        server = Server({'/bkapp': bokeh_app}, io_loop=io_loop,
                        allow_websocket_origin=["localhost:8000"])
        server.start()
        io_loop.start()

    def stop(self):
        IOLoop.current().stop()

bokeh_server = BackgroundBokeh()

def start_server():
    print('> Starting Bokeh server')
    bokeh_server.start()

def stop_server():
    print('> Stopping Bokeh server')
    bokeh_server.stop()
<commit_msg>fix code to work with Bokeh 0.12.7<commit_after>import base64
import threading
from urllib.parse import parse_qs

import pandas as pd

from tornado.ioloop import IOLoop
from bokeh.application import Application
from bokeh.application.handlers import FunctionHandler
from bokeh.layouts import widgetbox, row
from bokeh.models import ColumnDataSource, RangeSlider, CustomJS
from bokeh.models.widgets.inputs import Select
from bokeh.plotting import figure
from bokeh.server.server import Server
from bokeh.models import HoverTool

def update_querystring(window=None, cb_obj=None):
    '''
    This callback should be converted to JS! It updates
    the url querystring when chart options change.
    '''
    params = []
    for c in cb_obj.document.roots()[0].children[0].children:
        value = c.value
        params.append(
            window.encodeURIComponent(c.name) + '=' +
            window.encodeURIComponent(value)
        )
    query = '?' + '&'.join(params)
    window.history.pushState({}, '', query)

def load_db():
    df = pd.read_csv('test.csv')
    df.ano = df.ano.astype(str)
    return df

def hide_widget(widget):
    if not widget.css_classes:
        widget.css_classes = []
    if 'hidden' not in widget.css_classes:
        widget.css_classes.append('hidden')

def show_widget(widget):
    if widget.css_classes and 'hidden' in widget.css_classes:
        widget.css_classes.remove('hidden')

class Dashboard(object):
    '''
    This class represents a dashboard and should be instanciated
    for each user.
    The `generate` static method is used by the Bokeh server to
    instanciate dashboards.
    '''

    # TODO: maybe remove this?
    df = load_db()

    @staticmethod
    def generate(doc):
        '''
        Generate a new dashboard. This function should be passed
        to the Bokeh FunctionHandler.
        '''
        return Dashboard(Dashboard.df.copy(), doc)

    def __init__(self, df, doc):
        '''
        Called when a user connects. This dashboard should be used
        while the connection remain, and is used by only one user.
        '''

        state = doc.session_context.request.arguments.get('state')
        if state:
            state = parse_qs(base64.urlsafe_b64decode(state[0]).decode())
        else:
            state = {}

        self.chart_types = {
            'linha': self.plot_lines,
            'barras horizontais': self.plot_hbar,
            'barras verticais': self.plot_vbar,
            'círculos': self.plot_circles,
        }
        charts_names = list(self.chart_types)
        none_value = 'Nenhum'

        self.df = df
        self.doc = doc
        self.columns = sorted(df.columns)
        self.discrete = [x for x in self.columns if self.df[x].dtype == object]
        # continuous = [x for x in columns if x not in discrete]
        # discrete.append(continuous.pop(continuous.index('ano')))
        self.filter_options = [none_value]+self.columns

        gui_data = {
            'x_sel': {
                'name': 'x',
                'title': 'Eixo X',
                'value': 'ano',
                'options': self.columns,
                'class': Select,
            },
            'y_sel': {
                'name': 'y',
                'title': 'Eixo Y',
                'value': 'pop_masc',
                'options': self.columns,
                'class': Select,
            },
            'chart_type_sel': {
                'name': 'type',
                'title': 'Tipo de Gráfico',
                'value': charts_names[2],
                'options': charts_names,
                'class': Select,
            },
            'filter_sel': {
                'name': 'filter',
                'title': 'Filtro',
                'value': self.filter_options[0],
                'options': self.filter_options,
                'class': Select,
            },
            'filter_value_sel': {
                'name': 'f_value',
                'title': 'Valor',
                'value': none_value,
                'options': [none_value],
                'class': Select,
            },
            # TODO: this widget is bugged, but it seems they are improving it.
            'filter_range_sel': {
                'name': 'f_range',
                'title': 'Intervalo',
                'start': 0,
                'end': 10,
                'step': 1,
                'value': [1, 9],
                'class': RangeSlider,
            },
        }

        # If has a previous state (passed throught querystring)
        # restore it.
        # TODO: validate state? Dangerous?
        if state:
            for gui, data in gui_data.items():
                value = state.get(data['name'])
                if value:
                    value = value[0]
                    if data['name'] == 'f_range':
                        data['value'] = tuple(float(i) for i in value.split(','))
                    else:
                        data['value'] = value

        # Create gui inputs
        for k, v in gui_data.items():
            v = v.copy()
            class_ = v.pop('class')
            setattr(self, k, class_(**v))

        controls = [getattr(self, k) for k, v in gui_data.items()]

        # Bind callbacks
        js_update_querytstring = CustomJS.from_py_func(update_querystring)
        for control in controls:
            control.on_change('value', self.update)
            control.callback = js_update_querytstring

        # TODO: seems to have no effect
        self.filter_range_sel.callback_policy = 'mouseup'

        controls = widgetbox(controls, width=200)
        self.layout = row(controls, self.create_figure())
        doc.add_root(self.layout)

    def update(self, attr, old, new):
        '''
        Called when chart options are modified.
        '''
        # It seems sometimes, when restoring state, self has no layout.
        # This if avoids error.
        if self.layout:
            self.layout.children[1] = self.create_figure(attr, old, new)

    # TODO: move helpers (don't really need self) to a new file?
    def plot_lines(self, fig, x, y, source):
        fig.line(x, y, source=source)
        # plot.line(x=xs, width=0.5, bottom=0, top=ys)

    def plot_vbar(self, fig, x, y, source):
        fig.vbar(x, 0.5, y, source=source)
        # plot.vbar(x=xs, width=0.5, bottom=0, top=ys)

    def plot_hbar(self, fig, x, y, source):
        fig.hbar(y=y, height=0.5, right=x, left=0, source=source)

    def plot_circles(self, fig, x, y, source):
        fig.circle(x, y, source=source)
        # plot.circle(x=xs, y=ys)
    # -----------------------------------------

    def handle_filtering(self, df):
        '''
        Update filter selectors widgets and use their values
        to filter `df`.
        '''
        frs = self.filter_range_sel
        fvs = self.filter_value_sel
        if self.filter_sel.value != self.filter_options[0]:
            filter_column = df[self.filter_sel.value]
            if self.filter_sel.value in self.discrete:
                # Discrete filter

                # update selector
                options = sorted(list(filter_column.unique()))
                fvs.options = options
                if fvs.value not in options:
                    fvs.value = options[0]

                show_widget(fvs)
                hide_widget(frs)

                # filter df
                df = df[filter_column == fvs.value]

            else:
                # Continuous filter

                min_val = filter_column.min()
                max_val = filter_column.max()

                # update slider
                frs.start = min_val
                frs.end = max_val
                if frs.value[0] < min_val or frs.value[1] > max_val:
                    frs.value = (min_val, max_val)
                frs.step = (max_val-min_val)/10

                show_widget(frs)
                hide_widget(fvs)

                # filter df
                df = df[filter_column >= frs.value[0]]
                df = df[filter_column <= frs.value[1]]
        else:
            # No filter
            hide_widget(fvs)
            # TODO: hide when RangeSlider css_classes work again
            # hide_widget(frs)

        return df

    def create_figure(self, attr=None, old=None, new=None):
        '''
        Creates the chart.
        '''
        df = self.handle_filtering(self.df)
        df = df.groupby(self.x_sel.value).sum()
        source = ColumnDataSource(data=df)

        xs = self.df[self.x_sel.value].values
        ys = self.df[self.y_sel.value].values
        x_title = self.x_sel.value.title()
        y_title = self.y_sel.value.title()

        kw = dict()
        if self.x_sel.value in self.discrete:
            kw['x_range'] = sorted(set(xs))
        if self.y_sel.value in self.discrete:
            kw['y_range'] = sorted(set(ys))
        kw['title'] = "%s vs %s" % (x_title, y_title)

        fig = figure(plot_height=600, plot_width=800,
                     tools='pan,box_zoom,reset,save', **kw)
        fig.xaxis.axis_label = x_title
        fig.yaxis.axis_label = y_title

        if self.x_sel.value in self.discrete:
            fig.xaxis.major_label_orientation = pd.np.pi / 4

        self.chart_types[self.chart_type_sel.value](
            fig, self.x_sel.value, self.y_sel.value, source)

        hover = HoverTool(tooltips=[
            ("index", "$index"),
            (self.x_sel.value, '@'+self.x_sel.value),
            (self.y_sel.value, '@'+self.y_sel.value),
        ])
        fig.add_tools(hover)

        return fig

class BackgroundBokeh(threading.Thread):

    def run(self):
        bokeh_app = Application(FunctionHandler(Dashboard.generate))
        io_loop = IOLoop.current()
        server = Server({'/bkapp': bokeh_app}, io_loop=io_loop,
                        allow_websocket_origin=["localhost:8000"])
        server.start()
        io_loop.start()

    def stop(self):
        IOLoop.current().stop()

bokeh_server = BackgroundBokeh()

def start_server():
    print('> Starting Bokeh server')
    bokeh_server.start()

def stop_server():
    print('> Stopping Bokeh server')
    bokeh_server.stop()
<|endoftext|>


<commit_before>import re
import sys
import time
import binascii as ba
from warnings import warn
from copy import deepcopy

from carto.exceptions import CartoException, CartoRateLimitException

from ..columns import Column

try:
    import geopandas
    HAS_GEOPANDAS = True
except ImportError:
    HAS_GEOPANDAS = False

DEFAULT_RETRY_TIMES = 3

GEOM_COLUMN_NAMES = [
    'geometry',
    'the_geom',
    'wkt_geometry',
    'wkb_geometry',
    'geom',
    'wkt',
    'wkb'
]

LAT_COLUMN_NAMES = [
    'latitude',
    'lat'
]

LNG_COLUMN_NAMES = [
    'longitude',
    'lng',
    'lon',
    'long'
]

ENC_SHAPELY = 'shapely'
ENC_WKB = 'wkb'
ENC_WKB_HEX = 'wkb-hex'
ENC_WKB_BHEX = 'wkb-bhex'
ENC_WKT = 'wkt'
ENC_EWKT = 'ewkt'

if (sys.version_info < (3, 0)):
    ENC_WKB_BHEX = ENC_WKB_HEX

def compute_query(dataset):
    if dataset.table_name:
        return 'SELECT * FROM "{schema}"."{table}"'.format(
            schema=dataset.schema or dataset._get_schema() or 'public',
            table=dataset.table_name
        )

def compute_geodataframe(dataset):
    if HAS_GEOPANDAS and dataset.dataframe is not None:
        df = dataset.dataframe
        geom_column = _get_column(df, GEOM_COLUMN_NAMES)
        if geom_column is not None:
            df['geometry'] = _compute_geometry_from_geom(geom_column)
            _warn_new_geometry_column(df)
        else:
            lat_column = _get_column(df, LAT_COLUMN_NAMES)
            lng_column = _get_column(df, LNG_COLUMN_NAMES)
            if lat_column is not None and lng_column is not None:
                df['geometry'] = _compute_geometry_from_latlng(lat_column, lng_column)
                _warn_new_geometry_column(df)
            else:
                raise ValueError('''No geographic data found. '''
                                 '''If a geometry exists, change the column name ({0}) or '''
                                 '''ensure it is a DataFrame with a valid geometry. '''
                                 '''If there are latitude/longitude columns, rename to ({1}), ({2}).'''.format(
                                     ', '.join(GEOM_COLUMN_NAMES),
                                     ', '.join(LAT_COLUMN_NAMES),
                                     ', '.join(LNG_COLUMN_NAMES)
                                 ))
        return geopandas.GeoDataFrame(df)

def _get_column(df, options):
    for name in options:
        if name in df:
            return df[name]

def _warn_new_geometry_column(df):
    if 'geometry' not in df:
        warn('A new "geometry" column has been added to the original dataframe.')

def _compute_geometry_from_geom(geom_column):
    first_geom = next(item for item in geom_column if item is not None)
    enc_type = detect_encoding_type(first_geom)
    return geom_column.apply(lambda g: decode_geometry(g, enc_type))

def _compute_geometry_from_latlng(lat, lng):
    from shapely import geometry
    return [geometry.Point(xy) for xy in zip(lng, lat)]

def _encode_decode_decorator(func):
    """decorator for encoding and decoding geoms"""
    def wrapper(*args):
        """error catching"""
        try:
            processed_geom = func(*args)
            return processed_geom
        except ImportError as err:
            raise ImportError('The Python package `shapely` needs to be '
                              'installed to encode or decode geometries. '
                              '({})'.format(err))
    return wrapper

@_encode_decode_decorator
def decode_geometry(geom, enc_type):
    """Decode any geometry into a shapely geometry."""
    if geom:
        func = {
            ENC_SHAPELY: lambda: geom,
            ENC_WKB: lambda: _load_wkb(geom),
            ENC_WKB_HEX: lambda: _load_wkb_hex(geom),
            ENC_WKB_BHEX: lambda: _load_wkb_bhex(geom),
            ENC_WKT: lambda: _load_wkt(geom),
            ENC_EWKT: lambda: _load_ewkt(geom)
        }.get(enc_type)
        return func() if func else geom
    return ''

def detect_encoding_type(input_geom):
    """
    Detect geometry encoding type:
    - ENC_WKB: b'\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00H\x93@\x00\x00\x00\x00\x00\x9d\xb6@'
    - ENC_EWKB: b'\x01\x01\x00\x00 \xe6\x10\x00\x00\x00\x00\x00\x00\x00H\x93@\x00\x00\x00\x00\x00\x9d\xb6@'
    - ENC_WKB_HEX: '0101000000000000000048934000000000009DB640'
    - ENC_EWKB_HEX: '0101000020E6100000000000000048934000000000009DB640'
    - ENC_WKB_BHEX: b'0101000000000000000048934000000000009DB640'
    - ENC_EWKB_BHEX: b'0101000020E6100000000000000048934000000000009DB640'
    - ENC_WKT: 'POINT (1234 5789)'
    - ENC_EWKT: 'SRID=4326;POINT (1234 5789)'
    """
    from shapely import wkb
    from shapely.geometry.base import BaseGeometry

    if isinstance(input_geom, BaseGeometry):
        return ENC_SHAPELY

    if isinstance(input_geom, str):
        if re.match(r'^[0-9a-fA-F]+$', input_geom):
            return ENC_WKB_HEX
        else:
            result = re.match(r'^SRID=\d+;(.*)$', input_geom)
            if result:
                geom = result.group(1)
                if geom != '':
                    return ENC_EWKT
            elif input_geom != '':
                # This is required because in P27 bytes = str
                try:
                    wkb.loads(input_geom)
                    return ENC_WKB
                except Exception:
                    return ENC_WKT

    if isinstance(input_geom, bytes):
        try:
            ba.unhexlify(input_geom)
            return ENC_WKB_BHEX
        except Exception:
            return ENC_WKB

def _load_wkb(geom):
    """Load WKB or EWKB geometry."""
    from shapely.wkb import loads
    return loads(geom)

def _load_wkb_hex(geom):
    """Load WKB_HEX or EWKB_HEX geometry."""
    from shapely.wkb import loads
    return loads(geom, hex=True)

def _load_wkb_bhex(geom):
    """Load WKB_BHEX or EWKB_BHEX geometry.
    The geom must be converted to WKB/EWKB before loading.
    """
    from shapely.wkb import loads
    return loads(ba.unhexlify(geom))

def _load_wkt(geom):
    """Load WKT geometry."""
    from shapely.wkt import loads
    return loads(geom)

def _load_ewkt(egeom):
    """Load EWKT geometry.
    The SRID must be removed before loading and added after loading.
    """
    from shapely.wkt import loads
    (srid, geom) = _extract_srid(egeom)
    ogeom = loads(geom)
    if srid:
        from shapely.geos import lgeos
        lgeos.GEOSSetSRID(ogeom._geom, int(srid))
    return ogeom

def _extract_srid(egeom):
    result = re.match(r'^SRID=(\d+);(.*)$', egeom)
    if result:
        return (result.group(1), result.group(2))
    else:
        return (0, egeom)

def recursive_read(context, query, retry_times=DEFAULT_RETRY_TIMES):
    try:
        return context.copy_client.copyto_stream(query)
    except CartoRateLimitException as err:
        if retry_times > 0:
            retry_times -= 1
            warn('Read call rate limited. Waiting {s} seconds'.format(s=err.retry_after))
            time.sleep(err.retry_after)
            warn('Retrying...')
            return recursive_read(context, query, retry_times=retry_times)
        else:
            warn(('Read call was rate-limited. '
                  'This usually happens when there are multiple queries being read at the same time.'))
            raise err

def get_columns(context, query):
    col_query = '''SELECT * FROM ({query}) _q LIMIT 0'''.format(query=query)
    table_info = context.sql_client.send(col_query)
    return Column.from_sql_api_fields(table_info['fields'])

def setting_value_exception(prop, value):
    return CartoException(("Error setting {prop}. You must use the `update` method: "
                           "dataset_info.update({prop}='{value}')").format(prop=prop, value=value))

def get_public_context(context):
    api_key = 'default_public'

    public_context = deepcopy(context)
    public_context.auth_client.api_key = api_key
    public_context.auth_api_client.api_key = api_key

    return public_context
<commit_msg>Simplify detect_encoding method<commit_after>import re
import sys
import time
import binascii as ba
from warnings import warn
from copy import deepcopy

from carto.exceptions import CartoException, CartoRateLimitException

from ..columns import Column

try:
    import geopandas
    HAS_GEOPANDAS = True
except ImportError:
    HAS_GEOPANDAS = False

DEFAULT_RETRY_TIMES = 3

GEOM_COLUMN_NAMES = [
    'geometry',
    'the_geom',
    'wkt_geometry',
    'wkb_geometry',
    'geom',
    'wkt',
    'wkb'
]

LAT_COLUMN_NAMES = [
    'latitude',
    'lat'
]

LNG_COLUMN_NAMES = [
    'longitude',
    'lng',
    'lon',
    'long'
]

ENC_SHAPELY = 'shapely'
ENC_WKB = 'wkb'
ENC_WKB_HEX = 'wkb-hex'
ENC_WKB_BHEX = 'wkb-bhex'
ENC_WKT = 'wkt'
ENC_EWKT = 'ewkt'

if (sys.version_info < (3, 0)):
    ENC_WKB_BHEX = ENC_WKB_HEX

def compute_query(dataset):
    if dataset.table_name:
        return 'SELECT * FROM "{schema}"."{table}"'.format(
            schema=dataset.schema or dataset._get_schema() or 'public',
            table=dataset.table_name
        )

def compute_geodataframe(dataset):
    if HAS_GEOPANDAS and dataset.dataframe is not None:
        df = dataset.dataframe
        geom_column = _get_column(df, GEOM_COLUMN_NAMES)
        if geom_column is not None:
            df['geometry'] = _compute_geometry_from_geom(geom_column)
            _warn_new_geometry_column(df)
        else:
            lat_column = _get_column(df, LAT_COLUMN_NAMES)
            lng_column = _get_column(df, LNG_COLUMN_NAMES)
            if lat_column is not None and lng_column is not None:
                df['geometry'] = _compute_geometry_from_latlng(lat_column, lng_column)
                _warn_new_geometry_column(df)
            else:
                raise ValueError('''No geographic data found. '''
                                 '''If a geometry exists, change the column name ({0}) or '''
                                 '''ensure it is a DataFrame with a valid geometry. '''
                                 '''If there are latitude/longitude columns, rename to ({1}), ({2}).'''.format(
                                     ', '.join(GEOM_COLUMN_NAMES),
                                     ', '.join(LAT_COLUMN_NAMES),
                                     ', '.join(LNG_COLUMN_NAMES)
                                 ))
        return geopandas.GeoDataFrame(df)

def _get_column(df, options):
    for name in options:
        if name in df:
            return df[name]

def _warn_new_geometry_column(df):
    if 'geometry' not in df:
        warn('A new "geometry" column has been added to the original dataframe.')

def _compute_geometry_from_geom(geom_column):
    first_geom = next(item for item in geom_column if item is not None)
    enc_type = detect_encoding_type(first_geom)
    return geom_column.apply(lambda g: decode_geometry(g, enc_type))

def _compute_geometry_from_latlng(lat, lng):
    from shapely import geometry
    return [geometry.Point(xy) for xy in zip(lng, lat)]

def _encode_decode_decorator(func):
    """decorator for encoding and decoding geoms"""
    def wrapper(*args):
        """error catching"""
        try:
            processed_geom = func(*args)
            return processed_geom
        except ImportError as err:
            raise ImportError('The Python package `shapely` needs to be '
                              'installed to encode or decode geometries. '
                              '({})'.format(err))
    return wrapper

@_encode_decode_decorator
def decode_geometry(geom, enc_type):
    """Decode any geometry into a shapely geometry."""
    if geom:
        func = {
            ENC_SHAPELY: lambda: geom,
            ENC_WKB: lambda: _load_wkb(geom),
            ENC_WKB_HEX: lambda: _load_wkb_hex(geom),
            ENC_WKB_BHEX: lambda: _load_wkb_bhex(geom),
            ENC_WKT: lambda: _load_wkt(geom),
            ENC_EWKT: lambda: _load_ewkt(geom)
        }.get(enc_type)
        return func() if func else geom
    return ''

def detect_encoding_type(input_geom):
    """
    Detect geometry encoding type:
    - ENC_WKB: b'\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00H\x93@\x00\x00\x00\x00\x00\x9d\xb6@'
    - ENC_EWKB: b'\x01\x01\x00\x00 \xe6\x10\x00\x00\x00\x00\x00\x00\x00H\x93@\x00\x00\x00\x00\x00\x9d\xb6@'
    - ENC_WKB_HEX: '0101000000000000000048934000000000009DB640'
    - ENC_EWKB_HEX: '0101000020E6100000000000000048934000000000009DB640'
    - ENC_WKB_BHEX: b'0101000000000000000048934000000000009DB640'
    - ENC_EWKB_BHEX: b'0101000020E6100000000000000048934000000000009DB640'
    - ENC_WKT: 'POINT (1234 5789)'
    - ENC_EWKT: 'SRID=4326;POINT (1234 5789)'
    """
    from shapely import wkb
    from shapely.geometry.base import BaseGeometry

    if isinstance(input_geom, BaseGeometry):
        return ENC_SHAPELY

    if isinstance(input_geom, str):
        if _is_hex(input_geom):
            return ENC_WKB_HEX
        else:
            srid, geom = _extract_srid(input_geom)
            if not geom:
                return
            if srid:
                return ENC_EWKT
            else:
                try:
                    # This is required because in P27 bytes = str
                    wkb.loads(geom)
                    return ENC_WKB
                except Exception:
                    return ENC_WKT

    if isinstance(input_geom, bytes):
        try:
            ba.unhexlify(input_geom)
            return ENC_WKB_BHEX
        except Exception:
            return ENC_WKB

def _load_wkb(geom):
    """Load WKB or EWKB geometry."""
    from shapely.wkb import loads
    return loads(geom)

def _load_wkb_hex(geom):
    """Load WKB_HEX or EWKB_HEX geometry."""
    from shapely.wkb import loads
    return loads(geom, hex=True)

def _load_wkb_bhex(geom):
    """Load WKB_BHEX or EWKB_BHEX geometry.
    The geom must be converted to WKB/EWKB before loading.
    """
    from shapely.wkb import loads
    return loads(ba.unhexlify(geom))

def _load_wkt(geom):
    """Load WKT geometry."""
    from shapely.wkt import loads
    return loads(geom)

def _load_ewkt(egeom):
    """Load EWKT geometry.
    The SRID must be removed before loading and added after loading.
    """
    from shapely.wkt import loads
    srid, geom = _extract_srid(egeom)
    ogeom = loads(geom)
    if srid:
        from shapely.geos import lgeos
        lgeos.GEOSSetSRID(ogeom._geom, int(srid))
    return ogeom

def _is_hex(input_geom):
    return re.match(r'^[0-9a-fA-F]+$', input_geom)

def _extract_srid(egeom):
    result = re.match(r'^SRID=(\d+);(.*)$', egeom)
    if result:
        return (result.group(1), result.group(2))
    else:
        return (0, egeom)

def recursive_read(context, query, retry_times=DEFAULT_RETRY_TIMES):
    try:
        return context.copy_client.copyto_stream(query)
    except CartoRateLimitException as err:
        if retry_times > 0:
            retry_times -= 1
            warn('Read call rate limited. Waiting {s} seconds'.format(s=err.retry_after))
            time.sleep(err.retry_after)
            warn('Retrying...')
            return recursive_read(context, query, retry_times=retry_times)
        else:
            warn(('Read call was rate-limited. '
                  'This usually happens when there are multiple queries being read at the same time.'))
            raise err

def get_columns(context, query):
    col_query = '''SELECT * FROM ({query}) _q LIMIT 0'''.format(query=query)
    table_info = context.sql_client.send(col_query)
    return Column.from_sql_api_fields(table_info['fields'])

def setting_value_exception(prop, value):
    return CartoException(("Error setting {prop}. You must use the `update` method: "
                           "dataset_info.update({prop}='{value}')").format(prop=prop, value=value))

def get_public_context(context):
    api_key = 'default_public'

    public_context = deepcopy(context)
    public_context.auth_client.api_key = api_key
    public_context.auth_api_client.api_key = api_key

    return public_context
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import datetime
import os
import subprocess

VERSION = (0, 1, 0, 'beta', 2)

def get_version():
    "Returns a PEP 386-compliant version number from VERSION."
    # Taken from django.utils.version
    assert VERSION[3] in ('alpha', 'beta', 'rc', 'final')

    # Now build the two parts of the version number:
    # main = X.Y[.Z]
    # sub = .devN - for pre-alpha releases
    #     | {a|b|c}N - for alpha, beta and rc releases

    parts = 2 if VERSION[2] == 0 else 3
    main = '.'.join(str(x) for x in VERSION[:parts])

    sub = ''
    if VERSION[3] == 'alpha' and VERSION[4] == 0:
        git_changeset = get_git_changeset()
        if git_changeset:
            sub = '.dev%s' % git_changeset

    elif VERSION[3] != 'final':
        mapping = {'alpha': 'a', 'beta': 'b', 'rc': 'c'}
        sub = mapping[VERSION[3]] + str(VERSION[4])

    return str(main + sub)

def get_git_changeset():
    """Returns a numeric identifier of the latest git changeset.

    The result is the UTC timestamp of the changeset in YYYYMMDDHHMMSS format.
    This value isn't guaranteed to be unique, but collisions are very unlikely,
    so it's sufficient for generating the development version numbers.
    """
    repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    git_log = subprocess.Popen('git log --pretty=format:%ct --quiet -1 HEAD',
            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
            shell=True, cwd=repo_dir, universal_newlines=True)
    timestamp = git_log.communicate()[0]
    try:
        timestamp = datetime.datetime.utcfromtimestamp(int(timestamp))
    except ValueError:
        return None
    return timestamp.strftime('%Y%m%d%H%M%S')

<commit_msg>Prepare for 0.1b3<commit_after># -*- coding: utf-8 -*-
import datetime
import os
import subprocess

VERSION = (0, 1, 0, 'beta', 3)

def get_version():
    "Returns a PEP 386-compliant version number from VERSION."
    # Taken from django.utils.version
    assert VERSION[3] in ('alpha', 'beta', 'rc', 'final')

    # Now build the two parts of the version number:
    # main = X.Y[.Z]
    # sub = .devN - for pre-alpha releases
    #     | {a|b|c}N - for alpha, beta and rc releases

    parts = 2 if VERSION[2] == 0 else 3
    main = '.'.join(str(x) for x in VERSION[:parts])

    sub = ''
    if VERSION[3] == 'alpha' and VERSION[4] == 0:
        git_changeset = get_git_changeset()
        if git_changeset:
            sub = '.dev%s' % git_changeset

    elif VERSION[3] != 'final':
        mapping = {'alpha': 'a', 'beta': 'b', 'rc': 'c'}
        sub = mapping[VERSION[3]] + str(VERSION[4])

    return str(main + sub)

def get_git_changeset():
    """Returns a numeric identifier of the latest git changeset.

    The result is the UTC timestamp of the changeset in YYYYMMDDHHMMSS format.
    This value isn't guaranteed to be unique, but collisions are very unlikely,
    so it's sufficient for generating the development version numbers.
    """
    repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    git_log = subprocess.Popen('git log --pretty=format:%ct --quiet -1 HEAD',
            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
            shell=True, cwd=repo_dir, universal_newlines=True)
    timestamp = git_log.communicate()[0]
    try:
        timestamp = datetime.datetime.utcfromtimestamp(int(timestamp))
    except ValueError:
        return None
    return timestamp.strftime('%Y%m%d%H%M%S')

<|endoftext|>


<commit_before>#!/usr/bin/env python2
# -*- coding:utf-8 -*-

import os
import json
import time
import numpy as np
from collections import Counter
from datetime import datetime
from c2cstats.parser import Outings, ParserError

ACT_SHORT = { u'alpinisme neige, glace, mixte': 'alpinisme',
              u'escalade': 'escalade',
              u'cascade de glace': 'glace',
              u'randonnée pédestre': 'rando',
              u'raquette': 'raquette',
              u'rocher haute montagne': 'rocher',
              u'ski, surf': 'ski' }

def generate_json(user_id, filename):
    """Generate a json file with all computed data.

    Arguments:
    - `filename`:
    - `user_id`:
    """

    t0 = time.time()
    data = Outings(user_id)

    t1 = time.time()
    act_long = dict([(v, k) for k,v in ACT_SHORT.items()])

    act_list = [ACT_SHORT[i] for i in data.activities]
    ctx = { 'activities': sorted(act_list),
            'nb_outings': data.nboutings }

    g = Global(data)
    ctx['global'] = { 'activities': g.activities,
                      'activities_per_year': g.activities_per_year,
                      'area': g.area }

    for act in act_list:
        d = globals()[act.title()](data)
        ctx[act] = { 'full_name': act_long[act].title(),
                     'cotation': getattr(d, 'cotation', []) }

    d = datetime.now()
    ctx['date_generated'] = unicode(d.strftime('%d %B %Y à %X'), 'utf-8')
    ctx['generation_time'] = 0 # TODO

    ctx['download_time'] = '{:.2}'.format(data.download_time)
    ctx['parse_time'] = '{:.2}'.format(data.parse_time)
    ctx['generation_time'] = '{:.3}'.format(time.time() - t1)
    ctx['total_time'] = '{:.2}'.format(time.time() - t0)

    with open(filename, 'w') as f:
        json.dump(ctx, f)

class Generator:
    def __init__(self, data):
        self.data = data

        self.year = np.array([int(i.split()[2]) for i in self.data.date])
        self.year_uniq = np.unique(self.year)
        self.year_labels = [str(i) for i in self.year_uniq]

    def filter_activity(self, arr, activity):
        arr_filtered = np.copy(arr)
        ind = (self.data.activity == activity)
        arr_filtered = arr_filtered[ind]
        return arr_filtered

    def cot_globale(self, activity=''):
        "Count number of outings per bin of cot_globale"

        if activity:
            c = Counter(filter_activity(self.data.cot_globale, activity))
        else:
            c = Counter(self.data.cot_globale)

        return [c[k] for k in COTATION_GLOBALE]

    def gain_per_year(self):

        if activity:
            gain = filter_activity(self.data.gain, activity)
            year = filter_activity(self.data.year, activity)
        else:
            gain = self.data.gain
            year = self.year

        counts = []
        for i in self.year_uniq:
            ind = (year == i)
            counts.append(np.sum(gain[ind]))

        return counts

class Global(Generator):
    def __init__(self, *args, **kwargs):
        Generator.__init__(self, *args, **kwargs)

    @property
    def activities(self):
        "Count number of outings per activity"
        c = Counter(self.data.activity)
        return { 'title': u'Répartition par activité',
                 'values': c.items() }

    @property
    def activities_per_year(self):
        "Compute the number of outings per year and per activity"
        h = []
        for i in self.data.activities:
            ind = (self.data.activity == i)
            h.append(list(self.year[ind]))

        return h

    @property
    def area(self):
        "Count the number of outings per area"
        c = Counter(self.data.area)
        use = c.most_common(10)
        sum_use = sum(zip(*use)[1])
        use.append((u'Autres', sum(c.values()) - sum_use))
        print use

        return { 'title': u'Répartition par région',
                 'values': use }

class Escalade(Generator):

    COTATION_REF = ('3a', '3b', '3c', '4a', '4b', '4c', '5a', '5a+', '5b',
                '5b+', '5c', '5c+', '6a', '6a+', '6b', '6b+', '6c',
                '6c+', '7a', '7a+', '7b', '7b+', '7c', '7c+', '8a')
    # '2', '8a+', '8b', '8b+', '8c', '8c+', '9a', '9a+', '9b', '9b+'

    def __init__(self, *args, **kwargs):
        Generator.__init__(self, *args, **kwargs)

    @property
    def cotation(self):
        return {
            'title': u'Cotation escalade',
            'labels': self.COTATION_REF,
            'values': self.cot_libre
            }

    @property
    def cot_libre(self):
        c1 = Counter(self.data.cot_libre)
        return [c1[k] for k in self.COTATION_REF]

    @property
    def cot_oblige(self):
        c2 = Counter(self.data.cot_oblige)
        return [c2[k] for k in self.COTATION_REF]

class Glace(Generator):

    COTATION_REF = ('2', '3', '3+', '4', '4+', '5', '5+', '6', '6+', '7', '7+')

    def __init__(self, *args, **kwargs):
        Generator.__init__(self, *args, **kwargs)

    @property
    def cotation(self):
        c = Counter(self.data.cot_glace)
        return {
            'title': u'Cotation glace',
            'labels': self.COTATION_REF,
            'values': [c[k] for k in self.COTATION_REF]
            }

class Rando(Generator):

    COTATION_REF = ('T1', 'T2', 'T3', 'T4', 'T5', 'T6')

    def __init__(self, *args, **kwargs):
        Generator.__init__(self, *args, **kwargs)

    @property
    def cotation(self):
        c = Counter(self.data.cot_rando)
        return {
            'title': u'Cotation rando',
            'labels': self.COTATION_REF,
            'values': [c[k] for k in self.COTATION_REF]
            }

class Alpinisme(Generator):
    pass

class Raquette(Generator):
    pass

class Rocher(Generator):
    pass

class Ski(Generator):
    pass

<commit_msg>cotation escalade - remove '+'<commit_after>#!/usr/bin/env python2
# -*- coding:utf-8 -*-

import os
import json
import time
import numpy as np
from collections import Counter
from datetime import datetime
from c2cstats.parser import Outings, ParserError

ACT_SHORT = { u'alpinisme neige, glace, mixte': 'alpinisme',
              u'escalade': 'escalade',
              u'cascade de glace': 'glace',
              u'randonnée pédestre': 'rando',
              u'raquette': 'raquette',
              u'rocher haute montagne': 'rocher',
              u'ski, surf': 'ski' }

def generate_json(user_id, filename):
    """Generate a json file with all computed data.

    Arguments:
    - `filename`:
    - `user_id`:
    """

    t0 = time.time()
    data = Outings(user_id)

    t1 = time.time()
    act_long = dict([(v, k) for k,v in ACT_SHORT.items()])

    act_list = [ACT_SHORT[i] for i in data.activities]
    ctx = { 'activities': sorted(act_list),
            'nb_outings': data.nboutings }

    g = Global(data)
    ctx['global'] = { 'activities': g.activities,
                      'activities_per_year': g.activities_per_year,
                      'area': g.area }

    for act in act_list:
        d = globals()[act.title()](data)
        ctx[act] = { 'full_name': act_long[act].title(),
                     'cotation': getattr(d, 'cotation', []) }

    d = datetime.now()
    ctx['date_generated'] = unicode(d.strftime('%d %B %Y à %X'), 'utf-8')
    ctx['generation_time'] = 0 # TODO

    ctx['download_time'] = '{:.2}'.format(data.download_time)
    ctx['parse_time'] = '{:.2}'.format(data.parse_time)
    ctx['generation_time'] = '{:.3}'.format(time.time() - t1)
    ctx['total_time'] = '{:.2}'.format(time.time() - t0)

    with open(filename, 'w') as f:
        json.dump(ctx, f)

class Generator:
    def __init__(self, data):
        self.data = data

        self.year = np.array([int(i.split()[2]) for i in self.data.date])
        self.year_uniq = np.unique(self.year)
        self.year_labels = [str(i) for i in self.year_uniq]

    def filter_activity(self, arr, activity):
        arr_filtered = np.copy(arr)
        ind = (self.data.activity == activity)
        arr_filtered = arr_filtered[ind]
        return arr_filtered

    def cot_globale(self, activity=''):
        "Count number of outings per bin of cot_globale"

        if activity:
            c = Counter(filter_activity(self.data.cot_globale, activity))
        else:
            c = Counter(self.data.cot_globale)

        return [c[k] for k in COTATION_GLOBALE]

    def gain_per_year(self):

        if activity:
            gain = filter_activity(self.data.gain, activity)
            year = filter_activity(self.data.year, activity)
        else:
            gain = self.data.gain
            year = self.year

        counts = []
        for i in self.year_uniq:
            ind = (year == i)
            counts.append(np.sum(gain[ind]))

        return counts

class Global(Generator):
    def __init__(self, *args, **kwargs):
        Generator.__init__(self, *args, **kwargs)

    @property
    def activities(self):
        "Count number of outings per activity"
        c = Counter(self.data.activity)
        return { 'title': u'Répartition par activité',
                 'values': c.items() }

    @property
    def activities_per_year(self):
        "Compute the number of outings per year and per activity"
        h = []
        for i in self.data.activities:
            ind = (self.data.activity == i)
            h.append(list(self.year[ind]))

        return h

    @property
    def area(self):
        "Count the number of outings per area"
        c = Counter(self.data.area)
        use = c.most_common(10)
        sum_use = sum(zip(*use)[1])
        use.append((u'Autres', sum(c.values()) - sum_use))
        print use

        return { 'title': u'Répartition par région',
                 'values': use }

class Escalade(Generator):

    COTATION_REF = ('3a', '3b', '3c', '4a', '4b', '4c', '5a', '5b',
                    '5c', '6a', '6b', '6c', '7a', '7b', '7c', '8a')

    def __init__(self, *args, **kwargs):
        Generator.__init__(self, *args, **kwargs)

    @property
    def cotation(self):
        return {
            'title': u'Cotation escalade',
            'labels': self.COTATION_REF,
            'values': self.cot_libre
            }

    @property
    def cot_libre(self):
        remove_plus = lambda x: x.replace('+','')
        vrem = np.vectorize(remove_plus)
        c1 = Counter(vrem(self.data.cot_libre))
        return [c1[k] for k in self.COTATION_REF]

    @property
    def cot_oblige(self):
        c2 = Counter(self.data.cot_oblige)
        return [c2[k] for k in self.COTATION_REF]

class Glace(Generator):

    COTATION_REF = ('2', '3', '3+', '4', '4+', '5', '5+', '6', '6+', '7', '7+')

    def __init__(self, *args, **kwargs):
        Generator.__init__(self, *args, **kwargs)

    @property
    def cotation(self):
        c = Counter(self.data.cot_glace)
        return {
            'title': u'Cotation glace',
            'labels': self.COTATION_REF,
            'values': [c[k] for k in self.COTATION_REF]
            }

class Rando(Generator):

    COTATION_REF = ('T1', 'T2', 'T3', 'T4', 'T5', 'T6')

    def __init__(self, *args, **kwargs):
        Generator.__init__(self, *args, **kwargs)

    @property
    def cotation(self):
        c = Counter(self.data.cot_rando)
        return {
            'title': u'Cotation rando',
            'labels': self.COTATION_REF,
            'values': [c[k] for k in self.COTATION_REF]
            }

class Alpinisme(Generator):
    pass

class Raquette(Generator):
    pass

class Rocher(Generator):
    pass

class Ski(Generator):
    pass

<|endoftext|>


<commit_before>from datetime import date as vanilla_date, timedelta

from base import BasicBCEDate
from ..helpers import ordinal, month_string
from ..constants import first_julian_date

class BCEDate(BasicBCEDate):
    def __str__(self):
        date_string = "%s %s %s BCE" % (
            ordinal(self.day),
            month_string(self.month),
            -self.year
        )
        display_name = (
            "Julian Calendar"
            if (self >= first_julian_date)
            else "Proleptic Julian Calendar"
        )
        return "%s (%s)" % (date_string, display_name)

    def __ge__(self, other):
        if self.year != other.year:
            return self.year > other.year
        if self.month != other.month:
            return self.month > other.month
        return self.day >= other.day

    @property
    def _date(self):
        return self

    @staticmethod
    def _is_julian_leap_year(y):
        return (y % 4) == 0

    @staticmethod
    def _month_length(y, m):
        if BCEDate._is_julian_leap_year(y):
            return [None, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][m]
        else:
            return [None, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][m]

    @staticmethod
    def _year_length(y):
        if y == 0:
            return 0
        if BCEDate._is_julian_leap_year(y):
            return 366
        else:
            return 365

    @staticmethod
    def _subtract(a, b):
        if (a.year == b.year and a.month == b.month and a.day == b.day):
            return 0
        if ((a.year < b.year) or
                (a.year == b.year and a.month < b.month) or
                (a.year == b.year and a.month == b.month and a.day < b.day)):
            return -BCEDate._subtract(b, a)
        if (a.year == b.year and a.month == b.month):
            return a.day - b.day
        if (a.day != 1):
            return a.day - 1 + BCEDate._subtract(BCEDate(a.year, a.month, 1), b)
        if (a.month != 3):
            if a.month == 1:
                previous_year = a.year - 1
                if previous_year == 0:
                    previous_year = -1
                return 31 + BCEDate._subtract(BCEDate(previous_year, 12, 1), b)
            return BCEDate._month_length(a.year, a.month - 1) + BCEDate._subtract(BCEDate(a.year, a.month - 1, 1), b)
        if a.year - b.year > 3:
            n_cycles = (a.year - b.year) // 4
            return n_cycles * (365 * 3 + 366) + BCEDate._subtract(BCEDate(a.year - 4 * n_cycles, 3, 1), b)
        return BCEDate._year_length(a.year) + BCEDate._subtract(BCEDate(a.year - 1, 3, 1), b)

    def __sub__(self, other):
        try:
            other_date = other._date
        except:
            other_date = other
        return timedelta(days=BCEDate._subtract(self._date, other_date))
<commit_msg>Spacing and line length cleanups.<commit_after>from datetime import timedelta

from base import BasicBCEDate
from ..helpers import ordinal, month_string
from ..constants import first_julian_date

class BCEDate(BasicBCEDate):
    def __str__(self):
        date_string = "%s %s %s BCE" % (
            ordinal(self.day),
            month_string(self.month),
            -self.year
        )
        display_name = (
            "Julian Calendar"
            if (self >= first_julian_date)
            else "Proleptic Julian Calendar"
        )
        return "%s (%s)" % (date_string, display_name)

    def __ge__(self, other):
        if self.year != other.year:
            return self.year > other.year
        if self.month != other.month:
            return self.month > other.month
        return self.day >= other.day

    @property
    def _date(self):
        return self

    @staticmethod
    def _is_julian_leap_year(y):
        return (y % 4) == 0

    @staticmethod
    def _month_length(y, m):
        if BCEDate._is_julian_leap_year(y):
            return [None, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][m]
        else:
            return [None, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][m]

    @staticmethod
    def _year_length(y):
        if y == 0:
            return 0
        if BCEDate._is_julian_leap_year(y):
            return 366
        else:
            return 365

    @staticmethod  # noqa
    def _subtract(a, b):
        if (a.year == b.year and a.month == b.month and a.day == b.day):
            return 0
        if ((a.year < b.year) or
                (a.year == b.year and a.month < b.month) or
                (a.year == b.year and a.month == b.month and a.day < b.day)):
            return -BCEDate._subtract(b, a)
        if (a.year == b.year and a.month == b.month):
            return a.day - b.day
        if (a.day != 1):
            return a.day - 1 + BCEDate._subtract(BCEDate(a.year, a.month, 1), b)
        if (a.month != 3):
            if a.month == 1:
                previous_year = a.year - 1
                if previous_year == 0:
                    previous_year = -1
                return 31 + BCEDate._subtract(BCEDate(previous_year, 12, 1), b)
            return (
                BCEDate._month_length(a.year, a.month - 1) +
                BCEDate._subtract(BCEDate(a.year, a.month - 1, 1), b)
            )
        if a.year - b.year > 3:
            n_cycles = (a.year - b.year) // 4
            n_days = n_cycles * (365 * 3 + 366)
            return n_days + BCEDate._subtract(BCEDate(a.year - 4 * n_cycles, 3, 1), b)
        return (
            BCEDate._year_length(a.year) +
            BCEDate._subtract(BCEDate(a.year - 1, 3, 1), b)
        )

    def __sub__(self, other):
        try:
            other_date = other._date
        except:
            other_date = other
        return timedelta(days=BCEDate._subtract(self._date, other_date))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2015 Metaswitch Networks
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
calico.datamodel
~~~~~~~~~~~~~~~~

Shared etcd data-model definitions for version 1 of the data model.

This file is versioned.  The idea is that only back-compatible changes
should be made to this file and non-back-compatible changes should be
made in a new copy of the file with revved version suffix.  That allows
us to maintain multiple copies of the data model in parallel during
migrations.
"""
import logging
import re

_log = logging.getLogger(__name__)

# All Calico data is stored under this path.
ROOT_DIR = "/calico"

# Current versions
FELIX_VERSION = "/v1"
OPENSTACK_VERSION = "/v1"

# OpenStack data is stored under this path.
OPENSTACK_DIR = ROOT_DIR + "/openstack"
OPENSTACK_VERSION_DIR = OPENSTACK_DIR + OPENSTACK_VERSION

# Status data and reporting
FELIX_STATUS_DIR = ROOT_DIR + "/felix" + FELIX_VERSION + "/host"

# Data that flows from orchestrator to felix is stored under a versioned
# sub-tree.
VERSION_DIR = ROOT_DIR + FELIX_VERSION
# Global ready flag.  Stores 'true' or 'false'.
READY_KEY = VERSION_DIR + "/Ready"
# Global config (directory).
CONFIG_DIR = VERSION_DIR + '/config'
HOST_DIR = VERSION_DIR + '/host'
POLICY_DIR = VERSION_DIR + '/policy'
PROFILE_DIR = POLICY_DIR + "/profile"

# Key used for leader election by Neutron mechanism drivers.
NEUTRON_ELECTION_KEY = OPENSTACK_VERSION_DIR + '/neutron_election'

# Regex to match profile rules, capturing the profile ID in capture group
# "profile_id".
RULES_KEY_RE = re.compile(
    r'^' + PROFILE_DIR + r'/(?P<profile_id>[^/]+)/rules')
# Regex to match profile tags, capturing the profile ID in capture group
# "profile_id".
TAGS_KEY_RE = re.compile(
    r'^' + PROFILE_DIR + r'/(?P<profile_id>[^/]+)/tags')
# Regex to match endpoints, captures "hostname" and "endpoint_id".  Works for
# endpoint configuration and endpoint status paths.
ENDPOINT_KEY_RE = re.compile(
    r'^(?:' + HOST_DIR + r'|' + FELIX_STATUS_DIR + r')'
    r'/(?P<hostname>[^/]+)/'
    r'workload/'
    r'(?P<orchestrator>[^/]+)/'
    r'(?P<workload_id>[^/]+)/'
    r'endpoint/(?P<endpoint_id>[^/]+)')

HOST_IP_KEY_RE = re.compile(r'^' + HOST_DIR +
                            r'/(?P<hostname>[^/]+)/bird_ip')

IPAM_V4_CIDR_KEY_RE = re.compile(r'^' + VERSION_DIR +
                                 r'/ipam/v4/pool/(?P<encoded_cidr>[^/]+)')

ENDPOINT_STATUS_UP = "up"
ENDPOINT_STATUS_DOWN = "down"
ENDPOINT_STATUS_ERROR = "error"

def dir_for_host(hostname):
    return HOST_DIR + "/%s" % hostname

def dir_for_per_host_config(hostname):
    return dir_for_host(hostname) + "/config"

def dir_for_felix_status(hostname):
    return FELIX_STATUS_DIR + "/%s" % hostname

def key_for_last_status(hostname):
    return dir_for_felix_status(hostname) + "/last_reported_status"

def key_for_status(hostname):
    return dir_for_felix_status(hostname) + "/status"

def key_for_endpoint(host, orchestrator, workload_id, endpoint_id):
    return (HOST_DIR + "/%s/workload/%s/%s/endpoint/%s" %
            (host, orchestrator, workload_id, endpoint_id))

def key_for_profile(profile_id):
    return PROFILE_DIR + "/" + profile_id

def key_for_profile_rules(profile_id):
    return PROFILE_DIR + "/%s/rules" % profile_id

def key_for_profile_tags(profile_id):
    return PROFILE_DIR + "/%s/tags" % profile_id

def key_for_config(config_name):
    return CONFIG_DIR + "/%s" % config_name

def get_profile_id_for_profile_dir(key):
    """
    :param str key: etcd key.
    :returns The profile ID if this is a profile dir or None if not.
    """
    key = key.rstrip('/')
    if "/" not in key:
        return None
    prefix, final_node = key.rsplit("/", 1)
    return final_node if prefix == PROFILE_DIR else None

def get_endpoint_id_from_key(key):
    m = ENDPOINT_KEY_RE.match(key)
    if m:
        # Got an endpoint.
        host = m.group("hostname")
        orch = m.group("orchestrator")
        workload_id = m.group("workload_id")
        endpoint_id = m.group("endpoint_id")
        combined_id = EndpointId(host, orch, workload_id, endpoint_id)
        return combined_id
    else:
        return None

def hostname_from_status_key(key):
    """
    Get hostname from a status key (or None if this is not a status key).

    :param: key for felix status
            expected key format: FELIX_STATUS_DIR/<hostname>/
                                                      <some path or not>/status
    """
    if not key.startswith(FELIX_STATUS_DIR) or not key.endswith("/status"):
        return None
    in_host_dir = key[len(FELIX_STATUS_DIR + '/'):]
    path = in_host_dir.split('/', 1)
    hostname = path[0]
    return hostname

class EndpointId(object):
    __slots__ = ["host", "orchestrator", "workload", "endpoint"]

    def __init__(self, host, orchestrator, workload, endpoint):
        # We intern these strings since they can occur in many IDs.  The
        # host and orchestrator are trivially repeated for all endpoints
        # on a host.  The others get repeated over time.
        self.host = intern(host.encode("utf8"))
        self.orchestrator = intern(orchestrator.encode("utf8"))
        self.workload = intern(workload.encode("utf8"))
        self.endpoint = intern(endpoint.encode("utf8"))

    @property
    def path_for_status(self):
        return "/".join([FELIX_STATUS_DIR, self.host,
                         "workload", self.orchestrator, self.workload,
                         "endpoint", self.endpoint])

    def __str__(self):
        return self.__class__.__name__ + ("<%s>" % self.endpoint)

    def __repr__(self):
        return self.__class__.__name__ + ("(%r,%r,%r,%r)" % (self.host,
                                                             self.orchestrator,
                                                             self.workload,
                                                             self.endpoint))

    def __eq__(self, other):
        if other is self:
            return True
        if not isinstance(other, EndpointId):
            return False
        return (other.endpoint == self.endpoint and
                other.workload == self.workload and
                other.host == self.host and
                other.orchestrator == self.orchestrator)

    def __ne__(self, other):
        return not (self == other)

    def __hash__(self):
        return hash(self.endpoint) + hash(self.workload)
<commit_msg>Add DHCP agent information to data model<commit_after># -*- coding: utf-8 -*-
# Copyright 2015 Metaswitch Networks
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
calico.datamodel
~~~~~~~~~~~~~~~~

Shared etcd data-model definitions for version 1 of the data model.

This file is versioned.  The idea is that only back-compatible changes
should be made to this file and non-back-compatible changes should be
made in a new copy of the file with revved version suffix.  That allows
us to maintain multiple copies of the data model in parallel during
migrations.
"""
import logging
import re

_log = logging.getLogger(__name__)

# All Calico data is stored under this path.
ROOT_DIR = "/calico"

# Current versions
FELIX_VERSION = "/v1"
OPENSTACK_VERSION = "/v1"
DHCP_VERSION = "/v1"

# OpenStack data is stored under this path.
OPENSTACK_DIR = ROOT_DIR + "/openstack"
OPENSTACK_VERSION_DIR = OPENSTACK_DIR + OPENSTACK_VERSION

# Status data and reporting
FELIX_STATUS_DIR = ROOT_DIR + "/felix" + FELIX_VERSION + "/host"

# Data that flows from orchestrator to felix is stored under a versioned
# sub-tree.
VERSION_DIR = ROOT_DIR + FELIX_VERSION
# Global ready flag.  Stores 'true' or 'false'.
READY_KEY = VERSION_DIR + "/Ready"
# Global config (directory).
CONFIG_DIR = VERSION_DIR + '/config'
HOST_DIR = VERSION_DIR + '/host'
POLICY_DIR = VERSION_DIR + '/policy'
PROFILE_DIR = POLICY_DIR + "/profile"

# Key used for leader election by Neutron mechanism drivers.
NEUTRON_ELECTION_KEY = OPENSTACK_VERSION_DIR + '/neutron_election'

# Regex to match profile rules, capturing the profile ID in capture group
# "profile_id".
RULES_KEY_RE = re.compile(
    r'^' + PROFILE_DIR + r'/(?P<profile_id>[^/]+)/rules')
# Regex to match profile tags, capturing the profile ID in capture group
# "profile_id".
TAGS_KEY_RE = re.compile(
    r'^' + PROFILE_DIR + r'/(?P<profile_id>[^/]+)/tags')
# Regex to match endpoints, captures "hostname" and "endpoint_id".  Works for
# endpoint configuration and endpoint status paths.
ENDPOINT_KEY_RE = re.compile(
    r'^(?:' + HOST_DIR + r'|' + FELIX_STATUS_DIR + r')'
    r'/(?P<hostname>[^/]+)/'
    r'workload/'
    r'(?P<orchestrator>[^/]+)/'
    r'(?P<workload_id>[^/]+)/'
    r'endpoint/(?P<endpoint_id>[^/]+)')

HOST_IP_KEY_RE = re.compile(r'^' + HOST_DIR +
                            r'/(?P<hostname>[^/]+)/bird_ip')

IPAM_V4_CIDR_KEY_RE = re.compile(r'^' + VERSION_DIR +
                                 r'/ipam/v4/pool/(?P<encoded_cidr>[^/]+)')

ENDPOINT_STATUS_UP = "up"
ENDPOINT_STATUS_DOWN = "down"
ENDPOINT_STATUS_ERROR = "error"

# Information intended for use by the DHCP agent.
DHCP_DIR = ROOT_DIR + "/dhcp" + DHCP_VERSION
SUBNET_DIR= DHCP_DIR + "/subnet"

def dir_for_host(hostname):
    return HOST_DIR + "/%s" % hostname

def dir_for_per_host_config(hostname):
    return dir_for_host(hostname) + "/config"

def dir_for_felix_status(hostname):
    return FELIX_STATUS_DIR + "/%s" % hostname

def key_for_last_status(hostname):
    return dir_for_felix_status(hostname) + "/last_reported_status"

def key_for_status(hostname):
    return dir_for_felix_status(hostname) + "/status"

def key_for_endpoint(host, orchestrator, workload_id, endpoint_id):
    return (HOST_DIR + "/%s/workload/%s/%s/endpoint/%s" %
            (host, orchestrator, workload_id, endpoint_id))

def key_for_profile(profile_id):
    return PROFILE_DIR + "/" + profile_id

def key_for_profile_rules(profile_id):
    return PROFILE_DIR + "/%s/rules" % profile_id

def key_for_profile_tags(profile_id):
    return PROFILE_DIR + "/%s/tags" % profile_id

def key_for_config(config_name):
    return CONFIG_DIR + "/%s" % config_name

def key_for_subnet(subnet_id):
    return SUBNET_DIR + "/%s" % subnet_id

def get_profile_id_for_profile_dir(key):
    """
    :param str key: etcd key.
    :returns The profile ID if this is a profile dir or None if not.
    """
    key = key.rstrip('/')
    if "/" not in key:
        return None
    prefix, final_node = key.rsplit("/", 1)
    return final_node if prefix == PROFILE_DIR else None

def get_endpoint_id_from_key(key):
    m = ENDPOINT_KEY_RE.match(key)
    if m:
        # Got an endpoint.
        host = m.group("hostname")
        orch = m.group("orchestrator")
        workload_id = m.group("workload_id")
        endpoint_id = m.group("endpoint_id")
        combined_id = EndpointId(host, orch, workload_id, endpoint_id)
        return combined_id
    else:
        return None

def hostname_from_status_key(key):
    """
    Get hostname from a status key (or None if this is not a status key).

    :param: key for felix status
            expected key format: FELIX_STATUS_DIR/<hostname>/
                                                      <some path or not>/status
    """
    if not key.startswith(FELIX_STATUS_DIR) or not key.endswith("/status"):
        return None
    in_host_dir = key[len(FELIX_STATUS_DIR + '/'):]
    path = in_host_dir.split('/', 1)
    hostname = path[0]
    return hostname

class EndpointId(object):
    __slots__ = ["host", "orchestrator", "workload", "endpoint"]

    def __init__(self, host, orchestrator, workload, endpoint):
        # We intern these strings since they can occur in many IDs.  The
        # host and orchestrator are trivially repeated for all endpoints
        # on a host.  The others get repeated over time.
        self.host = intern(host.encode("utf8"))
        self.orchestrator = intern(orchestrator.encode("utf8"))
        self.workload = intern(workload.encode("utf8"))
        self.endpoint = intern(endpoint.encode("utf8"))

    @property
    def path_for_status(self):
        return "/".join([FELIX_STATUS_DIR, self.host,
                         "workload", self.orchestrator, self.workload,
                         "endpoint", self.endpoint])

    def __str__(self):
        return self.__class__.__name__ + ("<%s>" % self.endpoint)

    def __repr__(self):
        return self.__class__.__name__ + ("(%r,%r,%r,%r)" % (self.host,
                                                             self.orchestrator,
                                                             self.workload,
                                                             self.endpoint))

    def __eq__(self, other):
        if other is self:
            return True
        if not isinstance(other, EndpointId):
            return False
        return (other.endpoint == self.endpoint and
                other.workload == self.workload and
                other.host == self.host and
                other.orchestrator == self.orchestrator)

    def __ne__(self, other):
        return not (self == other)

    def __hash__(self):
        return hash(self.endpoint) + hash(self.workload)
<|endoftext|>


<commit_before>'''
transforming (:mod:`calour.transforming`)
=========================================

.. currentmodule:: calour.transforming

Functions
^^^^^^^^^
.. autosummary::
   :toctree: generated

   normalize
   normalize_by_subset_features
   scale
   binarize
   log_n
   transform
'''

# ----------------------------------------------------------------------------
# Copyright (c) 2016--,  Calour development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

from logging import getLogger
from copy import deepcopy
from collections import defaultdict

import numpy as np
from sklearn import preprocessing

from .experiment import Experiment

logger = getLogger(__name__)

@Experiment._record_sig
def normalize(exp, total=10000, axis=0, inplace=False):
    '''Normalize the sum of each sample (axis=0) or feature (axis=1) to sum total

    Parameters
    ----------
    exp : Experiment
    total : float
        the sum (along axis) to normalize to
    axis : 0, 1, 's', or 'f', optional
        the axis to normalize. 0 or 's' (default) is normalize each sample;
        1 or 'f' to normalize each feature
    inplace : bool (optional)
        False (default) to create a copy, True to replace values in exp

    Returns
    -------
    ``Experiment``
        the normalized experiment
    '''
    if isinstance(total, bool):
        raise ValueError('Normalization total (%s) not numeric' % total)
    if total <= 0:
        raise ValueError('Normalization total (%s) must be positive' % total)
    if not inplace:
        exp = deepcopy(exp)
    exp.data = preprocessing.normalize(exp.data, norm='l1', axis=1-axis) * total
    # store the normalization depth into the experiment metadata
    exp.exp_metadata['normalized'] = total
    return exp

@Experiment._record_sig
def rescale(exp, total=10000, axis=0, inplace=False):
    '''Rescale the data to mean sum of all samples (axis=0) or features (axis=1) to be total.

    This function rescales by multiplying ALL entries in exp.data by same number.

    Parameters
    ----------
    exp : Experiment
    total : float
        the mean sum (along axis) to normalize to
    axis : 0, 1, 's', or 'f', optional
        the axis to normalize. 0 or 's' (default) is normalize each sample;
        1 or 'f' to normalize each feature
    inplace : bool (optional)
        False (default) to create a copy, True to replace values in exp

    Returns
    -------
    ``Experiment``
        the normalized experiment
    '''
    if isinstance(total, bool):
        raise ValueError('Rescaling total (%s) not numeric' % total)
    if not inplace:
        exp = deepcopy(exp)
    current_mean = np.mean(exp.data.sum(axis=1-axis))
    exp.data = exp.data * total / current_mean
    return exp

@Experiment._record_sig
def scale(exp, axis=0, inplace=False):
    '''Standardize a dataset along an axis

    .. warning:: It will convert the ``Experiment.data`` from the sparse matrix to dense array.

    Parameters
    ----------
    axis : 0, 1, 's', or 'f'
        0 or 's'  means scaling occur sample-wise; 1 or 'f' feature-wise.

    Returns
    -------
    ``Experiment``
    '''
    logger.debug('scaling the data, axis=%d' % axis)
    if not inplace:
        exp = deepcopy(exp)
    if exp.sparse:
        exp.sparse = False
    preprocessing.scale(exp.data, axis=1-axis, copy=False)
    return exp

@Experiment._record_sig
def binarize(exp, threshold=1, inplace=False):
    '''Binarize the data with a threshold.

    It calls scikit-learn to do the real work.

    Parameters
    ----------
    threshold : Numeric
        the cutoff value. Any values below or equal to this will be replaced by 0,
        above it by 1.

    Returns
    -------
    ``Experiment``
    '''
    logger.debug('binarizing the data. threshold=%f' % threshold)
    if not inplace:
        exp = deepcopy(exp)
    preprocessing.binarize(exp.data, threshold=threshold, copy=False)
    return exp

@Experiment._record_sig
def log_n(exp, n=1, inplace=False):
    '''Log transform the data

    Parameters
    ----------
    n : numeric, optional
        cap the tiny values and then log transform the data.
    inplace : bool, optional

    Returns
    -------
    ``Experiment``
    '''
    logger.debug('log_n transforming the data, min. threshold=%f' % n)
    if not inplace:
        exp = deepcopy(exp)

    if exp.sparse:
        exp.sparse = False

    exp.data[exp.data < n] = n
    exp.data = np.log2(exp.data)

    return exp

@Experiment._record_sig
def transform(exp, steps=[], inplace=False, **kwargs):
    '''Chain transformations together.

    Parameters
    ----------
    steps : list of callable
        each callable is a transformer that takes ``Experiment`` object as
        its 1st argument and has a boolean parameter of ``inplace``. Each
        callable should return an ``Experiment`` object.
    inplace : bool
        transformation occuring in the original data or a copy
    kwargs : dict
        keyword arguments to pass to each transformers. The key should
        be in the form of "<transformer_name>__<param_name>". For
        example, "transform(exp, steps=[log_n], log_n__n=3)" will set
        "n" of function "log_n" to 3

    Returns
    -------
    ``Experiment``
        with its data transformed

    '''
    if not inplace:
        exp = deepcopy(exp)
    params = defaultdict(dict)
    for k, v in kwargs.items():
        transformer, param_name = k.split('__')
        if param_name == 'inplace':
            raise ValueError('You should not give %s argument. It should be '
                             'set thru `inplace` argument for this function.')
        params[transformer][param_name] = v
    for step in steps:
        step(exp, inplace=True, **params[step.__name__])
    return exp

@Experiment._record_sig
def normalize_by_subset_features(exp, features, total=10000, negate=True, inplace=False):
    '''Normalize each sample by their total sums without a list of features

    Normalizes all features (including in the exclude list) by the
    total sum calculated without the excluded features. This is to
    alleviate the compositionality in the data set by only keeping the
    features that you think are not changing across samples.

    .. note:: sum is not identical in all samples after normalization
       (since also keeps the excluded features)

    Parameters
    ----------
    features : list of str
        The feature IDs to exclude (or include if negate=False)
    total : int (optional)
        The total abundance for the non-excluded features per sample
    negate : bool (optional)
        True (default) to calculate normalization factor without features in features list.
        False to calculate normalization factor only with features in features list.
    inplace : bool (optional)
        False (default) to create a new experiment, True to normalize in place

    Returns
    -------
    ``Experiment``
        The normalized experiment
    '''
    feature_pos = exp.feature_metadata.index.isin(features)
    if negate:
        feature_pos = np.invert(feature_pos)
    data = exp.get_data(sparse=False)
    use_reads = np.sum(data[:, feature_pos], axis=1)
    if inplace:
        newexp = exp
    else:
        newexp = deepcopy(exp)
    newexp.data = total * data / use_reads[:, None]
    # store the normalization depth into the experiment metadata
    newexp.exp_metadata['normalized'] = total
    return newexp

def normalize_compositional(exp, min_frac=0.05, total=10000, inplace=False):
    '''Normalize each sample by ignoring the features with mean>=min_frac in all the experiment

    This assumes that the majority of features have less than min_frac mean, and that the majority of features don't change
    between samples in a constant direction

    Parameters
    ----------
    min_frac : float (optional)
        ignore features with mean (over all samples) >= min_frac.
    total : int (optional)
        The total abundance for the non-excluded features per sample
    inplace : bool (optional)
        False (default) to create a new experiment, True to normalize in place

    Returns
    -------
    ``Experiment``
        The normalized experiment. Note that all features are normalized (including the ones with mean>=min_frac)
    '''
    comp_features = exp.filter_mean(min_frac)
    logger.info('ignoring %d features' % comp_features.shape[1])
    newexp = exp.normalize_by_subset_features(comp_features.feature_metadata.index.values,
                                              total=total, negate=True, inplace=inplace)
    return newexp

def random_permute_data(exp, normalize=True):
    '''Shuffle independently the reads of each feature

    Creates a new experiment with no dependence between the features.

    Parameters
    ----------
    normalize : bool (optional)
        True (default) to normalize each sample after completing the feature shuffling.
        False to not normalize

    Returns
    -------
    ``Experiment``
        With each feature shuffled independently
    '''
    newexp = exp.copy()
    newexp.sparse = False
    for cfeature in range(newexp.shape[1]):
        np.random.shuffle(newexp.data[:, cfeature])
    if normalize:
        newexp.normalize(np.mean(exp.data.sum(axis=1)), inplace=True)
    return newexp
<commit_msg>add new functions to doc<commit_after>'''
transforming (:mod:`calour.transforming`)
=========================================

.. currentmodule:: calour.transforming

Functions
^^^^^^^^^
.. autosummary::
   :toctree: generated

   normalize
   normalize_by_subset_features
   normalize_compositional
   scale
   random_permute_data
   binarize
   log_n
   transform
'''

# ----------------------------------------------------------------------------
# Copyright (c) 2016--,  Calour development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

from logging import getLogger
from copy import deepcopy
from collections import defaultdict

import numpy as np
from sklearn import preprocessing

from .experiment import Experiment

logger = getLogger(__name__)

@Experiment._record_sig
def normalize(exp, total=10000, axis=0, inplace=False):
    '''Normalize the sum of each sample (axis=0) or feature (axis=1) to sum total

    Parameters
    ----------
    exp : Experiment
    total : float
        the sum (along axis) to normalize to
    axis : 0, 1, 's', or 'f', optional
        the axis to normalize. 0 or 's' (default) is normalize each sample;
        1 or 'f' to normalize each feature
    inplace : bool (optional)
        False (default) to create a copy, True to replace values in exp

    Returns
    -------
    ``Experiment``
        the normalized experiment
    '''
    if isinstance(total, bool):
        raise ValueError('Normalization total (%s) not numeric' % total)
    if total <= 0:
        raise ValueError('Normalization total (%s) must be positive' % total)
    if not inplace:
        exp = deepcopy(exp)
    exp.data = preprocessing.normalize(exp.data, norm='l1', axis=1-axis) * total
    # store the normalization depth into the experiment metadata
    exp.exp_metadata['normalized'] = total
    return exp

@Experiment._record_sig
def rescale(exp, total=10000, axis=0, inplace=False):
    '''Rescale the data to mean sum of all samples (axis=0) or features (axis=1) to be total.

    This function rescales by multiplying ALL entries in exp.data by same number.

    Parameters
    ----------
    exp : Experiment
    total : float
        the mean sum (along axis) to normalize to
    axis : 0, 1, 's', or 'f', optional
        the axis to normalize. 0 or 's' (default) is normalize each sample;
        1 or 'f' to normalize each feature
    inplace : bool (optional)
        False (default) to create a copy, True to replace values in exp

    Returns
    -------
    ``Experiment``
        the normalized experiment
    '''
    if isinstance(total, bool):
        raise ValueError('Rescaling total (%s) not numeric' % total)
    if not inplace:
        exp = deepcopy(exp)
    current_mean = np.mean(exp.data.sum(axis=1-axis))
    exp.data = exp.data * total / current_mean
    return exp

@Experiment._record_sig
def scale(exp, axis=0, inplace=False):
    '''Standardize a dataset along an axis

    .. warning:: It will convert the ``Experiment.data`` from the sparse matrix to dense array.

    Parameters
    ----------
    axis : 0, 1, 's', or 'f'
        0 or 's'  means scaling occur sample-wise; 1 or 'f' feature-wise.

    Returns
    -------
    ``Experiment``
    '''
    logger.debug('scaling the data, axis=%d' % axis)
    if not inplace:
        exp = deepcopy(exp)
    if exp.sparse:
        exp.sparse = False
    preprocessing.scale(exp.data, axis=1-axis, copy=False)
    return exp

@Experiment._record_sig
def binarize(exp, threshold=1, inplace=False):
    '''Binarize the data with a threshold.

    It calls scikit-learn to do the real work.

    Parameters
    ----------
    threshold : Numeric
        the cutoff value. Any values below or equal to this will be replaced by 0,
        above it by 1.

    Returns
    -------
    ``Experiment``
    '''
    logger.debug('binarizing the data. threshold=%f' % threshold)
    if not inplace:
        exp = deepcopy(exp)
    preprocessing.binarize(exp.data, threshold=threshold, copy=False)
    return exp

@Experiment._record_sig
def log_n(exp, n=1, inplace=False):
    '''Log transform the data

    Parameters
    ----------
    n : numeric, optional
        cap the tiny values and then log transform the data.
    inplace : bool, optional

    Returns
    -------
    ``Experiment``
    '''
    logger.debug('log_n transforming the data, min. threshold=%f' % n)
    if not inplace:
        exp = deepcopy(exp)

    if exp.sparse:
        exp.sparse = False

    exp.data[exp.data < n] = n
    exp.data = np.log2(exp.data)

    return exp

@Experiment._record_sig
def transform(exp, steps=[], inplace=False, **kwargs):
    '''Chain transformations together.

    Parameters
    ----------
    steps : list of callable
        each callable is a transformer that takes ``Experiment`` object as
        its 1st argument and has a boolean parameter of ``inplace``. Each
        callable should return an ``Experiment`` object.
    inplace : bool
        transformation occuring in the original data or a copy
    kwargs : dict
        keyword arguments to pass to each transformers. The key should
        be in the form of "<transformer_name>__<param_name>". For
        example, "transform(exp, steps=[log_n], log_n__n=3)" will set
        "n" of function "log_n" to 3

    Returns
    -------
    ``Experiment``
        with its data transformed

    '''
    if not inplace:
        exp = deepcopy(exp)
    params = defaultdict(dict)
    for k, v in kwargs.items():
        transformer, param_name = k.split('__')
        if param_name == 'inplace':
            raise ValueError('You should not give %s argument. It should be '
                             'set thru `inplace` argument for this function.')
        params[transformer][param_name] = v
    for step in steps:
        step(exp, inplace=True, **params[step.__name__])
    return exp

@Experiment._record_sig
def normalize_by_subset_features(exp, features, total=10000, negate=True, inplace=False):
    '''Normalize each sample by their total sums without a list of features

    Normalizes all features (including in the exclude list) by the
    total sum calculated without the excluded features. This is to
    alleviate the compositionality in the data set by only keeping the
    features that you think are not changing across samples.

    .. note:: sum is not identical in all samples after normalization
       (since also keeps the excluded features)

    Parameters
    ----------
    features : list of str
        The feature IDs to exclude (or include if negate=False)
    total : int (optional)
        The total abundance for the non-excluded features per sample
    negate : bool (optional)
        True (default) to calculate normalization factor without features in features list.
        False to calculate normalization factor only with features in features list.
    inplace : bool (optional)
        False (default) to create a new experiment, True to normalize in place

    Returns
    -------
    ``Experiment``
        The normalized experiment
    '''
    feature_pos = exp.feature_metadata.index.isin(features)
    if negate:
        feature_pos = np.invert(feature_pos)
    data = exp.get_data(sparse=False)
    use_reads = np.sum(data[:, feature_pos], axis=1)
    if inplace:
        newexp = exp
    else:
        newexp = deepcopy(exp)
    newexp.data = total * data / use_reads[:, None]
    # store the normalization depth into the experiment metadata
    newexp.exp_metadata['normalized'] = total
    return newexp

def normalize_compositional(exp, min_frac=0.05, total=10000, inplace=False):
    '''Normalize each sample by ignoring the features with mean>=min_frac in all the experiment

    This assumes that the majority of features have less than min_frac mean, and that the majority of features don't change
    between samples in a constant direction

    Parameters
    ----------
    min_frac : float (optional)
        ignore features with mean (over all samples) >= min_frac.
    total : int (optional)
        The total abundance for the non-excluded features per sample
    inplace : bool (optional)
        False (default) to create a new experiment, True to normalize in place

    Returns
    -------
    ``Experiment``
        The normalized experiment. Note that all features are normalized (including the ones with mean>=min_frac)
    '''
    comp_features = exp.filter_mean(min_frac)
    logger.info('ignoring %d features' % comp_features.shape[1])
    newexp = exp.normalize_by_subset_features(comp_features.feature_metadata.index.values,
                                              total=total, negate=True, inplace=inplace)
    return newexp

def random_permute_data(exp, normalize=True):
    '''Shuffle independently the reads of each feature

    Creates a new experiment with no dependence between the features.

    Parameters
    ----------
    normalize : bool (optional)
        True (default) to normalize each sample after completing the feature shuffling.
        False to not normalize

    Returns
    -------
    ``Experiment``
        With each feature shuffled independently
    '''
    newexp = exp.copy()
    newexp.sparse = False
    for cfeature in range(newexp.shape[1]):
        np.random.shuffle(newexp.data[:, cfeature])
    if normalize:
        newexp.normalize(np.mean(exp.data.sum(axis=1)), inplace=True)
    return newexp
<|endoftext|>