<commit_before>import os
import pdb

import matplotlib.pyplot as plt
import seaborn.apionly as seaborn

def _savefig(fig, figname, extra=None, asPNG=True, asPDF=False, load=False):
    """ Helper function to effectively save a figure.

    Parameters
    ----------
    fig : matplotlib.Figure
        The figure to be written to disk.
    figname : string
        The name of the image of the figure without any file extension.
    extra : string or None (default)
        Relative path to subdirectories below './output/img' where the
        figure will be saved.
    as[PNG|PDF] : bool, optional (default = True)
        Toggles if the figure will be saved as a .png and/or .pdf file.
        If both are False, the figure is not saved in any form.
    load : bool, optional (default = False)
        If True, "-load" is appended to ``figname``.

    Writes
    ------
    Images of the matplotlib figure.

    Returns
    -------
    None

    """

    if load:
        figname += '-load'
    figpath = os.path.join('output', 'img')
    if extra is not None:
        figpath = os.path.join(figpath, extra)

    figpathname = os.path.join(figpath, figname)

    figopts = dict(dpi=300, transparent=True, bbox_inches='tight')
    if asPNG:
        fig.savefig(figpathname + '.png', **figopts)
    if asPDF:
        fig.savefig(figpathname + '.pdf', **figopts)
    plt.close(fig)

def _try_to_plot_ts(data, rescol, datecol, **plotkwds):
    """ Helper function to check that a dataframe is not empty and then
    to plot the data as a time series

    Parameters
    ----------
    data :pandas.DataFrame
        The data that you want to plot
    rescol, datecol : str
        Labels of the columns that contrain the result (y) and date (x)
        values of the plot.
    plotkwds : keyword arguments
        Valid matplotlib/pandas kwargs for plotting passed directly to
        ``pandas.DataFrame.plot()``

    Writes
    ------
    None

    Returns
    -------
    N : int
        The number of records in the dataframe.

    """

    label = plotkwds.pop('label', None)
    N = data.shape[0]
    datatoplot = data[[datecol, rescol]].set_index(datecol)
    if label is not None:
        datatoplot = datatoplot.rename(columns={rescol: label})

    if N > 0:
        datatoplot.plot(legend=False, **plotkwds)

    return N

def _make_time_series(wqcomp, axes, addNote, load=False):
    """ Time series helper function

    Parameters
    ----------
    wqcomp : cvc.summary.WQComparison object
        The comparison object to be plotted
    axes : matplotlib.Axes
        The Axes object on which the data should be plotted.
    addNote : bool
        Whether or not the note should be added.
    load : bool
        Whether or not the data represents loads. Otherwise
        concentration data are assumed.
    """

    wqcomp.parameterTimeSeries(ax=axes[0], addNote=addNote, addLegend=False,
                               finalOutput=False, load=load)

def _make_statplots(wqcomp, axes, addNote, load=False):
    """ Statplot helper function

    Parameters
    ----------
    wqcomp : cvc.summary.WQComparison object
        The comparison object to be plotted
    axes : matplotlib.Axes
        The Axes object on which the data should be plotted.
    addNote : bool
        Whether or not the note should be added.
    load : bool
        Whether or not the data represents loads. Otherwise
        concentration data are assumed.
    """

    wqcomp.parameterStatPlot(ax1=axes[0], ax2=axes[1], finalOutput=False,
                             labelax1=False, load=load)

def _make_bmp_boxplot(wqcomp, axes, addNote, load=False):
    """ BMP database comparison boxplot helper function

    Parameters
    ----------
    wqcomp : cvc.summary.WQComparison object
        The comparison object to be plotted
    axes : matplotlib.Axes
        The Axes object on which the data should be plotted.
    addNote : bool
        Whether or not the note should be added.
    load : bool
        Whether or not the data represents loads. Otherwise
        concentration data are assumed.
    """

    wqcomp.bmpCategoryBoxplots(ax=axes[0], addNote=addNote, finalOutput=False)

def _make_landuse_boxplot(wqcomp, axes, addNote, load=False):
    """ NSQD comparison boxplot helper function

    Parameters
    ----------
    wqcomp : cvc.summary.WQComparison object
        The comparison object to be plotted
    axes : matplotlib.Axes
        The Axes object on which the data should be plotted.
    addNote : bool
        Whether or not the note should be added.
    load : bool
        Whether or not the data represents loads. Otherwise
        concentration data are assumed.
    """

    wqcomp.landuseBoxplots(ax=axes[0], addNote=addNote, finalOutput=False)

def _make_seasonal_boxplot(wqcomp, axes, addNote=False, load=False):
    """ Seasonal comparison boxplot helper function

    Parameters
    ----------
    wqcomp : cvc.summary.WQComparison object
        The comparison object to be plotted
    axes : matplotlib.Axes
        The Axes object on which the data should be plotted.
    addNote : bool
        Whether or not the note should be added.
    load : bool
        Whether or not the data represents loads. Otherwise
        concentration data are assumed.
    """

    wqcomp.seasonalBoxplots(ax=axes[0], load=load)

def formatGSAxes(ax, axtype, col, xticks, ylabel, sublabel=None, labelsize=8):
    """ Helper function to format the Gridspec Axes in a MegaFigure

    Parameters
    ----------
    ax : matplotlib.Axes
        The Axes object on which the data should be plotted.
    axtype : string ("inner" or "outer")
        The position of the Axes in its row/col.
    col : int
        Which column the Axes occupies.
    xticks : bool
        Whether the xtick should be kept (True).
    ylabel : string or None
        Label for the y-axis.
    sublabel : string or None
        The sub-figure label for the Axes (e.g., "A" for the upper left
        Axes).
    labelsize : int
        The fontsize in points of the labels.

    """

    # common stuff
    if sublabel is None:
        sublabel = ''

    seaborn.despine(
        ax=ax,
        left=(col == 1 and axtype == 'outer') or axtype == 'inner',
        right=(col == 0 and axtype == 'outer') or axtype == 'inner',
        top=True,
        bottom=not ax.is_last_row()
    )

    # left side of the figure
    if axtype == 'outer' and col == 0:
        # remove ticks from the right
        ax.yaxis.tick_left()
        ax.set_ylabel(ylabel, fontsize=8)
        ax.tick_params(axis='y', right='off', which='both', labelsize=8)
        ax.annotate(sublabel, (0.0, 1.0), xytext=(5, -10),
                    xycoords='axes fraction', textcoords='offset points',
                    fontsize=8, zorder=20)
    # right side of the figure
    elif axtype == 'outer' and col == 1:
        ax.tick_params(axis='y', left='off', which='both', labelsize=8)

        # remove ticks from the left
        ax.yaxis.tick_right()
        ax.yaxis.set_label_position("right")
        ax.set_ylabel(ylabel, rotation=270, fontsize=8)
        #label = ax.yaxis.get_label()
        #label.set_rotation(270)
        ax.annotate(sublabel, (1.0, 1.0), xytext=(-20, -10),
                    xycoords='axes fraction', textcoords='offset points',
                    fontsize=8, zorder=20)
    else:
        ax.set_yticklabels([])
        ax.tick_params(axis='y', right='off', left='off', which='both')

    # clear tick labels if xticks is False or Nones
    if not xticks:
        ax.set_xticklabels([])
        ax.set_xlabel('')
    else:
        ax.tick_params(axis='x', labelsize=7)
<commit_msg>new viz functions<commit_after>import os
import pdb

import numpy as np
import matplotlib.pyplot as plt
import seaborn.apionly as seaborn
from wqio import utils

def _savefig(fig, figname, extra=None, asPNG=True, asPDF=False, load=False):
    """ Helper function to effectively save a figure.

    Parameters
    ----------
    fig : matplotlib.Figure
        The figure to be written to disk.
    figname : string
        The name of the image of the figure without any file extension.
    extra : string or None (default)
        Relative path to subdirectories below './output/img' where the
        figure will be saved.
    as[PNG|PDF] : bool, optional (default = True)
        Toggles if the figure will be saved as a .png and/or .pdf file.
        If both are False, the figure is not saved in any form.
    load : bool, optional (default = False)
        If True, "-load" is appended to ``figname``.

    Writes
    ------
    Images of the matplotlib figure.

    Returns
    -------
    None

    """

    if load:
        figname += '-load'
    figpath = os.path.join('output', 'img')
    if extra is not None:
        figpath = os.path.join(figpath, extra)

    figpathname = os.path.join(figpath, figname)

    figopts = dict(dpi=300, transparent=True, bbox_inches='tight')
    if asPNG:
        fig.savefig(figpathname + '.png', **figopts)
    if asPDF:
        fig.savefig(figpathname + '.pdf', **figopts)
    plt.close(fig)

def _try_to_plot_ts(data, rescol, datecol, **plotkwds):
    """ Helper function to check that a dataframe is not empty and then
    to plot the data as a time series

    Parameters
    ----------
    data :pandas.DataFrame
        The data that you want to plot
    rescol, datecol : str
        Labels of the columns that contrain the result (y) and date (x)
        values of the plot.
    plotkwds : keyword arguments
        Valid matplotlib/pandas kwargs for plotting passed directly to
        ``pandas.DataFrame.plot()``

    Writes
    ------
    None

    Returns
    -------
    N : int
        The number of records in the dataframe.

    """

    label = plotkwds.pop('label', None)
    N = data.shape[0]
    datatoplot = data[[datecol, rescol]].set_index(datecol)
    if label is not None:
        datatoplot = datatoplot.rename(columns={rescol: label})

    if N > 0:
        datatoplot.plot(legend=False, **plotkwds)

    return N

def _make_time_series(wqcomp, axes, addNote, load=False):
    """ Time series helper function

    Parameters
    ----------
    wqcomp : cvc.summary.WQComparison object
        The comparison object to be plotted
    axes : matplotlib.Axes
        The Axes object on which the data should be plotted.
    addNote : bool
        Whether or not the note should be added.
    load : bool
        Whether or not the data represents loads. Otherwise
        concentration data are assumed.
    """

    wqcomp.parameterTimeSeries(ax=axes[0], addNote=addNote, addLegend=False,
                               finalOutput=False, load=load)

def _make_statplots(wqcomp, axes, addNote, load=False):
    """ Statplot helper function

    Parameters
    ----------
    wqcomp : cvc.summary.WQComparison object
        The comparison object to be plotted
    axes : matplotlib.Axes
        The Axes object on which the data should be plotted.
    addNote : bool
        Whether or not the note should be added.
    load : bool
        Whether or not the data represents loads. Otherwise
        concentration data are assumed.
    """

    wqcomp.parameterStatPlot(ax1=axes[0], ax2=axes[1], finalOutput=False,
                             labelax1=False, load=load)

def _make_bmp_boxplot(wqcomp, axes, addNote, load=False):
    """ BMP database comparison boxplot helper function

    Parameters
    ----------
    wqcomp : cvc.summary.WQComparison object
        The comparison object to be plotted
    axes : matplotlib.Axes
        The Axes object on which the data should be plotted.
    addNote : bool
        Whether or not the note should be added.
    load : bool
        Whether or not the data represents loads. Otherwise
        concentration data are assumed.
    """

    wqcomp.bmpCategoryBoxplots(ax=axes[0], addNote=addNote, finalOutput=False)

def _make_landuse_boxplot(wqcomp, axes, addNote, load=False):
    """ NSQD comparison boxplot helper function

    Parameters
    ----------
    wqcomp : cvc.summary.WQComparison object
        The comparison object to be plotted
    axes : matplotlib.Axes
        The Axes object on which the data should be plotted.
    addNote : bool
        Whether or not the note should be added.
    load : bool
        Whether or not the data represents loads. Otherwise
        concentration data are assumed.
    """

    wqcomp.landuseBoxplots(ax=axes[0], addNote=addNote, finalOutput=False)

def _make_seasonal_boxplot(wqcomp, axes, addNote=False, load=False):
    """ Seasonal comparison boxplot helper function

    Parameters
    ----------
    wqcomp : cvc.summary.WQComparison object
        The comparison object to be plotted
    axes : matplotlib.Axes
        The Axes object on which the data should be plotted.
    addNote : bool
        Whether or not the note should be added.
    load : bool
        Whether or not the data represents loads. Otherwise
        concentration data are assumed.
    """

    wqcomp.seasonalBoxplots(ax=axes[0], load=load)

def formatGSAxes(ax, axtype, col, xticks, ylabel, sublabel=None, labelsize=8):
    """ Helper function to format the Gridspec Axes in a MegaFigure

    Parameters
    ----------
    ax : matplotlib.Axes
        The Axes object on which the data should be plotted.
    axtype : string ("inner" or "outer")
        The position of the Axes in its row/col.
    col : int
        Which column the Axes occupies.
    xticks : bool
        Whether the xtick should be kept (True).
    ylabel : string or None
        Label for the y-axis.
    sublabel : string or None
        The sub-figure label for the Axes (e.g., "A" for the upper left
        Axes).
    labelsize : int
        The fontsize in points of the labels.

    """

    # common stuff
    if sublabel is None:
        sublabel = ''

    seaborn.despine(
        ax=ax,
        left=(col == 1 and axtype == 'outer') or axtype == 'inner',
        right=(col == 0 and axtype == 'outer') or axtype == 'inner',
        top=True,
        bottom=not ax.is_last_row()
    )

    # left side of the figure
    if axtype == 'outer' and col == 0:
        # remove ticks from the right
        ax.yaxis.tick_left()
        ax.set_ylabel(ylabel, fontsize=8)
        ax.tick_params(axis='y', right='off', which='both', labelsize=8)
        ax.annotate(sublabel, (0.0, 1.0), xytext=(5, -10),
                    xycoords='axes fraction', textcoords='offset points',
                    fontsize=8, zorder=20)
    # right side of the figure
    elif axtype == 'outer' and col == 1:
        ax.tick_params(axis='y', left='off', which='both', labelsize=8)

        # remove ticks from the left
        ax.yaxis.tick_right()
        ax.yaxis.set_label_position("right")
        ax.set_ylabel(ylabel, rotation=270, fontsize=8)
        #label = ax.yaxis.get_label()
        #label.set_rotation(270)
        ax.annotate(sublabel, (1.0, 1.0), xytext=(-20, -10),
                    xycoords='axes fraction', textcoords='offset points',
                    fontsize=8, zorder=20)
    else:
        ax.set_yticklabels([])
        ax.tick_params(axis='y', right='off', left='off', which='both')

    # clear tick labels if xticks is False or Nones
    if not xticks:
        ax.set_xticklabels([])
        ax.set_xlabel('')
    else:
        ax.tick_params(axis='x', labelsize=7)

def _reduction_range_bars(y, ylow, yhigh, sitecol, data=None, **kwargs):
    """ Draws upper/lower bound bars on `reduction_plots` """
    ax = plt.gca()
    offset_map = {
        'ED-1': 0.0,
        'LV-2': -0.2,
        'LV-4': +0.2
    }
    data = (
        data.assign(offset=data[sitecol].map(offset_map))
            .assign(err_low=data[y] - data[ylow])
            .assign(err_high=data[yhigh] - data[y])
    )

    value_selection = None if data[sitecol].unique().shape[0] > 1 else 0
    data['x'] = (
        data.groupby(sitecol)
            .apply(lambda g: np.arange(g['offset'].size) + g['offset'])
            .values
            .flatten()
    )
    ax.errorbar(x=data['x'], y=data[y], yerr=data[['err_low', 'err_high']].values.T, ls='none', fmt='k')

def reduction_plot(df, params, paramcol, sitecol, xvaluecol, leg_loc, **load_cols):
    """ Draws the percent reduction bar plots with the upper and lower
    estimates.

    Parameters
    ----------
    df : pandas DataFrame
        DataFrame with columns for the parameter, site, x-values, and
        load reduction estimates (lower, primary and upper estimates).
    params : list of str
        The names of the parameters to be included in the plot.
    paramcol, sitecol, xvaluecol : str
        Column labels in `df` that represent the columns containing the
        parameters, site labels, and x-axis value to be shown in the
        plots.
    leg_loc : tuple of floats
        The legend for these figures is built manually to the figure.
        `leg_loc` is a tuple of `(xpos, ypos)` of the lower left corner
        of the legend relative to the lower left corner of the figure.
        In other words `(0.0, 0.0)` will place the legend firmly in the
        lower left corner of the figure, while `(1.0, 1.0)` will place
        the legend just beyon the upper right corner.
    lower, reduction, upper : string, optional
        Column labels for the percent reduction estimates. Default
        values are 'load_reduction_lower', 'load_reduction', and
        'load_reduction_upper', respectively.

    Returns
    -------
    fg : seaborn FacetGrid
        The FaceGrid on which the data have been drawn.

    """
    lower = load_cols.pop('lower', 'load_reduction_lower')
    reduction = load_cols.pop('reduction', 'load_reduction')
    upper = load_cols.pop('upper', 'load_reduction_upper')
    fg = seaborn.factorplot(
        x=xvaluecol, y=reduction, aspect=1.6, size=2, hue=sitecol,
        col=paramcol, col_order=params, col_wrap=2,
        ci=None,  kind='bar', data=df,
        margin_titles=True, legend=False
    )

    fg.add_legend(ncol=2)
    fg._legend.set_bbox_to_anchor(leg_loc)
    fg.map_dataframe(_reduction_range_bars, reduction, lower, upper, sitecol)

    return fg

def hydro_pairplot(hydro, site, sitecol='site', by='season', palette=None, save=True):
    """ Creates a pairplot of hydrologic quantities.

    Parameters
    ----------
    by : string, optional (default = 'season')
        The column in Site.storm_info that defines how the data
        should be grouped.
    palette : seaborn.color_palette or None (default)
        Color scheme for the plot.

    Returns
    -------
    None

    See Also
    --------
    http://web.stanford.edu/~mwaskom/software/seaborn/generated/seaborn.pairplot.html

    """

    cols = [
        'site',
        'antecedent_days',
        'duration_hours',
        'peak_precip_intensity',
        'total_precip_depth',
        'outflow_mm',
        'season',
        'Seasons',
        'year',
        'Has outflow?'
    ]

    var_cols = [
        'antecedent days',
        'duration hours',
        'peak precip intensity',
        'total precip depth',
        'outflow mm',
    ]
    sinfo = (
        hydro[hydro[sitecol] == site]
            .rename(columns={'has_outflow': 'Has outflow?', 'grouped_season': 'Seasons'})
            .select(lambda c: c in cols, axis=1)
            .rename(columns=lambda c: c.replace('_', ' '))
    )

    if by == 'season':
        opts = dict(
            palette=palette or 'BrBG_r',
            hue='season',
            markers=['o', 's', '^', 'd'],
            hue_order=['winter', 'spring', 'summer', 'autumn'],
            vars=var_cols
        )

    elif by =='year':
        opts = dict(
            hue='year',
            palette=palette or 'deep',
            vars=var_cols
        )

    elif by == 'outflow':
        opts = dict(
            hue='Has outflow?',
            palette=palette or 'deep',
            markers=['o', 's'],
            hue_order=['Yes', 'No'],
            vars=var_cols
        )

    elif by == 'grouped_season':
        opts = dict(
            palette=palette or 'BrBG_r',
            hue='Seasons',
            markers=['o', '^'],
            hue_order=['winter/spring', 'summer/autumn'],
            vars=var_cols
        )

    pg = seaborn.pairplot(sinfo, **opts)
    for ax in pg.axes.flat:
        ax.set_xlim(left=0)
        ax.set_ylim(bottom=0)

    if save:
        figname = '{}-HydroPairPlot_by_{}'.format(site, by)
        _savefig(pg.fig, figname, extra='HydroPairPlot')

    return pg

def hydro_histogram(hydro, valuecol='total_precip_depth', bins=None,
                    save=True, **factoropts):
    """ Plot a faceted, categorical histogram of storms.

    valuecol : str, optional
        The name of the column that should be categorized and plotted.
    bins : array-like, optional
        The right-edges of the histogram bins.
    factoropts : keyword arguments, optional
        Options passed directly to seaborn.factorplot

    Returns
    -------
    fig : seaborn.FacetGrid

    See also
    --------
    utils.figutils.categorical_histogram
    seaborn.factorplot

    """

    if bins is None:
        bins = np.arange(5, 30, 5)

    fig = utils.figutils.categorical_histogram(
        hydro, valuecol, bins, **factoropts
    )

    if save:
        figname = 'HydroHistogram_{}'.format(valuecol)
        _savefig(fig, figname, extra='HydroHistogram')

    return fig

def hydro_jointplot(hydro, site, xcol, ycol, sitecol='site',
                    color=None, conditions=None, one2one=True,
                    save=True):
    """ Creates a joint distribution plot of two hydrologic
    quantities.

    Parameters
    ----------
    xcol, ycol : string
        Column names found in Site.storm_info
    conditions : string or None (default)
        Query strings to be passed to Site.storm_info.query(...)
    one2one : bool, optional (default = True)
        Shows the 1:1 line on the scatter portion of the joint
        distribution plot.

    Returns
    -------
    None

    See Also
    --------
    http://web.stanford.edu/~mwaskom/software/seaborn/generated/seaborn.jointplot.html

    """

    column_labels = {
        'total_precip_depth': 'Storm Precipitation Depth (mm)',
        'peak_precip_intensity': 'Peak Precipitation Intensity (mm/hr)',
        'outflow_mm': 'BMP Outflow (watershed mm)',
        'peak_outflow': 'Peak BMP Outflow (L/s)',
        'duration_hours': 'Storm Duration (hr)',
        'antecedent_days': 'Antecedent Dry Period (days)',
    }

    data = hydro[hydro[sitecol] == site]
    if conditions is not None:
        data = data.query(conditions)

    jg = utils.figutils.jointplot(
        x=xcol, y=ycol, data=data, one2one=one2one, color=color,
        xlabel=column_labels[xcol], ylabel=column_labels[ycol],
    )

    if save:
        figname = '{}-HydroJoinPlot_{}_vs_{}'.format(site, xcol, ycol)
        _savefig(jg.fig, figname, extra='HydroJointPlot')
<|endoftext|>


<commit_before>import os
import sys
import argparse

import django
from django.conf import settings

class QuickDjangoTest(object):
    """
    A quick way to run the Django test suite without a fully-configured project.

    Example usage:

        >>> QuickDjangoTest('app1', 'app2')

    Based on a script published by Lukasz Dziedzia at:
    http://stackoverflow.com/questions/3841725/how-to-launch-tests-for-django-reusable-app
    """
    DIRNAME = os.path.dirname(__file__)
    INSTALLED_APPS = (
        'django.contrib.admin',
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.humanize',
        'django.contrib.messages',
        'django.contrib.sessions',
        'django.contrib.sites',
        'django.contrib.staticfiles',
        'bootstrapform',
    )
    MIDDLEWARE = [
        'django.middleware.security.SecurityMiddleware',
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.common.CommonMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.middleware.clickjacking.XFrameOptionsMiddleware',
    ]

    TEMPLATES = [
        {
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'APP_DIRS': True,
            'OPTIONS': {
                'context_processors': (
                    # Defaults:
                    "django.contrib.auth.context_processors.auth",
                    "django.template.context_processors.debug",
                    "django.template.context_processors.i18n",
                    "django.template.context_processors.media",
                    "django.template.context_processors.static",
                    "django.template.context_processors.tz",
                    "django.contrib.messages.context_processors.messages",
                    # Our extra:
                    "django.template.context_processors.request",
                ),
            },
        },
    ]

    def __init__(self, *args, **kwargs):
        self.apps = args
        self._tests()

    def _tests(self):

        settings.configure(
            DEBUG=True,
            DATABASES={
                'default': {
                    'ENGINE': 'django.db.backends.sqlite3',
                    'NAME': os.path.join(self.DIRNAME, 'database.db'),
                    'USER': '',
                    'PASSWORD': '',
                    'HOST': '',
                    'PORT': '',
                }
            },
            INSTALLED_APPS=self.INSTALLED_APPS + self.apps,
            MIDDLEWARE=self.MIDDLEWARE,
            ROOT_URLCONF='helpdesk.tests.urls',
            STATIC_URL='/static/',
            TEMPLATES=self.TEMPLATES
        )

        from django.test.runner import DiscoverRunner
        test_runner = DiscoverRunner(verbosity=1)
        django.setup()

        failures = test_runner.run_tests(self.apps)
        if failures:
            sys.exit(failures)

if __name__ == '__main__':
    """
    What do when the user hits this file from the shell.

    Example usage:

        $ python quicktest.py app1 app2

    """
    parser = argparse.ArgumentParser(
        usage="[args]",
        description="Run Django tests on the provided applications."
    )
    parser.add_argument('apps', nargs='+', type=str)
    args = parser.parse_args()
    QuickDjangoTest(*args.apps)
<commit_msg>fix for unpack requires a string argument of length 44 on py27<commit_after>import os
import sys
import argparse

import django
from django.conf import settings

class QuickDjangoTest(object):
    """
    A quick way to run the Django test suite without a fully-configured project.

    Example usage:

        >>> QuickDjangoTest('app1', 'app2')

    Based on a script published by Lukasz Dziedzia at:
    http://stackoverflow.com/questions/3841725/how-to-launch-tests-for-django-reusable-app
    """
    DIRNAME = os.path.dirname(__file__)
    INSTALLED_APPS = (
        'django.contrib.admin',
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.humanize',
        'django.contrib.messages',
        'django.contrib.sessions',
        'django.contrib.sites',
        'django.contrib.staticfiles',
        'bootstrapform',
    )
    MIDDLEWARE = [
        'django.middleware.security.SecurityMiddleware',
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.common.CommonMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.middleware.clickjacking.XFrameOptionsMiddleware',
    ]

    TEMPLATES = [
        {
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'APP_DIRS': True,
            'OPTIONS': {
                'context_processors': (
                    # Defaults:
                    "django.contrib.auth.context_processors.auth",
                    "django.template.context_processors.debug",
                    "django.template.context_processors.i18n",
                    "django.template.context_processors.media",
                    "django.template.context_processors.static",
                    "django.template.context_processors.tz",
                    "django.contrib.messages.context_processors.messages",
                    # Our extra:
                    "django.template.context_processors.request",
                ),
            },
        },
    ]

    def __init__(self, *args, **kwargs):
        self.apps = args
        self._tests()

    def _tests(self):

        settings.configure(
            DEBUG=True,
            TIME_ZONE='UTC',
            DATABASES={
                'default': {
                    'ENGINE': 'django.db.backends.sqlite3',
                    'NAME': os.path.join(self.DIRNAME, 'database.db'),
                    'USER': '',
                    'PASSWORD': '',
                    'HOST': '',
                    'PORT': '',
                }
            },
            INSTALLED_APPS=self.INSTALLED_APPS + self.apps,
            MIDDLEWARE=self.MIDDLEWARE,
            ROOT_URLCONF='helpdesk.tests.urls',
            STATIC_URL='/static/',
            TEMPLATES=self.TEMPLATES
        )

        from django.test.runner import DiscoverRunner
        test_runner = DiscoverRunner(verbosity=1)
        django.setup()

        failures = test_runner.run_tests(self.apps)
        if failures:
            sys.exit(failures)

if __name__ == '__main__':
    """
    What do when the user hits this file from the shell.

    Example usage:

        $ python quicktest.py app1 app2

    """
    parser = argparse.ArgumentParser(
        usage="[args]",
        description="Run Django tests on the provided applications."
    )
    parser.add_argument('apps', nargs='+', type=str)
    args = parser.parse_args()
    QuickDjangoTest(*args.apps)
<|endoftext|>


<commit_before>#!/usr/bin/python

# Copyright (c) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import openstack.compute
import datetime
import cherrypy
from mako.template import Template
from mako.lookup import TemplateLookup
from uuid import uuid4
from ConfigParser import SafeConfigParser
import os

class Root(object):
    """Define methods necessary to make our web page work with cherrypy"""
    crond = "* * * * * root /bin/bash /root/install.sh\n"

    def __init__(self, cfg="cloud.cfg"):
        from openstack.compute import Compute
        cp = SafeConfigParser()
        cp.read([cfg])
        self.first = ""
        self.user = cp.get("rackspacecloud", "user")
        self.apikey = cp.get("rackspacecloud", "apikey")
        self.prefix = cp.get("radioedit", "prefix")
        self.pubkey = cp.get("radioedit", "pubkey")
        self.compute = Compute(username=self.user, apikey=self.apikey)
        self.root_install = """#!/bin/bash

export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
exec >/tmp/install.log
exec 2>&1

echo STARTING
mkdir /root/.ssh
chmod 700 /root/.ssh
echo %s > /root/.ssh/authorized_keys
chmod 600 /root/.ssh/authorized_keys
rm -f /etc/cron.d/firstboot
touch /tmp/foo
apt-get install -y curl
cd /opt/
curl -skS https://github.com/cloudbuilders/deploy.sh/raw/master/auto.sh | /bin/bash
echo FINISHED
""" % (self.pubkey)

    @cherrypy.expose
    def index(self):
        return '<form method="get" action="/new"><input type="text" name="name" /></form>
<form method="post" action="/reset"><input type="submit" value="kill all" /></form>

' + reduce(
               lambda x,y: x+y,
               map(lambda x: '
%s	%s	%s
' %
               (x['ip'], x['name'].split('-')[1], x['ip'], x.metadata['created']), self.list()), "") + "
"

    @cherrypy.expose
    def new(self, name=None):
        img = [i for i in self.compute.images.list()
                if i.name.find("Ubuntu 10.10") != -1][0]
        flav = [f for f in self.compute.flavors.list() if f.ram == 512][0]
        if name is None:
            name = str(uuid4()).replace('-', '')
        full_name = self.prefix + '-' + name + '-' + str(datetime.datetime.today()).replace('-', ' ')
        srv = self.compute.servers.create(full_name.replace(' ','_'), img.id, flav.id,
            files={"/etc/cron.d/firstboot": self.crond,
                   "/root/install.sh": self.root_install},
            meta={"created", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
        if self.first == "":
            self.first = srv.public_ip
        raise cherrypy.HTTPRedirect("/")

    @cherrypy.expose
    def reset(self):
        self.first = ""
        servers = self.compute.servers.list()
        for server in servers:
            if server.name.find(self.prefix) == 0:
                self.compute.servers.delete(server)
        raise cherrypy.HTTPRedirect("/")

    def list(self):
        return [{"ip": s.public_ip, "name": s.name}
                  for s in self.compute.servers.list()
                  if s.name.find(self.prefix) == 0]

application = cherrypy.Application(Root(cfg="/etc/cloud.cfg"), script_name=None, config=None)
<commit_msg>fix issue by replace , with : in dict creation<commit_after>#!/usr/bin/python

# Copyright (c) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import openstack.compute
import datetime
import cherrypy
from mako.template import Template
from mako.lookup import TemplateLookup
from uuid import uuid4
from ConfigParser import SafeConfigParser
import os

class Root(object):
    """Define methods necessary to make our web page work with cherrypy"""
    crond = "* * * * * root /bin/bash /root/install.sh\n"

    def __init__(self, cfg="cloud.cfg"):
        from openstack.compute import Compute
        cp = SafeConfigParser()
        cp.read([cfg])
        self.first = ""
        self.user = cp.get("rackspacecloud", "user")
        self.apikey = cp.get("rackspacecloud", "apikey")
        self.prefix = cp.get("radioedit", "prefix")
        self.pubkey = cp.get("radioedit", "pubkey")
        self.compute = Compute(username=self.user, apikey=self.apikey)
        self.root_install = """#!/bin/bash

export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
exec >/tmp/install.log
exec 2>&1

echo STARTING
mkdir /root/.ssh
chmod 700 /root/.ssh
echo %s > /root/.ssh/authorized_keys
chmod 600 /root/.ssh/authorized_keys
rm -f /etc/cron.d/firstboot
touch /tmp/foo
apt-get install -y curl
cd /opt/
curl -skS https://github.com/cloudbuilders/deploy.sh/raw/master/auto.sh | /bin/bash
echo FINISHED
""" % (self.pubkey)

    @cherrypy.expose
    def index(self):
        return '<form method="get" action="/new"><input type="text" name="name" /></form>
<form method="post" action="/reset"><input type="submit" value="kill all" /></form>

' + reduce(
               lambda x,y: x+y,
               map(lambda x: '
%s	%s	%s
' %
               (x['ip'], x['name'].split('-')[1], x['ip'], x.metadata['created']), self.list()), "") + "
"

    @cherrypy.expose
    def new(self, name=None):
        img = [i for i in self.compute.images.list()
                if i.name.find("Ubuntu 10.10") != -1][0]
        flav = [f for f in self.compute.flavors.list() if f.ram == 512][0]
        if name is None:
            name = str(uuid4()).replace('-', '')
        full_name = self.prefix + '-' + name + '-' + str(datetime.datetime.today()).replace('-', ' ')
        srv = self.compute.servers.create(full_name.replace(' ','_'), img.id, flav.id,
            files={"/etc/cron.d/firstboot": self.crond,
                   "/root/install.sh": self.root_install},
            meta={"created": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")})
        if self.first == "":
            self.first = srv.public_ip
        raise cherrypy.HTTPRedirect("/")

    @cherrypy.expose
    def reset(self):
        self.first = ""
        servers = self.compute.servers.list()
        for server in servers:
            if server.name.find(self.prefix) == 0:
                self.compute.servers.delete(server)
        raise cherrypy.HTTPRedirect("/")

    def list(self):
        return [{"ip": s.public_ip, "name": s.name}
                  for s in self.compute.servers.list()
                  if s.name.find(self.prefix) == 0]

application = cherrypy.Application(Root(cfg="/etc/cloud.cfg"), script_name=None, config=None)
<|endoftext|>


<commit_before>import logging
import socket
import struct
import argparse

logging.basicConfig(level=logging.WARN)
logger = logging.getLogger(__name__)

DEFAULT_CLIENT_ID = 42

SERVERDATA_AUTH = 3
SERVERDATA_EXECCOMMAND = 2
SERVERDATA_AUTH_RESPONSE = 2
SERVERDATA_RESPONSE_VALUE = 0

PACKET_FMT = '<iii{0}ss'

class Packet:
    def __init__(self, id, type, body):
        self.id = id
        self.type = type
        self.body = body
    def __str__(self):
        return '<ID: %s, Type: %s, Body: %s>' % (self.id, self.type, self.body)

class Rcon:
    def __init__(self, server, port, password, client_id=DEFAULT_CLIENT_ID):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.connect((server, port))
        self.client_id = client_id
        authenticate(self.sock, password, client_id)

    def send(self, command):
        packet = send(self.sock, self.client_id, command)
        return packet.body

    def close(self):
        self.sock.close()

def send(sock, client_id, command):
    sock.send(make_buf(SERVERDATA_EXECCOMMAND, client_id, command))
    packet = receive_packet(sock)
    assert packet.type == SERVERDATA_RESPONSE_VALUE
    assert packet.id == client_id
    return packet

def authenticate(sock, password, client_id):
    # Start auth
    sock.send(make_buf(SERVERDATA_AUTH, client_id, password))
    # Receive empty SERVERDATA_RESPONSE_VALUE
    empty_pack = receive_packet(sock)

    if empty_pack.id == -1:
        raise Exception('Authentication error')
    assert empty_pack.type == SERVERDATA_RESPONSE_VALUE
    assert empty_pack.body == ''

    # Receive SERVERDATA_AUTH
    auth_pack = receive_packet(sock)
    assert auth_pack.type == SERVERDATA_AUTH_RESPONSE
    assert auth_pack.id == client_id

def receive_packet(s):
    length = unpack_length(s.recv(4))
    logger.debug('receiving packet of length %s', length)
    packet = unpack(s.recv(length), length - 9)
    logger.debug('receieved %s', packet)
    return packet

def make_buf(type, client_id, body):
    length = len(body)
    return struct.pack(PACKET_FMT.format(length + 1), 10 + length, client_id, type, body.encode('ascii'), b'')

def unpack_length(buf):
    return struct.unpack('<i', buf)[0]

def unpack(buf, body_length):
    logger.debug(buf)
    id, type, body, nil = struct.unpack('<ii{0}ss'.format(body_length), buf)
    return Packet(id, type, body[0:-1].decode('ascii'))

def parse_args(args):
    parser = argparse.ArgumentParser(description='Send a command to a Starbound RCON server')
    parser.add_argument('--server', '-s', metavar='server', help='RCON server url')
    parser.add_argument('--port', '-P', metavar='port', type=int, default=21026, help='RCON server port')
    parser.add_argument('--password', '-p', metavar='password', help='RCON server password')
    parser.add_argument('command', help='command to send to the server')
    return vars(parser.parse_args(args))

def main(args):
    opts = parse_args(args)
    rcon = Rcon(opts['server'], opts['port'], opts['password'])
    response = rcon.send(opts['command'])
    rcon.close()
    return response
<commit_msg>Removing buffer debug logging<commit_after>import logging
import socket
import struct
import argparse

logging.basicConfig(level=logging.WARN)
logger = logging.getLogger(__name__)

DEFAULT_CLIENT_ID = 42

SERVERDATA_AUTH = 3
SERVERDATA_EXECCOMMAND = 2
SERVERDATA_AUTH_RESPONSE = 2
SERVERDATA_RESPONSE_VALUE = 0

PACKET_FMT = '<iii{0}ss'

class Packet:
    def __init__(self, id, type, body):
        self.id = id
        self.type = type
        self.body = body
    def __str__(self):
        return '<ID: %s, Type: %s, Body: %s>' % (self.id, self.type, self.body)

class Rcon:
    def __init__(self, server, port, password, client_id=DEFAULT_CLIENT_ID):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.connect((server, port))
        self.client_id = client_id
        authenticate(self.sock, password, client_id)

    def send(self, command):
        packet = send(self.sock, self.client_id, command)
        return packet.body

    def close(self):
        self.sock.close()

def send(sock, client_id, command):
    sock.send(make_buf(SERVERDATA_EXECCOMMAND, client_id, command))
    packet = receive_packet(sock)
    assert packet.type == SERVERDATA_RESPONSE_VALUE
    assert packet.id == client_id
    return packet

def authenticate(sock, password, client_id):
    # Start auth
    sock.send(make_buf(SERVERDATA_AUTH, client_id, password))
    # Receive empty SERVERDATA_RESPONSE_VALUE
    empty_pack = receive_packet(sock)

    if empty_pack.id == -1:
        raise Exception('Authentication error')
    assert empty_pack.type == SERVERDATA_RESPONSE_VALUE
    assert empty_pack.body == ''

    # Receive SERVERDATA_AUTH
    auth_pack = receive_packet(sock)
    assert auth_pack.type == SERVERDATA_AUTH_RESPONSE
    assert auth_pack.id == client_id

def receive_packet(s):
    length = unpack_length(s.recv(4))
    logger.debug('receiving packet of length %s', length)
    packet = unpack(s.recv(length), length - 9)
    logger.debug('receieved %s', packet)
    return packet

def make_buf(type, client_id, body):
    length = len(body)
    return struct.pack(PACKET_FMT.format(length + 1), 10 + length, client_id, type, body.encode('ascii'), b'')

def unpack_length(buf):
    return struct.unpack('<i', buf)[0]

def unpack(buf, body_length):
    id, type, body, nil = struct.unpack('<ii{0}ss'.format(body_length), buf)
    return Packet(id, type, body[0:-1].decode('ascii'))

def parse_args(args):
    parser = argparse.ArgumentParser(description='Send a command to a Starbound RCON server')
    parser.add_argument('--server', '-s', metavar='server', help='RCON server url')
    parser.add_argument('--port', '-P', metavar='port', type=int, default=21026, help='RCON server port')
    parser.add_argument('--password', '-p', metavar='password', help='RCON server password')
    parser.add_argument('command', help='command to send to the server')
    return vars(parser.parse_args(args))

def main(args):
    opts = parse_args(args)
    rcon = Rcon(opts['server'], opts['port'], opts['password'])
    response = rcon.send(opts['command'])
    rcon.close()
    return response
<|endoftext|>


<commit_before>#!/usr/bin/env python
# tuned-resonator
# experiments with google physical-web mdns broadcast

'''
from google's repo:
The Physical Web is about getting URLs into the physical world.
However, this isn't limited to just Bluetooth Low Energy (BLE) beacons.
mDNS is a service broadcast technique used in Wifi networks.
It has a two advantages over BLE:

Only people logged into your wifi can see the mDNS URLs. This means that in an appartment, your neighbors can't see your devices.
It doesn't have the length restrictions of BLE has so a URL can be along as you'd like (well, at least up to 100 characters).
Below is an example of how to setup a Raspberry Pi to broadcast a Physical Web URL using mDNS. We hope others are willing to contribute and offer more versions.
If so, we'll create an mDNS directory for all the alternatives.

You'll first need a mDNS service on your RPi. Avahi is the one we use here: $ sudo apt-get install avahi-daemon

You'll then place a '.service' file into the /etc/avahi/services directory. Our sample file 'physical-web-url.service' looks like this:

<?xml version="1.0" standalone='no'?><!--*-nxml-*-->
<!DOCTYPE service-group SYSTEM "avahi-service.dtd">

<service-group>
  <name replace-wildcards="yes">http://www.mycompany.com/xyz.html</name>
  <service>
    <host-name>www.mycompany.com</host-name>
    <type>_http._tcp</type>
    <port>80</port>
    <txt-record>path=/xyz.html</txt-record>
  </service>
</service-group>
Then <name> tag must be unique on your network.
We are suggesting that you use the URL as your name to be safe.
That should be it.
If you have the latest client on your phone, the web page http://www.mycompany.com/xyz.html will now show up in your list of nearby devices.
Note: the iOS app supports mDNS, but not the Android app yet.
It will be updated within a few days for Android 5.0 devices.
If people are stuck on older versions, please let us know (or feel free add it yourself ;-)

'''

from lxml import etree
from lxml import objectify

# create XML
# root = etree.XMLID('<service-group><name replace-wildcards="yes">https://data.sparkfun.com/streams/RMMAa2YvMyHxRr123jqG/</name><service><host-name>data.sparkfun.com</host-name><type>_http._tcp</type><port>80</port><txt-record>path=/streams/RMMAa2YvMyHxRr123jqG/</txt-record></service></service-group>')
# another child with text
# for i in enumerate(root[0]):
# 	print i
# child = etree.Element('child')
# child.text = 'some text'
# # root.append(child)
# print root.get('name')
# # pretty string
# s = etree.tostring(root, pretty_print=True)
# print s

root = objectify.Element('service-group')
t_name = objectify.SubElement(root, 'name')
t_name.attrib['replace-wildcards'] = 'yes'
t_name._setText('https://data.sparkfun.com/streams/RMMAa2YvMyHxRr123jqG/')
t_service = objectify.SubElement(root, 'service')
tt_hostname = objectify.SubElement(t_service, 'host-name')
tt_hostname._setText('data.sparkfun.com')
tt_type = objectify.SubElement(t_service, 'type')
tt_type._setText('_http._tcp')
tt_port = objectify.SubElement(t_service, 'port')
tt_port._setText('80')
tt_txtrecord = objectify.SubElement(t_service, 'txt-record')
tt_txtrecord._setText('path=/streams/RMMAa2YvMyHxRr123jqG/')
objectify.deannotate(root, cleanup_namespaces=True)
print etree.tostring(root, pretty_print=True)

<commit_msg>got lxml rendering ok-looking service-group files<commit_after>#!/usr/bin/env python
# tuned-resonator
# experiments with google physical-web mdns broadcast

'''
from google's repo:
The Physical Web is about getting URLs into the physical world.
However, this isn't limited to just Bluetooth Low Energy (BLE) beacons.
mDNS is a service broadcast technique used in Wifi networks.
It has a two advantages over BLE:

Only people logged into your wifi can see the mDNS URLs. This means that in an appartment, your neighbors can't see your devices.
It doesn't have the length restrictions of BLE has so a URL can be along as you'd like (well, at least up to 100 characters).
Below is an example of how to setup a Raspberry Pi to broadcast a Physical Web URL using mDNS. We hope others are willing to contribute and offer more versions.
If so, we'll create an mDNS directory for all the alternatives.

You'll first need a mDNS service on your RPi. Avahi is the one we use here: $ sudo apt-get install avahi-daemon

You'll then place a '.service' file into the /etc/avahi/services directory. Our sample file 'physical-web-url.service' looks like this:

<?xml version="1.0" standalone='no'?><!--*-nxml-*-->
<!DOCTYPE service-group SYSTEM "avahi-service.dtd">

<service-group>
  <name replace-wildcards="yes">http://www.mycompany.com/xyz.html</name>
  <service>
    <host-name>www.mycompany.com</host-name>
    <type>_http._tcp</type>
    <port>80</port>
    <txt-record>path=/xyz.html</txt-record>
  </service>
</service-group>
Then <name> tag must be unique on your network.
We are suggesting that you use the URL as your name to be safe.
That should be it.
If you have the latest client on your phone, the web page http://www.mycompany.com/xyz.html will now show up in your list of nearby devices.
Note: the iOS app supports mDNS, but not the Android app yet.
It will be updated within a few days for Android 5.0 devices.
If people are stuck on older versions, please let us know (or feel free add it yourself ;-)

'''

from lxml import etree
from lxml import objectify
from Adafruit_BBIO import ADC as adc

# create XML
# root = etree.XMLID('<service-group><name replace-wildcards="yes">https://data.sparkfun.com/streams/RMMAa2YvMyHxRr123jqG/</name><service><host-name>data.sparkfun.com</host-name><type>_http._tcp</type><port>80</port><txt-record>path=/streams/RMMAa2YvMyHxRr123jqG/</txt-record></service></service-group>')
# another child with text
# for i in enumerate(root[0]):
# 	print i
# child = etree.Element('child')
# child.text = 'some text'
# # root.append(child)
# print root.get('name')
# # pretty string
# s = etree.tostring(root, pretty_print=True)
# print s

root = objectify.Element('service-group')
t_name = objectify.SubElement(root, 'name')
t_name.attrib['replace-wildcards'] = 'yes'
t_name._setText('https://data.sparkfun.com/streams/RMMAa2YvMyHxRr123jqG/')
t_service = objectify.SubElement(root, 'service')
tt_hostname = objectify.SubElement(t_service, 'host-name')
tt_hostname._setText('data.sparkfun.com')
tt_type = objectify.SubElement(t_service, 'type')
tt_type._setText('_http._tcp')
tt_port = objectify.SubElement(t_service, 'port')
tt_port._setText('80')
tt_txtrecord = objectify.SubElement(t_service, 'txt-record')
tt_txtrecord._setText('path=/streams/RMMAa2YvMyHxRr123jqG/')
objectify.deannotate(root, cleanup_namespaces=True)
print etree.tostring(root, pretty_print=True)

<|endoftext|>


<commit_before>import json
import logging
import sys
from datetime import datetime
from operator import itemgetter

import requests

from crate.client.exceptions import (
    ConnectionError, DigestNotFoundException, ProgrammingError)

logger = logging.getLogger(__name__)

if sys.version_info[0] > 2:
    basestring = str

class Client(object):
    """
    Crate connection client using crate's HTTP API.
    """

    sql_path = '_sql'
    """Crate URI path for issuing SQL statements."""

    retry_interval = 30
    """Retry interval for failed servers in seconds."""

    default_server = "127.0.0.1:9200"
    """Default server to use if no servers are given on instantiation."""

    def __init__(self, servers=None, timeout=None):
        if not servers:
            servers = self.default_server
        if isinstance(servers, basestring):
            servers = [servers]
        self._active_servers = servers
        self._http_timeout = timeout
        self._inactive_servers = []

    def sql(self, stmt, parameters=None):
        """
        Execute SQL stmt against the crate server.
        """
        if stmt is None:
            return None

        if not isinstance(stmt, basestring):
            raise ValueError("stmt is not a string type")

        data = {
            'stmt': stmt
        }
        if parameters:
            data['args'] = list(parameters)
        logger.debug(
            'Sending request to %s with payload: %s', self.sql_path, data)
        content = self._json_request('POST', self.sql_path, data=data)
        logger.debug("JSON response for stmt(%s): %s", stmt, content)

        return content

    def _blob_path(self, table, digest=None):
        path = table + '/_blobs/'
        if digest:
            path += digest
        return path

    def blob_put(self, table, digest, data):
        """
        Stores the contents of the file like @data object in a blob under the
        given table and digest.
        """
        response = self._request('PUT', self._blob_path(table, digest), data=data)
        if response.status_code == 201:
            return True
        elif response.status_code == 409:
            return False
        self._raise_for_status(response)

    def blob_del(self, table, digest):
        """
        Deletes the blob with given digest under the given table.
        """
        response = self._request('DELETE', self._blob_path(table, digest))
        if response.status_code == 204:
            return True
        elif response.status_code == 404:
            return False
        self._raise_for_status(response)

    def blob_get(self, table, digest, chunk_size=1024 * 128):
        """
        Returns a file like object representing the contents of the blob with the given
        digest.
        """
        response = self._request('GET', self._blob_path(table, digest), stream=True)

        if response.status_code == 404:
            raise DigestNotFoundException(table, digest)
        self._raise_for_status(response)
        return response.iter_content(chunk_size=chunk_size)

    def blob_exists(self, table, digest):
        """
        Returns true if the blob with the given digest exists under the given table.
        """
        response = self._request('HEAD', self._blob_path(table, digest))
        if response.status_code == 200:
            return True
        elif response.status_code == 404:
            return False
        self._raise_for_status(response)

    def _request(self, method, path, **kwargs):
        while True:
            server = self._get_server()
            try:
                # build uri and send http request
                uri = "http://{server}/{path}".format(server=server, path=path)
                return requests.request(method, uri, timeout=self._http_timeout, **kwargs)
            except (requests.ConnectionError, requests.Timeout,
                    requests.TooManyRedirects) as ex:
                # drop server from active ones
                ex_message = hasattr(ex, 'message') and ex.message or str(ex)
                self._drop_server(server, ex_message)
                # if this is the last server raise exception, otherwise try next
                if not self._active_servers:
                    raise ConnectionError(
                        ("No more Servers available, "
                         "exception from last server: %s") % ex_message)
            except requests.HTTPError as e:
                if hasattr(e, 'response') and e.response:
                    raise ProgrammingError(e.response.content)
                raise ProgrammingError()

    def _raise_for_status(self, response):
        """ make sure that only crate.exceptions are raised that are defined in
        the DB-API specification """

        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError as e:
            if hasattr(e, 'response') and e.response:
                raise ProgrammingError(e.response.content)
            raise ProgrammingError()

    def _json_request(self, method, path, data=None):
        """
        Issue request against the crate HTTP API.
        """

        if data:
            data = json.dumps(data)
        response = self._request(method, path, data=data)

        # raise error if occurred, otherwise nothing is raised
        self._raise_for_status(response)
        # return parsed json response
        if len(response.content) > 0:
            return response.json()
        return response.content

    def _get_server(self):
        """
        Get server to use for request.
        Also process inactive server list, re-add them after given interval.
        """
        if self._inactive_servers:
            sorted_inactive_servers = sorted(self._inactive_servers, key=itemgetter('timestamp'))

            # first, check for failed servers older than ``interval``
            for server_data in sorted_inactive_servers:
                delta = datetime.now() - server_data.get('timestamp')
                if delta.seconds >= self.retry_interval:
                    self._active_servers.append(server_data.get('server'))
                    logger.info("Restored server %s into active pool", server_data.get('server'))

            # if no one is old enough, use oldest
            if not self._active_servers:
                server_to_add = sorted_inactive_servers.pop(0).get('server')
                self._active_servers.append(server_to_add)
                logger.info("Restored server %s into active pool", server_to_add)

        server = self._active_servers[0]
        self._roundrobin()

        return server

    def _drop_server(self, server, message):
        """
        Drop server from active list and adds it to the inactive ones.
        """
        try:
            self._active_servers.remove(server)
        except ValueError:
            pass
        else:
            self._inactive_servers.append({
                'server': server,
                'message': message,
                'timestamp': datetime.now()
            })
            logger.warning("Removed server %s from active pool", server)

    def _roundrobin(self):
        """
        Very simple round-robin implementation
        """
        self._active_servers.append(self._active_servers.pop(0))

    def __repr__(self):
        return '<Client {}>'.format(str(self._active_servers))
<commit_msg>remove unnecessary list copy<commit_after>import json
import logging
import sys
from datetime import datetime
from operator import itemgetter

import requests

from crate.client.exceptions import (
    ConnectionError, DigestNotFoundException, ProgrammingError)

logger = logging.getLogger(__name__)

if sys.version_info[0] > 2:
    basestring = str

class Client(object):
    """
    Crate connection client using crate's HTTP API.
    """

    sql_path = '_sql'
    """Crate URI path for issuing SQL statements."""

    retry_interval = 30
    """Retry interval for failed servers in seconds."""

    default_server = "127.0.0.1:9200"
    """Default server to use if no servers are given on instantiation."""

    def __init__(self, servers=None, timeout=None):
        if not servers:
            servers = self.default_server
        if isinstance(servers, basestring):
            servers = [servers]
        self._active_servers = servers
        self._http_timeout = timeout
        self._inactive_servers = []

    def sql(self, stmt, parameters=None):
        """
        Execute SQL stmt against the crate server.
        """
        if stmt is None:
            return None

        if not isinstance(stmt, basestring):
            raise ValueError("stmt is not a string type")

        data = {
            'stmt': stmt
        }
        if parameters:
            data['args'] = parameters
        logger.debug(
            'Sending request to %s with payload: %s', self.sql_path, data)
        content = self._json_request('POST', self.sql_path, data=data)
        logger.debug("JSON response for stmt(%s): %s", stmt, content)

        return content

    def _blob_path(self, table, digest=None):
        path = table + '/_blobs/'
        if digest:
            path += digest
        return path

    def blob_put(self, table, digest, data):
        """
        Stores the contents of the file like @data object in a blob under the
        given table and digest.
        """
        response = self._request('PUT', self._blob_path(table, digest), data=data)
        if response.status_code == 201:
            return True
        elif response.status_code == 409:
            return False
        self._raise_for_status(response)

    def blob_del(self, table, digest):
        """
        Deletes the blob with given digest under the given table.
        """
        response = self._request('DELETE', self._blob_path(table, digest))
        if response.status_code == 204:
            return True
        elif response.status_code == 404:
            return False
        self._raise_for_status(response)

    def blob_get(self, table, digest, chunk_size=1024 * 128):
        """
        Returns a file like object representing the contents of the blob with the given
        digest.
        """
        response = self._request('GET', self._blob_path(table, digest), stream=True)

        if response.status_code == 404:
            raise DigestNotFoundException(table, digest)
        self._raise_for_status(response)
        return response.iter_content(chunk_size=chunk_size)

    def blob_exists(self, table, digest):
        """
        Returns true if the blob with the given digest exists under the given table.
        """
        response = self._request('HEAD', self._blob_path(table, digest))
        if response.status_code == 200:
            return True
        elif response.status_code == 404:
            return False
        self._raise_for_status(response)

    def _request(self, method, path, **kwargs):
        while True:
            server = self._get_server()
            try:
                # build uri and send http request
                uri = "http://{server}/{path}".format(server=server, path=path)
                return requests.request(method, uri, timeout=self._http_timeout, **kwargs)
            except (requests.ConnectionError, requests.Timeout,
                    requests.TooManyRedirects) as ex:
                # drop server from active ones
                ex_message = hasattr(ex, 'message') and ex.message or str(ex)
                self._drop_server(server, ex_message)
                # if this is the last server raise exception, otherwise try next
                if not self._active_servers:
                    raise ConnectionError(
                        ("No more Servers available, "
                         "exception from last server: %s") % ex_message)
            except requests.HTTPError as e:
                if hasattr(e, 'response') and e.response:
                    raise ProgrammingError(e.response.content)
                raise ProgrammingError()

    def _raise_for_status(self, response):
        """ make sure that only crate.exceptions are raised that are defined in
        the DB-API specification """

        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError as e:
            if hasattr(e, 'response') and e.response:
                raise ProgrammingError(e.response.content)
            raise ProgrammingError()

    def _json_request(self, method, path, data=None):
        """
        Issue request against the crate HTTP API.
        """

        if data:
            data = json.dumps(data)
        response = self._request(method, path, data=data)

        # raise error if occurred, otherwise nothing is raised
        self._raise_for_status(response)
        # return parsed json response
        if len(response.content) > 0:
            return response.json()
        return response.content

    def _get_server(self):
        """
        Get server to use for request.
        Also process inactive server list, re-add them after given interval.
        """
        if self._inactive_servers:
            sorted_inactive_servers = sorted(self._inactive_servers, key=itemgetter('timestamp'))

            # first, check for failed servers older than ``interval``
            for server_data in sorted_inactive_servers:
                delta = datetime.now() - server_data.get('timestamp')
                if delta.seconds >= self.retry_interval:
                    self._active_servers.append(server_data.get('server'))
                    logger.info("Restored server %s into active pool", server_data.get('server'))

            # if no one is old enough, use oldest
            if not self._active_servers:
                server_to_add = sorted_inactive_servers.pop(0).get('server')
                self._active_servers.append(server_to_add)
                logger.info("Restored server %s into active pool", server_to_add)

        server = self._active_servers[0]
        self._roundrobin()

        return server

    def _drop_server(self, server, message):
        """
        Drop server from active list and adds it to the inactive ones.
        """
        try:
            self._active_servers.remove(server)
        except ValueError:
            pass
        else:
            self._inactive_servers.append({
                'server': server,
                'message': message,
                'timestamp': datetime.now()
            })
            logger.warning("Removed server %s from active pool", server)

    def _roundrobin(self):
        """
        Very simple round-robin implementation
        """
        self._active_servers.append(self._active_servers.pop(0))

    def __repr__(self):
        return '<Client {}>'.format(str(self._active_servers))
<|endoftext|>


<commit_before># -*- coding: utf-8; -*-
# vi: set encoding=utf-8
#
# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
# license agreements.  See the NOTICE file distributed with this work for
# additional information regarding copyright ownership.  Crate licenses
# this file to you under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.  You may
# obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
# License for the specific language governing permissions and limitations
# under the License.
#
# However, if you have executed another commercial license agreement
# with Crate these terms will supersede the license and you may use the
# software solely pursuant to the terms of the relevant commercial agreement.

from __future__ import print_function

from datetime import datetime
from colorama import Fore, Style

class ColorLog(object):

    def __init__(self, name):
        self.name = name
        self.stream = open('ctop.log', 'a')

    def _print(self, color, level, *args):
        msg = '[{1} {5} {0:<20}] {3}{2}{4}'.format(self.name, level,
            ' '.join([str(x) for x in args]), color, Style.RESET_ALL,
            datetime.now().isoformat())
        print(msg, file=self.stream)
        self.stream.flush()

    def info(self, *args):
        self._print(Fore.GREEN, 'I', *args)

    def warn(self, *args):
        self._print(Fore.YELLOW, 'W', *args)

    def error(self, *args):
        self._print(Fore.RED, 'E', *args)

<commit_msg>improved logging<commit_after># -*- coding: utf-8; -*-
# vi: set encoding=utf-8
#
# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
# license agreements.  See the NOTICE file distributed with this work for
# additional information regarding copyright ownership.  Crate licenses
# this file to you under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.  You may
# obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
# License for the specific language governing permissions and limitations
# under the License.
#
# However, if you have executed another commercial license agreement
# with Crate these terms will supersede the license and you may use the
# software solely pursuant to the terms of the relevant commercial agreement.

from __future__ import print_function

from os import path, mkdir
from datetime import datetime
from colorama import Fore, Style

class ColorLog(object):

    def __init__(self, name):
        self.name = name
        self.stream = open(path.join(self.logdir(), 'ctop.log'), 'a')

    def __del__(self):
        self.stream.close()

    def logdir(self):
        dir = path.expanduser(path.join('~', '.ctop'))
        if not path.exists(dir):
            mkdir(dir)
        return dir

    def _print(self, color, level, *args):
        msg = '[{0}]{4}[{1:<5}]{5}[{2:<20}] {3}'.format(
                datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S'),
                level,
                self.name,
                ' '.join([str(x) for x in args]),
                color,
                Style.RESET_ALL,
                )
        print(msg, file=self.stream)
        self.stream.flush()

    def info(self, *args):
        self._print(Fore.GREEN, 'INFO', *args)

    def warn(self, *args):
        self._print(Fore.YELLOW, 'WARN', *args)

    def error(self, *args):
        self._print(Fore.RED, 'ERROR', *args)

<|endoftext|>


<commit_before>import logging
import sys
import string
import re
from collections import defaultdict

try:
    import pyparsing
    from pyparsing import Literal, Word, Group, Combine, Optional, Forward, alphanums, SkipTo, LineEnd, nums, delimitedList
except ImportError:
    logging.critical("PyParsing has to be installed on the computer")
    sys.exit(-1)

from langtools.string.encoding import decode_from_proszeky

from machine import Machine
from constants import deep_cases
from control import ConceptControl

def create_machine(name, partitions):
    return Machine(decode_from_proszeky(name), ConceptControl(), partitions)

def unify(machine):
    def __collect_machines(m, machines, is_root=False):
        if not is_root:
            machines[m.printname(), __has_other(m)].append(m)
        for partition in m.partitions:
            for m_ in partition:
                __collect_machines(m_, machines)

    def __has_other(m):
        for m_ in m.partitions[0]:
            if m_.printname() == "other":
                return True
        return False

    def __get_unified(machines):
        prototype = machines[0]
        res = create_machine(prototype.printname(), len(prototype.partitions))
        for m in machines:
            for p_i, p in enumerate(m.partitions):
                for part_m in p:
                    if part_m.printname() != "other":
                        res.partitions[p_i].append(part_m)
        return res

    def __replace(where, for_what, is_other=False):
        pn = for_what.printname()
        for p_i, p in enumerate(where.partitions):
            for part_m_i, part_m in enumerate(p):
                if part_m.printname() == pn and __has_other(part_m) == is_other:
                    p[part_m_i] = for_what
                __replace(p[part_m_i], for_what, is_other)

    print machine.to_debug_str()
    machines = defaultdict(list)
    __collect_machines(machine, machines, is_root=True)
    for k, machines_to_unify in machines.iteritems():
        printname, is_other = k
        unified = __get_unified(machines_to_unify)
        __replace(machine, unified, is_other)

class ParserException(Exception):
    pass

class DefinitionParser:
    _str = set([str, unicode])

    lb = "["
    rb = "]"
    lp = "("
    rp = ")"

    def_sep = ":"
    arg_sep = ","
    part_sep = ";"
    comment_sep = "%"
    prime = "'"
    hyphen = "-"
    at = "@"
    dollar = "$"
    hashmark = "#"
    unary_p = re.compile("^[a-z_#\-/0-9]+$")
    binary_p = re.compile("^[A-Z_]+$")

    def __init__(self):
        self.init_parser()

    @classmethod
    def _is_binary(cls, s):
        return (type(s) in cls._str and
               (cls.binary_p.match(s) is not None
                and not s in deep_cases))

    @classmethod
    def _is_unary(cls, s):
        return (type(s) in cls._str and
               (cls.unary_p.match(s) is not None
                or s in deep_cases))

    @classmethod
    def _is_deep_case(cls, s):
        return s in deep_cases

    def init_parser(self):
        self.lb_lit = Literal(DefinitionParser.lb)
        self.rb_lit = Literal(DefinitionParser.rb)
        self.lp_lit = Literal(DefinitionParser.lp)
        self.rp_lit = Literal(DefinitionParser.rp)

        self.def_sep_lit = Literal(DefinitionParser.def_sep)
        self.arg_sep_lit = Literal(DefinitionParser.arg_sep)
        self.part_sep_lit = Literal(DefinitionParser.part_sep)
        self.comment_sep_lit = Literal(DefinitionParser.comment_sep)
        self.prime_lit = Literal(DefinitionParser.prime)
        self.hyphen_lit = Literal(DefinitionParser.hyphen)
        self.at_lit = Literal(DefinitionParser.at)
        self.hashmark_lit = Literal(DefinitionParser.hashmark)
        self.dollar_lit = Literal(DefinitionParser.dollar)

        self.deep_cases = reduce(lambda a, b: a | b,
            (Literal(dc) for dc in deep_cases))

        self.unary = Combine(Optional("-") + Word(string.lowercase + "_" + nums) +
                             Optional(Word(nums))) | self.deep_cases
        self.binary = Word(string.uppercase + "_" + nums)
        self.syntax_supp = self.dollar_lit + Word(string.uppercase + "_")
        self.syntax_avm = self.hashmark_lit+ Word(string.ascii_letters + "_")
        self.syntax_exturl = self.at_lit+ Word(string.ascii_letters + "_")
        self.dontcare = SkipTo(LineEnd())

        # main expression
        self.expression = Forward()
        self.binexpr = Forward()
        self.unexpr = Forward()
        self.argexpr = Forward()

        # "enumerable expression"
        # D -> E | E, D
        self.definition = Group(delimitedList(self.expression,
            delim=DefinitionParser.arg_sep))

        self.expression << Group(
            # E -> UE
            (self.unexpr) ^

            # E -> BE
            (self.binexpr) ^

            # E -> U ( BE )
            (self.unary + self.lp_lit + self.binexpr + self.rp_lit)
        )

        self.binexpr << Group(
            # BE -> A B
            (self.argexpr + self.binary) ^

            # BE -> B A
            (self.binary + self.argexpr) ^

            # BE -> A B A
            (self.argexpr + self.binary + self.argexpr) ^

            # BE -> B [ E; E ]
            (self.binary + self.lb_lit + self.expression + self.part_sep_lit + self.expression + self.rb_lit) ^

            # BE -> 'B
            (self.prime_lit + self.binary) ^

            # BE -> B'
            (self.binary + self.prime_lit)
        )

        self.unexpr << Group(
            # UE -> U
            (self.unary) ^

            # UE -> SS
            (self.syntax_supp) ^

            # UE -> AVM
            (self.syntax_avm) ^

            # UE -> ExtUrl
            (self.syntax_exturl) ^

            # UE -> U [ D ]
            (self.unary + self.lb_lit + self.definition + self.rb_lit) ^

            # UE -> U ( U )
            (self.unary + self.lp_lit + self.unary + self.rp_lit)
        )

        self.argexpr << Group(
            # A -> UE
            (self.unexpr) ^

            # A -> [ D ]
            (self.lb_lit + self.definition + self.rb_lit)
        )

        self.hu, self.pos, self.en, self.lt, self.pt = (Word(alphanums + "#-/_" ),) * 5
        self.defid = Word(nums)
        self.word = Group(self.hu + self.pos + self.en + self.lt + self.pt)

        # S -> W : D | W : D % _
        self.sen = (self.defid + self.word + self.def_sep_lit.suppress() + Optional(self.definition) + Optional(self.comment_sep_lit + self.dontcare).suppress()) + LineEnd()

    def parse(self, s):
        return self.sen.parseString(s).asList()

    def __parse_expr(self, expr, parent, root):
        """
        creates machines from a parse node and its children
        there should be one handler for every rule
        """

        logging.debug("Parsing expression: {0}".format(expr))

        # name shortening for classmethods
        cls = DefinitionParser

        is_binary = cls._is_binary
        is_unary = cls._is_unary
        is_tree = lambda r: type(r) == list

        if (len(expr) == 1):
            # E -> UE | BE
            # A -> UE
            if (is_tree(expr[0])):
                return self.__parse_expr(expr[0], parent, root)

            # UE -> U
            if (is_unary(expr[0])):
                return create_machine(expr[0], 1)

        if (len(expr) == 2):
            # BE -> A B
            if (is_tree(expr[0]) and
                    is_binary(expr[1])):
                m = create_machine(expr[1], 2)
                m.append(self.__parse_expr(expr[0], m, root), 0)
                m.append(root, 1)
                return m

            # BE -> B A
            if (is_binary(expr[0]) and
                    is_tree(expr[1])):
                m = create_machine(expr[0], 2)
                m.append(self.__parse_expr(expr[1], m, root), 1)
                m.append(root, 0)
                return m

            # BE -> 'B
            if (expr[0] == "'" and
                    is_binary(expr[1])):
                m = create_machine(expr[1], 2)
                m.append(parent, 1)
                # nothing to append to any partitions
                return None

            # BE -> B'
            if (is_binary(expr[0]) and
                    expr[1] == "'"):
                m = create_machine(expr[0], 2)
                m.append(parent, 0)
                # nothing to append to any partitions
                return None

            # UE -> SS
            if (expr[0] == cls.dollar):
                return create_machine(cls.dollar + expr[1], 1)

            # UE -> AVM
            if (expr[0] == cls.hashmark):
                return create_machine(cls.hashmark + expr[1], 1)

            # UE -> External url
            if (expr[0] == cls.at):
                return create_machine(cls.at+ expr[1], 1)

        if (len(expr) == 3):
            # UB -> A B A
            if (is_tree(expr[0]) and
                    is_binary(expr[1]) and
                    is_tree(expr[2])):
                m = create_machine(expr[1], 2)
                logging.debug(expr[0])
                logging.debug(self.__parse_expr(expr[0], m, root))
                m.append(self.__parse_expr(expr[0], m, root), 0)
                m.append(self.__parse_expr(expr[2], m, root), 1)
                return m

            # A -> [ D ]
            if (expr[0] == "[" and
                    is_tree(expr[1]) and
                    expr[2] == "]"):
                return list(self.__parse_definition(expr[1], parent, root))

        if (len(expr) == 4):
            # E -> U ( BE )
            if (is_unary(expr[0]) and
                    expr[1] == "(" and
                    is_tree(expr[2]) and
                    expr[3] == ")"):
                m = self.__parse_expr(expr[2], parent, root)

                # if BE was an expression with an apostrophe, then
                # return of __parse_expr() is None
                if m is not None:
                    m.append(create_machine(expr[1], 1), 0)
                return m

            # UE -> U [ D ]
            if (is_unary(expr[0]) and
                    expr[1] == "[" and
                    is_tree(expr[2]) and
                    expr[3] == "]"):
                m = create_machine(expr[0], 1)
                for parsed_expr in self.__parse_definition(expr[2], m, root):
                    m.append(parsed_expr, 0)
                return m

            # UE -> U ( U )
            if (is_unary(expr[0]) and
                    expr[1] == "(" and
                    is_unary(expr[2]) and
                    expr[3] == ")"):
                m = create_machine(expr[2], 1)
                m.append(create_machine(expr[0], 1), 0)
                return m

        if (len(expr) == 6):
            # BE -> B [E; E]
            if (is_binary(expr[0]) and
                    expr[1] == "[" and
                    is_tree(expr[2]) and
                    expr[3] == ";" and
                    is_tree(expr[4]) and
                    expr[5] == "]"):
                m = create_machine(expr[0], 2)
                m.append(self.__parse_expr(expr[2], m, root), 0)
                m.append(self.__parse_expr(expr[4], m, root), 1)
                return m

        pe = ParserException("Unknown expression in definition: "+str(expr))
        logging.debug(str(pe))
        logging.debug(expr)
        raise pe

    def __parse_definition(self, definition, parent, root):
        for d in definition:
            yield self.__parse_expr(d, parent, root)

    def parse_into_machines(self, s, printname_index=0):
        parsed = self.parse(s)

        machine = create_machine(parsed[1][printname_index], 1)
        if len(parsed) > 2:
            for parsed_expr in self.__parse_definition(parsed[2], machine, machine):
                machine.append(parsed_expr, 0)

        unify(machine)
        return machine

def read(f, printname_index=0):
    d = {}
    dp = DefinitionParser()
    for line in f:
        l = line.strip()
        logging.info("Parsing: {0}".format(l))
        if len(l) == 0:
            continue
        if l.startswith("%"):
            continue
        try:
            m = dp.parse_into_machines(l, printname_index)
            d[m.printname()] = m
        except pyparsing.ParseException, pe:
            print l
            print "Error: ", str(pe)

    return d

if __name__ == "__main__":
    #logging.basicConfig(level=logging.DEBUG)
    logging.basicConfig(level=logging.DEBUG, format="%(asctime)s : %(module)s (%(lineno)s) - %(levelname)s - %(message)s")
    dp = DefinitionParser()
    pstr = sys.argv[-1]
    if sys.argv[1] == "-d":
        print Machine.to_debug_str(dp.parse_into_machines(pstr))
    elif sys.argv[1] == "-f":
        lexicon = read(file(sys.argv[2]))
    else:
        print dp.parse(pstr)

<commit_msg>modified:   definition_parser.py - half-prepared for later Machine.append_all() (see TODOs)<commit_after>import logging
import sys
import string
import re
from collections import defaultdict

try:
    import pyparsing
    from pyparsing import Literal, Word, Group, Combine, Optional, Forward, alphanums, SkipTo, LineEnd, nums, delimitedList
except ImportError:
    logging.critical("PyParsing has to be installed on the computer")
    sys.exit(-1)

from langtools.string.encoding import decode_from_proszeky

from machine import Machine
from constants import deep_cases
from control import ConceptControl

def create_machine(name, partitions):
    return Machine(decode_from_proszeky(name), ConceptControl(), partitions)

def unify(machine):
    def __collect_machines(m, machines, is_root=False):
        if not is_root:
            machines[m.printname(), __has_other(m)].append(m)
        for partition in m.partitions:
            for m_ in partition:
                __collect_machines(m_, machines)

    def __has_other(m):
        for m_ in m.partitions[0]:
            if m_.printname() == "other":
                return True
        return False

    def __get_unified(machines):
        prototype = machines[0]
        res = create_machine(prototype.printname(), len(prototype.partitions))
        for m in machines:
            for p_i, p in enumerate(m.partitions):
                for part_m in p:
                    if part_m.printname() != "other":
                        res.partitions[p_i].append(part_m)
        return res

    def __replace(where, for_what, is_other=False):
        pn = for_what.printname()
        for p_i, p in enumerate(where.partitions):
            for part_m_i, part_m in enumerate(p):
                if part_m.printname() == pn and __has_other(part_m) == is_other:
                    p[part_m_i] = for_what
                __replace(p[part_m_i], for_what, is_other)

    print machine.to_debug_str()
    machines = defaultdict(list)
    __collect_machines(machine, machines, is_root=True)
    for k, machines_to_unify in machines.iteritems():
        printname, is_other = k
        unified = __get_unified(machines_to_unify)
        __replace(machine, unified, is_other)

class ParserException(Exception):
    pass

class DefinitionParser:
    _str = set([str, unicode])

    lb = "["
    rb = "]"
    lp = "("
    rp = ")"

    def_sep = ":"
    arg_sep = ","
    part_sep = ";"
    comment_sep = "%"
    prime = "'"
    hyphen = "-"
    at = "@"
    dollar = "$"
    hashmark = "#"
    unary_p = re.compile("^[a-z_#\-/0-9]+$")
    binary_p = re.compile("^[A-Z_]+$")

    def __init__(self):
        self.init_parser()

    @classmethod
    def _is_binary(cls, s):
        return (type(s) in cls._str and
               (cls.binary_p.match(s) is not None
                and not s in deep_cases))

    @classmethod
    def _is_unary(cls, s):
        return (type(s) in cls._str and
               (cls.unary_p.match(s) is not None
                or s in deep_cases))

    @classmethod
    def _is_deep_case(cls, s):
        return s in deep_cases

    def init_parser(self):
        self.lb_lit = Literal(DefinitionParser.lb)
        self.rb_lit = Literal(DefinitionParser.rb)
        self.lp_lit = Literal(DefinitionParser.lp)
        self.rp_lit = Literal(DefinitionParser.rp)

        self.def_sep_lit = Literal(DefinitionParser.def_sep)
        self.arg_sep_lit = Literal(DefinitionParser.arg_sep)
        self.part_sep_lit = Literal(DefinitionParser.part_sep)
        self.comment_sep_lit = Literal(DefinitionParser.comment_sep)
        self.prime_lit = Literal(DefinitionParser.prime)
        self.hyphen_lit = Literal(DefinitionParser.hyphen)
        self.at_lit = Literal(DefinitionParser.at)
        self.hashmark_lit = Literal(DefinitionParser.hashmark)
        self.dollar_lit = Literal(DefinitionParser.dollar)

        self.deep_cases = reduce(lambda a, b: a | b,
            (Literal(dc) for dc in deep_cases))

        self.unary = Combine(Optional("-") + Word(string.lowercase + "_" + nums) +
                             Optional(Word(nums))) | self.deep_cases
        self.binary = Word(string.uppercase + "_" + nums)
        self.syntax_supp = self.dollar_lit + Word(string.uppercase + "_")
        self.syntax_avm = self.hashmark_lit+ Word(string.ascii_letters + "_")
        self.syntax_exturl = self.at_lit+ Word(string.ascii_letters + "_")
        self.dontcare = SkipTo(LineEnd())

        # main expression
        self.expression = Forward()
        self.binexpr = Forward()
        self.unexpr = Forward()
        self.argexpr = Forward()

        # "enumerable expression"
        # D -> E | E, D
        self.definition = Group(delimitedList(self.expression,
            delim=DefinitionParser.arg_sep))

        self.expression << Group(
            # E -> UE
            (self.unexpr) ^

            # E -> BE
            (self.binexpr) ^

            # E -> U ( BE )
            (self.unary + self.lp_lit + self.binexpr + self.rp_lit)
        )

        self.binexpr << Group(
            # BE -> A B
            (self.argexpr + self.binary) ^

            # BE -> B A
            (self.binary + self.argexpr) ^

            # BE -> A B A
            (self.argexpr + self.binary + self.argexpr) ^

            # BE -> B [ E; E ]
            (self.binary + self.lb_lit + self.expression + self.part_sep_lit + self.expression + self.rb_lit) ^

            # BE -> 'B
            (self.prime_lit + self.binary) ^

            # BE -> B'
            (self.binary + self.prime_lit)
        )

        self.unexpr << Group(
            # UE -> U
            (self.unary) ^

            # UE -> SS
            (self.syntax_supp) ^

            # UE -> AVM
            (self.syntax_avm) ^

            # UE -> ExtUrl
            (self.syntax_exturl) ^

            # UE -> U [ D ]
            (self.unary + self.lb_lit + self.definition + self.rb_lit) ^

            # UE -> U ( U )
            (self.unary + self.lp_lit + self.unary + self.rp_lit)
        )

        self.argexpr << Group(
            # A -> UE
            (self.unexpr) ^

            # A -> [ D ]
            (self.lb_lit + self.definition + self.rb_lit)
        )

        self.hu, self.pos, self.en, self.lt, self.pt = (Word(alphanums + "#-/_" ),) * 5
        self.defid = Word(nums)
        self.word = Group(self.hu + self.pos + self.en + self.lt + self.pt)

        # S -> W : D | W : D % _
        self.sen = (self.defid + self.word + self.def_sep_lit.suppress() + Optional(self.definition) + Optional(self.comment_sep_lit + self.dontcare).suppress()) + LineEnd()

    def parse(self, s):
        return self.sen.parseString(s).asList()

    def __parse_expr(self, expr, parent, root):
        """
        creates machines from a parse node and its children
        there should be one handler for every rule
        """

        logging.debug("Parsing expression: {0}".format(expr))

        # name shortening for classmethods
        cls = DefinitionParser

        is_binary = cls._is_binary
        is_unary = cls._is_unary
        is_tree = lambda r: type(r) == list

        if (len(expr) == 1):
            # E -> UE | BE
            # A -> UE
            if (is_tree(expr[0])):
                return self.__parse_expr(expr[0], parent, root)

            # UE -> U
            if (is_unary(expr[0])):
                return [create_machine(expr[0], 1)]

        if (len(expr) == 2):
            # BE -> A B
            if (is_tree(expr[0]) and
                    is_binary(expr[1])):
                m = create_machine(expr[1], 2)
                # TODO: append_all later
                m.append(self.__parse_expr(expr[0], m, root), 0)
                m.append(root, 1)
                return [m]

            # BE -> B A
            if (is_binary(expr[0]) and
                    is_tree(expr[1])):
                m = create_machine(expr[0], 2)
                # TODO: append_all later
                m.append(self.__parse_expr(expr[1], m, root), 1)
                m.append(root, 0)
                return [m]

            # BE -> 'B
            if (expr[0] == "'" and
                    is_binary(expr[1])):
                m = create_machine(expr[1], 2)
                m.append(parent, 1)
                # nothing to append to any partitions
                return []

            # BE -> B'
            if (is_binary(expr[0]) and
                    expr[1] == "'"):
                m = create_machine(expr[0], 2)
                m.append(parent, 0)
                # nothing to append to any partitions
                return []

            # UE -> SS
            if (expr[0] == cls.dollar):
                return [create_machine(cls.dollar + expr[1], 1)]

            # UE -> AVM
            if (expr[0] == cls.hashmark):
                return [create_machine(cls.hashmark + expr[1], 1)]

            # UE -> External url
            if (expr[0] == cls.at):
                return [create_machine(cls.at+ expr[1], 1)]

        if (len(expr) == 3):
            # UB -> A B A
            if (is_tree(expr[0]) and
                    is_binary(expr[1]) and
                    is_tree(expr[2])):
                m = create_machine(expr[1], 2)
                logging.debug(expr[0])
                logging.debug(self.__parse_expr(expr[0], m, root))
                # TODO: append_all later
                m.append(self.__parse_expr(expr[0], m, root), 0)
                # TODO: append_all later
                m.append(self.__parse_expr(expr[2], m, root), 1)
                return [m]

            # A -> [ D ]
            if (expr[0] == "[" and
                    is_tree(expr[1]) and
                    expr[2] == "]"):
                return list(self.__parse_definition(expr[1], parent, root))

        if (len(expr) == 4):
            # E -> U ( BE )
            if (is_unary(expr[0]) and
                    expr[1] == "(" and
                    is_tree(expr[2]) and
                    expr[3] == ")"):
                ms = self.__parse_expr(expr[2], parent, root)

                # if BE was an expression with an apostrophe, then
                # return of __parse_expr() is None
                if len(ms) != 0:
                    ms[0].append(create_machine(expr[1], 1), 0)
                if len(ms) != 1:
                    raise ParserException("semantics of U(BE) rule has errors")
                return ms

            # UE -> U [ D ]
            if (is_unary(expr[0]) and
                    expr[1] == "[" and
                    is_tree(expr[2]) and
                    expr[3] == "]"):
                m = create_machine(expr[0], 1)
                for parsed_expr in self.__parse_definition(expr[2], m, root):
                    m.append(parsed_expr, 0)
                return [m]

            # UE -> U ( U )
            if (is_unary(expr[0]) and
                    expr[1] == "(" and
                    is_unary(expr[2]) and
                    expr[3] == ")"):
                m = create_machine(expr[2], 1)
                m.append(create_machine(expr[0], 1), 0)
                return [m]

        if (len(expr) == 6):
            # BE -> B [E; E]
            if (is_binary(expr[0]) and
                    expr[1] == "[" and
                    is_tree(expr[2]) and
                    expr[3] == ";" and
                    is_tree(expr[4]) and
                    expr[5] == "]"):
                m = create_machine(expr[0], 2)
                # TODO: append_all later
                m.append(self.__parse_expr(expr[2], m, root), 0)
                # TODO: append_all later
                m.append(self.__parse_expr(expr[4], m, root), 1)
                return [m]

        pe = ParserException("Unknown expression in definition: "+str(expr))
        logging.debug(str(pe))
        logging.debug(expr)
        raise pe

    def __parse_definition(self, definition, parent, root):
        for d in definition:
            yield self.__parse_expr(d, parent, root)

    def parse_into_machines(self, s, printname_index=0):
        parsed = self.parse(s)

        machine = create_machine(parsed[1][printname_index], 1)
        if len(parsed) > 2:
            for parsed_expr in self.__parse_definition(parsed[2], machine, machine):
                machine.append(parsed_expr, 0)

        unify(machine)
        return machine

def read(f, printname_index=0):
    d = {}
    dp = DefinitionParser()
    for line in f:
        l = line.strip()
        logging.info("Parsing: {0}".format(l))
        if len(l) == 0:
            continue
        if l.startswith("%"):
            continue
        try:
            m = dp.parse_into_machines(l, printname_index)
            d[m.printname()] = m
        except pyparsing.ParseException, pe:
            print l
            print "Error: ", str(pe)

    return d

if __name__ == "__main__":
    #logging.basicConfig(level=logging.DEBUG)
    logging.basicConfig(level=logging.DEBUG, format="%(asctime)s : %(module)s (%(lineno)s) - %(levelname)s - %(message)s")
    dp = DefinitionParser()
    pstr = sys.argv[-1]
    if sys.argv[1] == "-d":
        print Machine.to_debug_str(dp.parse_into_machines(pstr))
    elif sys.argv[1] == "-f":
        lexicon = read(file(sys.argv[2]))
    else:
        print dp.parse(pstr)

<|endoftext|>