<commit_before>from __future__ import print_function

import os
import math
import warnings

import numpy as np

from matplotlib.collections import LineCollection
from matplotlib.transforms import Bbox
from matplotlib.patches import Polygon

from .geometry.path import Path, get_endpoints
from . import extract_pv_slice

def distance(x1, y1, x2, y2, x3, y3):
    """
    Find the shortest distance between a point (x3, y3) and the line passing
    through the points (x1, y1) and (x2, y2).
    """

    px = x2-x1
    py = y2-y1

    something = px * px + py * py

    u =  ((x3 - x1) * px + (y3 - y1) * py) / float(something)

    x = x1 + u * px
    y = y1 + u * py

    dx = x - x3
    dy = y - y3

    dist = math.sqrt(dx*dx + dy*dy)

    return dist

class MovableSliceBox(object):

    def __init__(self, box, callback):
        self.box = box
        self.press = None
        # self.background = None
        self.point_counter = 0
        self.callback = callback
        self.mode = 0
        self.show_poly = False
        self.cidpress = self.box.figure.canvas.mpl_connect('draw_event', self.draw_slicer)

    def connect(self):
        self.cidpress = self.box.figure.canvas.mpl_connect('key_press_event', self.key_press)
        self.cidpress = self.box.figure.canvas.mpl_connect('button_press_event', self.on_press)
        self.cidmotion = self.box.figure.canvas.mpl_connect('motion_notify_event', self.on_motion)

    def draw_slicer(self, event):

        axes = self.box.axes
        canvas = self.box.figure.canvas

        self.box.axes.draw_artist(self.box)

        if self.show_poly:

            path = Path(zip(self.box.x, self.box.y))
            path.width = self.box.width

            patches = path.to_patches(1, ec='green', fc='none',
                                          transform=self.box.axes.transData,
                                          clip_on=True, clip_box=self.box.axes.bbox)

            for patch in patches:
                self.box.axes.draw_artist(patch)

    def on_press(self, event):

        if self.box.figure.canvas.toolbar.mode != '':
            return

        if event.inaxes != self.box.axes:
            return

        if self.mode == 1:
            self.callback(self.box)
            self.mode += 1
            return

        if self.mode == 2:
            self.box.x = []
            self.box.y = []
            self.mode = 0
            self.point_counter = 0

        self.press = event.xdata, event.ydata

        self.point_counter += 1

        axes = self.box.axes
        canvas = self.box.figure.canvas

        if self.point_counter == 1:  # first point

            self.box.x.append(event.xdata)
            self.box.x.append(event.xdata)
            self.box.y.append(event.ydata)
            self.box.y.append(event.ydata)

            self.box.width = 0.

            self.box.set_animated(True)
            canvas.draw()
            self.background = canvas.copy_from_bbox(self.box.axes.bbox)

        elif self.mode == 0:

            self.box.x.append(event.xdata)
            self.box.y.append(event.ydata)

        self.box._update_segments()

        # now redraw just the lineangle
        axes.draw_artist(self.box)

    def key_press(self, event):

        if self.box.figure.canvas.toolbar.mode != '':
            return

        if event.key == 'enter' and self.mode == 0:
            self.mode += 1
            self.box.x = self.box.x[:-1]
            self.box.y = self.box.y[:-1]

        if event.key == 'y' and self.mode == 2:
            self.show_poly = not self.show_poly
            self.draw_slicer(event)
            self.box.figure.canvas.draw()

    def on_motion(self, event):

        if self.box.figure.canvas.toolbar.mode != '':
            return

        if self.point_counter == 0:
            return

        if self.mode == 2:
            return

        canvas = self.box.figure.canvas
        axes = self.box.axes
        canvas.restore_region(self.background)

        if event.inaxes != self.box.axes:
            return

        if self.mode == 0:
            self.box.x[-1] = event.xdata
            self.box.y[-1] = event.ydata
        elif self.mode == 1:
            self.box.width = distance(self.box.x[-2], self.box.y[-2], self.box.x[-1], self.box.y[-1], event.xdata, event.ydata) * 2

        self.box._update_segments()

        # redraw just the current lineangle
        axes.draw_artist(self.box)

        # blit just the redrawn area
        canvas.blit(axes.bbox)

    def disconnect(self):
        self.box.figure.canvas.mpl_disconnect(self.cidpress)
        self.box.figure.canvas.mpl_disconnect(self.cidmotion)

class SliceCurve(LineCollection):

    def __init__(self, x=[], y=[], width=None, **kwargs):

        super(SliceCurve, self).__init__([], **kwargs)

        self.x = x
        self.y = y
        self.width = width

        self._update_segments()

    def _update_segments(self):

        if not self.x or self.width is None or len(self.x) < 2:
            return

        # Find central line
        line = zip(self.x, self.y)

        if self.width:

            x1, y1, x2, y2 = get_endpoints(self.x, self.y, self.width)

            # Find bounding rectangle
            rect = zip(np.hstack([x1,x2[::-1], x1[0]]),
                       np.hstack([y1,y2[::-1], y1[0]]))

            self.set_segments([list(line), list(rect)])
            self.set_linestyles(['solid', 'dashed'])
            self.set_linewidths([2, 1])

        else:

            self.set_segments([list(line)])
            self.set_linestyles(['solid'])
            self.set_linewidths([2,])

def unitless(x):
    if hasattr(x, 'unit'):
        return x.value
    else:
        return x

class PVSlicer(object):

    def __init__(self, filename_or_cube, backend="Qt5Agg", clim=None):

        try:
            from spectral_cube import SpectralCube
            if isinstance(filename_or_cube, SpectralCube):
                cube = filename_or_cube
            else:
                cube = SpectralCube.read(filename_or_cube, format='fits')
            self.cube = cube
            self.array = self.cube
            self.shape = cube.shape
        except ImportError:
            warnings.warn("spectral_cube package is not available - using astropy.io.fits directly")
            from astropy.io import fits
            self.array = fits.getdata(filename_or_cube)
            self.shape = array.shape
            self.ok_mask = np.isfinite(self.array)
            if self.array.ndim != 3:
                raise ValueError("dataset does not have 3 dimensions (install the spectral_cube package to avoid this error)")

        self.backend = backend

        import matplotlib as mpl

        mpl.use(self.backend)
        import matplotlib.pyplot as plt

        self.fig = plt.figure(figsize=(14, 8))

        self.ax1 = self.fig.add_axes([0.1, 0.1, 0.4, 0.7])

        if clim is None:
            warnings.warn("clim not defined and will be determined from the data")
            # To work with large arrays, sub-sample the data
            # (but don't do it for small arrays)
            n1 = int(np.round(max(self.shape[0] / 10, 1)))
            n2 = int(np.round(max(self.shape[1] / 10, 1)))
            n3 = int(np.round(max(self.shape[2] / 10, 1)))
            if hasattr(self,'cube'):
                sub_cube = self.cube[::n1,::n2,::n3]
                cmin = sub_cube.min().value
                cmax = sub_cube.max().value
            else:
                sub_array = self.array[::n1,::n2,::n3]
                sub_mask = self.ok_mask[::n1,::n2,::n3]
                cmin = sub_array[sub_mask].min()
                cmax = sub_array[sub_mask].max()
            crange = cmax - cmin
            self._clim = (cmin - crange, cmax + crange)
        else:
            self._clim = clim

        self.slice = int(round(self.shape[0] / 2.))

        from matplotlib.widgets import Slider

        self.slice_slider_ax = self.fig.add_axes([0.1, 0.95, 0.4, 0.03])
        self.slice_slider_ax.set_xticklabels("")
        self.slice_slider_ax.set_yticklabels("")
        self.slice_slider = Slider(self.slice_slider_ax, "3-d slice", 0, self.shape[0]-1, valinit=self.slice, valfmt="%i")
        self.slice_slider.on_changed(self.update_slice)
        self.slice_slider.drawon = False

        self.image = self.ax1.imshow(unitless(self.array[self.slice, :,:]),
                                     origin='lower', interpolation='nearest',
                                     vmin=self._clim[0], vmax=self._clim[1],
                                     cmap=plt.cm.gray)

        self.vmin_slider_ax = self.fig.add_axes([0.1, 0.90, 0.4, 0.03])
        self.vmin_slider_ax.set_xticklabels("")
        self.vmin_slider_ax.set_yticklabels("")
        self.vmin_slider = Slider(self.vmin_slider_ax, "vmin", self._clim[0], self._clim[1], valinit=self._clim[0])
        self.vmin_slider.on_changed(self.update_vmin)
        self.vmin_slider.drawon = False

        self.vmax_slider_ax = self.fig.add_axes([0.1, 0.85, 0.4, 0.03])
        self.vmax_slider_ax.set_xticklabels("")
        self.vmax_slider_ax.set_yticklabels("")
        self.vmax_slider = Slider(self.vmax_slider_ax, "vmax", self._clim[0], self._clim[1], valinit=self._clim[1])
        self.vmax_slider.on_changed(self.update_vmax)
        self.vmax_slider.drawon = False

        self.grid1 = None
        self.grid2 = None
        self.grid3 = None

        self.ax2 = self.fig.add_axes([0.55, 0.1, 0.4, 0.7])

        # Add slicing box
        self.box = SliceCurve(colors=(0.8, 0.0, 0.0))
        self.ax1.add_collection(self.box)
        self.movable = MovableSliceBox(self.box, callback=self.update_pv_slice)
        self.movable.connect()

        # Add save button
        from matplotlib.widgets import Button
        self.save_button_ax = self.fig.add_axes([0.65, 0.90, 0.20, 0.05])
        self.save_button = Button(self.save_button_ax, 'Save slice to FITS')
        self.save_button.on_clicked(self.save_fits)
        self.file_status_text = self.fig.text(0.75, 0.875, "", ha='center', va='center')
        self.set_file_status(None)

        self.set_file_status(None)
        self.pv_slice = None

        self.cidpress = self.fig.canvas.mpl_connect('button_press_event', self.click)

    def set_file_status(self, status, filename=None):
        if status == 'instructions':
            self.file_status_text.set_text('Please enter filename in terminal')
            self.file_status_text.set_color('red')
        elif status == 'saved':
            self.file_status_text.set_text('File successfully saved to {0}'.format(filename))
            self.file_status_text.set_color('green')
        else:
            self.file_status_text.set_text('')
            self.file_status_text.set_color('black')
        self.fig.canvas.draw()

    def click(self, event):

        if event.inaxes != self.ax2:
            return

        self.slice_slider.set_val(event.ydata)

    def save_fits(self, *args, **kwargs):

        self.set_file_status('instructions')

        print("Enter filename: ", end='')
        try:
            plot_name = raw_input()
        except NameError:
            plot_name = input()

        if self.pv_slice is None:
            return

        from astropy.io import fits
        self.pv_slice.writeto(plot_name, clobber=True)
        print("Saved file to: ", plot_name)

        self.set_file_status('saved', filename=plot_name)

    def update_pv_slice(self, box):

        path = Path(zip(box.x, box.y))
        path.width = box.width

        self.pv_slice = extract_pv_slice(self.array, path)

        self.ax2.cla()
        self.ax2.imshow(self.pv_slice.data, origin='lower', aspect='auto', interpolation='nearest')

        self.fig.canvas.draw()

    def show(self, block=True):
        import matplotlib.pyplot as plt
        plt.show(block=block)

    def update_slice(self, pos=None):

        if self.array.ndim == 2:
            self.image.set_array(unitless(self.array))
        else:
            self.slice = int(round(pos))
            self.image.set_array(unitless(self.array[self.slice, :, :]))

        self.fig.canvas.draw()

    def update_vmin(self, vmin):
        if vmin > self._clim[1]:
            self._clim = (self._clim[1], self._clim[1])
        else:
            self._clim = (vmin, self._clim[1])
        self.image.set_clim(*self._clim)
        self.fig.canvas.draw()

    def update_vmax(self, vmax):
        if vmax < self._clim[0]:
            self._clim = (self._clim[0], self._clim[0])
        else:
            self._clim = (self._clim[0], vmax)
        self.image.set_clim(*self._clim)
        self.fig.canvas.draw()
<commit_msg>Uncomment code<commit_after>from __future__ import print_function

import os
import math
import warnings

import numpy as np

from matplotlib.collections import LineCollection
from matplotlib.transforms import Bbox
from matplotlib.patches import Polygon

from .geometry.path import Path, get_endpoints
from . import extract_pv_slice

def distance(x1, y1, x2, y2, x3, y3):
    """
    Find the shortest distance between a point (x3, y3) and the line passing
    through the points (x1, y1) and (x2, y2).
    """

    px = x2-x1
    py = y2-y1

    something = px * px + py * py

    u =  ((x3 - x1) * px + (y3 - y1) * py) / float(something)

    x = x1 + u * px
    y = y1 + u * py

    dx = x - x3
    dy = y - y3

    dist = math.sqrt(dx*dx + dy*dy)

    return dist

class MovableSliceBox(object):

    def __init__(self, box, callback):
        self.box = box
        self.press = None
        self.background = None
        self.point_counter = 0
        self.callback = callback
        self.mode = 0
        self.show_poly = False
        self.cidpress = self.box.figure.canvas.mpl_connect('draw_event', self.draw_slicer)

    def connect(self):
        self.cidpress = self.box.figure.canvas.mpl_connect('key_press_event', self.key_press)
        self.cidpress = self.box.figure.canvas.mpl_connect('button_press_event', self.on_press)
        self.cidmotion = self.box.figure.canvas.mpl_connect('motion_notify_event', self.on_motion)

    def draw_slicer(self, event):

        axes = self.box.axes
        canvas = self.box.figure.canvas

        self.box.axes.draw_artist(self.box)

        if self.show_poly:

            path = Path(zip(self.box.x, self.box.y))
            path.width = self.box.width

            patches = path.to_patches(1, ec='green', fc='none',
                                          transform=self.box.axes.transData,
                                          clip_on=True, clip_box=self.box.axes.bbox)

            for patch in patches:
                self.box.axes.draw_artist(patch)

    def on_press(self, event):

        if self.box.figure.canvas.toolbar.mode != '':
            return

        if event.inaxes != self.box.axes:
            return

        if self.mode == 1:
            self.callback(self.box)
            self.mode += 1
            return

        if self.mode == 2:
            self.box.x = []
            self.box.y = []
            self.mode = 0
            self.point_counter = 0

        self.press = event.xdata, event.ydata

        self.point_counter += 1

        axes = self.box.axes
        canvas = self.box.figure.canvas

        if self.point_counter == 1:  # first point

            self.box.x.append(event.xdata)
            self.box.x.append(event.xdata)
            self.box.y.append(event.ydata)
            self.box.y.append(event.ydata)

            self.box.width = 0.

            self.box.set_animated(True)
            canvas.draw()
            self.background = canvas.copy_from_bbox(self.box.axes.bbox)

        elif self.mode == 0:

            self.box.x.append(event.xdata)
            self.box.y.append(event.ydata)

        self.box._update_segments()

        # now redraw just the lineangle
        axes.draw_artist(self.box)

    def key_press(self, event):

        if self.box.figure.canvas.toolbar.mode != '':
            return

        if event.key == 'enter' and self.mode == 0:
            self.mode += 1
            self.box.x = self.box.x[:-1]
            self.box.y = self.box.y[:-1]

        if event.key == 'y' and self.mode == 2:
            self.show_poly = not self.show_poly
            self.draw_slicer(event)
            self.box.figure.canvas.draw()

    def on_motion(self, event):

        if self.box.figure.canvas.toolbar.mode != '':
            return

        if self.point_counter == 0:
            return

        if self.mode == 2:
            return

        canvas = self.box.figure.canvas
        axes = self.box.axes
        canvas.restore_region(self.background)

        if event.inaxes != self.box.axes:
            return

        if self.mode == 0:
            self.box.x[-1] = event.xdata
            self.box.y[-1] = event.ydata
        elif self.mode == 1:
            self.box.width = distance(self.box.x[-2], self.box.y[-2], self.box.x[-1], self.box.y[-1], event.xdata, event.ydata) * 2

        self.box._update_segments()

        # redraw just the current lineangle
        axes.draw_artist(self.box)

        # blit just the redrawn area
        canvas.blit(axes.bbox)

    def disconnect(self):
        self.box.figure.canvas.mpl_disconnect(self.cidpress)
        self.box.figure.canvas.mpl_disconnect(self.cidmotion)

class SliceCurve(LineCollection):

    def __init__(self, x=[], y=[], width=None, **kwargs):

        super(SliceCurve, self).__init__([], **kwargs)

        self.x = x
        self.y = y
        self.width = width

        self._update_segments()

    def _update_segments(self):

        if not self.x or self.width is None or len(self.x) < 2:
            return

        # Find central line
        line = zip(self.x, self.y)

        if self.width:

            x1, y1, x2, y2 = get_endpoints(self.x, self.y, self.width)

            # Find bounding rectangle
            rect = zip(np.hstack([x1,x2[::-1], x1[0]]),
                       np.hstack([y1,y2[::-1], y1[0]]))

            self.set_segments([list(line), list(rect)])
            self.set_linestyles(['solid', 'dashed'])
            self.set_linewidths([2, 1])

        else:

            self.set_segments([list(line)])
            self.set_linestyles(['solid'])
            self.set_linewidths([2,])

def unitless(x):
    if hasattr(x, 'unit'):
        return x.value
    else:
        return x

class PVSlicer(object):

    def __init__(self, filename_or_cube, backend="Qt5Agg", clim=None):

        try:
            from spectral_cube import SpectralCube
            if isinstance(filename_or_cube, SpectralCube):
                cube = filename_or_cube
            else:
                cube = SpectralCube.read(filename_or_cube, format='fits')
            self.cube = cube
            self.array = self.cube
            self.shape = cube.shape
        except ImportError:
            warnings.warn("spectral_cube package is not available - using astropy.io.fits directly")
            from astropy.io import fits
            self.array = fits.getdata(filename_or_cube)
            self.shape = array.shape
            self.ok_mask = np.isfinite(self.array)
            if self.array.ndim != 3:
                raise ValueError("dataset does not have 3 dimensions (install the spectral_cube package to avoid this error)")

        self.backend = backend

        import matplotlib as mpl

        mpl.use(self.backend)
        import matplotlib.pyplot as plt

        self.fig = plt.figure(figsize=(14, 8))

        self.ax1 = self.fig.add_axes([0.1, 0.1, 0.4, 0.7])

        if clim is None:
            warnings.warn("clim not defined and will be determined from the data")
            # To work with large arrays, sub-sample the data
            # (but don't do it for small arrays)
            n1 = int(np.round(max(self.shape[0] / 10, 1)))
            n2 = int(np.round(max(self.shape[1] / 10, 1)))
            n3 = int(np.round(max(self.shape[2] / 10, 1)))
            if hasattr(self,'cube'):
                sub_cube = self.cube[::n1,::n2,::n3]
                cmin = sub_cube.min().value
                cmax = sub_cube.max().value
            else:
                sub_array = self.array[::n1,::n2,::n3]
                sub_mask = self.ok_mask[::n1,::n2,::n3]
                cmin = sub_array[sub_mask].min()
                cmax = sub_array[sub_mask].max()
            crange = cmax - cmin
            self._clim = (cmin - crange, cmax + crange)
        else:
            self._clim = clim

        self.slice = int(round(self.shape[0] / 2.))

        from matplotlib.widgets import Slider

        self.slice_slider_ax = self.fig.add_axes([0.1, 0.95, 0.4, 0.03])
        self.slice_slider_ax.set_xticklabels("")
        self.slice_slider_ax.set_yticklabels("")
        self.slice_slider = Slider(self.slice_slider_ax, "3-d slice", 0, self.shape[0]-1, valinit=self.slice, valfmt="%i")
        self.slice_slider.on_changed(self.update_slice)
        self.slice_slider.drawon = False

        self.image = self.ax1.imshow(unitless(self.array[self.slice, :,:]),
                                     origin='lower', interpolation='nearest',
                                     vmin=self._clim[0], vmax=self._clim[1],
                                     cmap=plt.cm.gray)

        self.vmin_slider_ax = self.fig.add_axes([0.1, 0.90, 0.4, 0.03])
        self.vmin_slider_ax.set_xticklabels("")
        self.vmin_slider_ax.set_yticklabels("")
        self.vmin_slider = Slider(self.vmin_slider_ax, "vmin", self._clim[0], self._clim[1], valinit=self._clim[0])
        self.vmin_slider.on_changed(self.update_vmin)
        self.vmin_slider.drawon = False

        self.vmax_slider_ax = self.fig.add_axes([0.1, 0.85, 0.4, 0.03])
        self.vmax_slider_ax.set_xticklabels("")
        self.vmax_slider_ax.set_yticklabels("")
        self.vmax_slider = Slider(self.vmax_slider_ax, "vmax", self._clim[0], self._clim[1], valinit=self._clim[1])
        self.vmax_slider.on_changed(self.update_vmax)
        self.vmax_slider.drawon = False

        self.grid1 = None
        self.grid2 = None
        self.grid3 = None

        self.ax2 = self.fig.add_axes([0.55, 0.1, 0.4, 0.7])

        # Add slicing box
        self.box = SliceCurve(colors=(0.8, 0.0, 0.0))
        self.ax1.add_collection(self.box)
        self.movable = MovableSliceBox(self.box, callback=self.update_pv_slice)
        self.movable.connect()

        # Add save button
        from matplotlib.widgets import Button
        self.save_button_ax = self.fig.add_axes([0.65, 0.90, 0.20, 0.05])
        self.save_button = Button(self.save_button_ax, 'Save slice to FITS')
        self.save_button.on_clicked(self.save_fits)
        self.file_status_text = self.fig.text(0.75, 0.875, "", ha='center', va='center')
        self.set_file_status(None)

        self.set_file_status(None)
        self.pv_slice = None

        self.cidpress = self.fig.canvas.mpl_connect('button_press_event', self.click)

    def set_file_status(self, status, filename=None):
        if status == 'instructions':
            self.file_status_text.set_text('Please enter filename in terminal')
            self.file_status_text.set_color('red')
        elif status == 'saved':
            self.file_status_text.set_text('File successfully saved to {0}'.format(filename))
            self.file_status_text.set_color('green')
        else:
            self.file_status_text.set_text('')
            self.file_status_text.set_color('black')
        self.fig.canvas.draw()

    def click(self, event):

        if event.inaxes != self.ax2:
            return

        self.slice_slider.set_val(event.ydata)

    def save_fits(self, *args, **kwargs):

        self.set_file_status('instructions')

        print("Enter filename: ", end='')
        try:
            plot_name = raw_input()
        except NameError:
            plot_name = input()

        if self.pv_slice is None:
            return

        from astropy.io import fits
        self.pv_slice.writeto(plot_name, clobber=True)
        print("Saved file to: ", plot_name)

        self.set_file_status('saved', filename=plot_name)

    def update_pv_slice(self, box):

        path = Path(zip(box.x, box.y))
        path.width = box.width

        self.pv_slice = extract_pv_slice(self.array, path)

        self.ax2.cla()
        self.ax2.imshow(self.pv_slice.data, origin='lower', aspect='auto', interpolation='nearest')

        self.fig.canvas.draw()

    def show(self, block=True):
        import matplotlib.pyplot as plt
        plt.show(block=block)

    def update_slice(self, pos=None):

        if self.array.ndim == 2:
            self.image.set_array(unitless(self.array))
        else:
            self.slice = int(round(pos))
            self.image.set_array(unitless(self.array[self.slice, :, :]))

        self.fig.canvas.draw()

    def update_vmin(self, vmin):
        if vmin > self._clim[1]:
            self._clim = (self._clim[1], self._clim[1])
        else:
            self._clim = (vmin, self._clim[1])
        self.image.set_clim(*self._clim)
        self.fig.canvas.draw()

    def update_vmax(self, vmax):
        if vmax < self._clim[0]:
            self._clim = (self._clim[0], self._clim[0])
        else:
            self._clim = (self._clim[0], vmax)
        self.image.set_clim(*self._clim)
        self.fig.canvas.draw()
<|endoftext|>


<commit_before>import logging

from models import Dictionary, MessageWord, Word, MessageTopic, TweetWord, PrecalcCategoricalDistribution, TweetWordMessageConnection
from msgvis.apps.corpus.models import Dataset, Message
import codecs
import re
from time import time
import subprocess
import os
import glob
from nltk.stem import WordNetLemmatizer
import sys

logger = logging.getLogger(__name__)

__all__ = ['get_twitter_context', 'get_chat_context', 'Dictionary']

_stoplist = None

def get_stoplist():
    global _stoplist
    if not _stoplist:
        from nltk.corpus import stopwords

        _stoplist = stopwords.words('english')
    return _stoplist

class DbTextIterator(object):
    def __init__(self, queryset):
        self.queryset = queryset
        self.current_position = 0
        self.current = None

    def __iter__(self):
        self.current_position = 0
        for msg in self.queryset.iterator():
            self.current = msg
            self.current_position += 1
            if self.current_position % 10000 == 0:
                logger.info("Iterating through database texts: item %d" % self.current_position)

            #yield msg.text
            yield msg

class DbWordVectorIterator(object):
    def __init__(self, dictionary, freq_field='tfidf'):
        self.dictionary = dictionary
        self.freq_field = freq_field
        self.current_message_id = None
        self.current_vector = None

    def __iter__(self):
        qset = MessageWord.objects.filter(dictionary=self.dictionary).order_by('message')
        self.current_message_id = None
        self.current_vector = []
        current_position = 0
        for mw in qset.iterator():
            message_id = mw.message_id
            word_idx = mw.word_index
            freq = getattr(mw, self.freq_field)

            if self.current_message_id is None:
                self.current_message_id = message_id
                self.current_vector = []

            if self.current_message_id != message_id:
                yield self.current_vector
                self.current_vector = []
                self.current_message_id = message_id
                current_position += 1

                if current_position % 10000 == 0:
                    logger.info("Iterating through database word-vectors: item %d" % current_position)

            self.current_vector.append((word_idx, freq))

        # one more extra one
        yield self.current_vector

    def __len__(self):
        from django.db.models import Count

        count = MessageWord.objects \
            .filter(dictionary=self.dictionary) \
            .aggregate(Count('message', distinct=True))

        if count:
            return count['message__count']

class Tokenizer(object):
    def __init__(self, texts, lemmatizer, *filters):
        """
        Filters is a list of objects which can be used like sets
        to determine if a word should be removed: if word in filter, then word will
        be ignored.
        """
        self.texts = texts
        self.lemmatizer = lemmatizer
        self.filters = filters
        self.max_length = Word._meta.get_field('text').max_length

    def __iter__(self):
        if self.texts is None:
            raise RuntimeError("Tokenizer can only iterate if given texts")

        for text in self.texts:
            tokenized_words = self.tokenize(text)
            if self.lemmatizer:
                tokenized_words = map(lambda x: self.lemmatizer.lemmatize(x), tokenized_words)
            yield tokenized_words

    def tokenize(self, text):
        words = []

        for word in self.split(text.lower()):

            if self.lemmatizer:
                word = self.lemmatizer.lemmatize(word)

            filter_out = False
            for f in self.filters:
                if word in f:
                    filter_out = True
                    break

            if filter_out:
                # skip this word
                continue

            if len(word) >= self.max_length:
                word = word[:self.max_length - 1]

            words.append(word)

        return words

    def split(self, text):
        return text.split()

class WordTokenizer(Tokenizer):
    def __init__(self, *args, **kwargs):
        super(WordTokenizer, self).__init__(*args, **kwargs)

        import nltk

        self._tokenize = nltk.word_tokenize

    def split(self, text):
        return self._tokenize(text)

class SimpleTokenizer(Tokenizer):
    def __init__(self, *args, **kwargs):
        super(SimpleTokenizer, self).__init__(*args, **kwargs)

        from nltk.tokenize import sent_tokenize, wordpunct_tokenize
        self._sent_tokenize = sent_tokenize
        self._tokenize = wordpunct_tokenize

        import re
        self._strip_punct = re.compile(r'[^\w\s]')

    def split(self, text):
        # split into sentence, then remove all punctuation
        sents = (self._strip_punct.sub('', sent) for sent in self._sent_tokenize(text))
        # then split on non-words
        return [token for sent in sents for token in self._tokenize(sent)]

class TweetParserTokenizer(Tokenizer):
    def __init__(self, *args, **kwargs):
        super(TweetParserTokenizer, self).__init__(*args, **kwargs)

    def tokenize(self, message):

        words = []

        for word in message.tweet_words.all():
            word = word.text

            filter_out = False
            for f in self.filters:
                if word in f:
                    filter_out = True
                    break

            if filter_out:
                # skip this word
                continue

            if len(word) >= self.max_length:
                word = word[:self.max_length - 1]

            words.append(word)

        return words

    def split(self, message):
        return self.tokenize(message)

class TopicContext(object):
    def __init__(self, name, queryset, tokenizer, lemmatizer, filters, minimum_frequency=4):
        self.name = name
        self.queryset = queryset
        self.tokenizer = tokenizer
        self.lemmatizer = lemmatizer
        self.filters = filters
        self.minimum_frequency = minimum_frequency

    def queryset_str(self):
        return str(self.queryset.query)

    def get_dict_settings(self):
        settings = dict(
            name=self.name,
            tokenizer=self.tokenizer.__name__,
            dataset=self.queryset_str(),
            filters=repr(self.filters),
            minimum_frequency=self.minimum_frequency
        )

        import json

        return json.dumps(settings, sort_keys=True)

    # This emthod doesn't work in a different environment as the instances
    # of filters have a different string representation
    def find_dictionary(self):
        results = Dictionary.objects.filter(settings=self.get_dict_settings())
        return results.last()

    def find_dictionary_by_name_dataset(self, name, dataset_id):
        results = Dictionary.objects.filter(name=name, dataset_id=dataset_id)
        return results.last()

    def build_dictionary(self, dataset_id):
        texts = DbTextIterator(self.queryset)

        tokenized_texts = self.tokenizer(texts, self.lemmatizer, *self.filters)
        dataset = Dataset.objects.get(pk=dataset_id)
        return Dictionary._create_from_texts(tokenized_texts=tokenized_texts,
                                             name=self.name,
                                             minimum_frequency=self.minimum_frequency,
                                             dataset=dataset,
                                             settings=self.get_dict_settings())

    def build_features(self, dictionary):
        texts = DbTextIterator(self.queryset)
        tokenized_texts = self.tokenizer(texts, self.lemmatizer, *self.filters)
        tokenizer = self.tokenizer(texts, self.lemmatizer, *self.filters)

        dictionary._create_features_from_texts(dict_model=dictionary,
                                             tokenized_texts=tokenized_texts,
                                             name=self.name,
                                             minimum_frequency=self.minimum_frequency,
                                             queryset=self.queryset)

    def bows_exist(self, dictionary):
        return MessageWord.objects.filter(dictionary=dictionary).exists()

    def build_bows(self, dictionary):
        texts = DbTextIterator(self.queryset)
        tokenized_texts = self.tokenizer(texts, self.lemmatizer, *self.filters)

        dictionary._vectorize_corpus(queryset=self.queryset,
                                     tokenizer=tokenized_texts)

    def build_lda(self, dictionary, num_topics=30, **kwargs):
        corpus = DbWordVectorIterator(dictionary)
        return dictionary._build_lda(self.name, corpus, num_topics=num_topics, **kwargs)

    def apply_lda(self, dictionary, model, lda=None):
        corpus = DbWordVectorIterator(dictionary)
        return dictionary._apply_lda(model, corpus, lda=lda)

    def evaluate_lda(self, dictionary, model, lda=None):
        corpus = DbWordVectorIterator(dictionary)
        return dictionary._evaluate_lda(model, corpus, lda=lda)

class LambdaWordFilter(object):
    def __init__(self, fn):
        self.fn = fn

    def __contains__(self, item):
        return self.fn(item)

def standard_features_pipeline(context, name, dataset_id):

    dictionary = context.find_dictionary_by_name_dataset(name, dataset_id)
    print dictionary
    context.build_features(dictionary)

def standard_topic_pipeline(context, dataset_id, num_topics, **kwargs):
    dictionary = context.find_dictionary()
    if dictionary is None:
        dictionary = context.build_dictionary(dataset_id=dataset_id)

    if not context.bows_exist(dictionary):
        context.build_bows(dictionary)

    model, lda = context.build_lda(dictionary, num_topics=num_topics, **kwargs)
    context.apply_lda(dictionary, model, lda)
    context.evaluate_lda(dictionary, model, lda)

def default_topic_context(name, dataset_id):
    dataset = Dataset.objects.get(pk=dataset_id)
    queryset = dataset.message_set.all()#filter(language__code='en')

    filters = [
        set(get_stoplist()),
        ['ive', 'wasnt', 'didnt', 'dont'],
        LambdaWordFilter(lambda word: word == 'rt' or len(word) <= 2),
        LambdaWordFilter(lambda word: word.startswith('http') and len(word) > 4)
    ]

    return TopicContext(name=name, queryset=queryset,
                        tokenizer=TweetParserTokenizer,
                        lemmatizer=None,#WordNetLemmatizer(),
                        filters=filters,
                        minimum_frequency=4)

def import_from_tweet_parser_results(dataset_id, filename):
    current_msg_id = -1
    current_msg = None
    word_list = []
    count = 0
    order = 0
    with codecs.open(filename, encoding='utf-8', mode='r') as f:
        print "Reading file %s" % filename

        start = time()
        for line in f:
            if re.search("ID=(\d+)", line):
                # save the previous word list
                if len(word_list) > 0:
                    TweetWordMessageConnection.objects.bulk_create(word_list)
                    word_list = []
                    count += 1
                    order = 0
                    if count % 1000 == 0:
                        print "Processed %d messages" % count
                        print "Time: %.2fs" % (time() - start)
                        start = time()

                results = re.match('ID=(\d+)', line)
                groups = results.groups()
                current_msg_id = int(groups[0])
                #print "current msg id = %d" %(current_msg_id)
                current_msg = Message.objects.get(id=current_msg_id)

            elif re.search("(.+)\t(.+)\t(.+)", line):
                results = re.match('(.+)\t(.+)\t(.+)', line)
                groups = results.groups()
                original_text = groups[0]
                pos = groups[1]
                text = groups[2]
                if re.search('[,~U]', pos):
                    continue
                else:
                    word_obj, created = TweetWord.objects.get_or_create(dataset_id=dataset_id, original_text=original_text, pos=pos, text=text)

                    order += 1
                    word_list.append(TweetWordMessageConnection(message=current_msg, tweet_word=word_obj, order=order))
        # save the previous word list
        if len(word_list) > 0:
            TweetWordMessageConnection.objects.bulk_create(word_list)
            count += 1
            word_list = []
        print "Processed %d messages" % count
        print "Time: %.2fs" % (time() - start)

def dump_tweets(dataset_id, save_path):
    dataset = Dataset.objects.get(id=dataset_id)
    total_count = dataset.message_set.count()
    messages = dataset.message_set.all()#.exclude(time__isnull=True)

    start = 0
    limit = 10000

    while start < total_count:
        filename = "%s/dataset_%d_message_%d_%d.txt" %(save_path, dataset_id, start, start + limit)
        with codecs.open(filename, encoding='utf-8', mode='w') as f:
            for msg in messages[start:start + limit]:

                try:
                    tweet_id = msg.id
                    #full_name = msg.sender.full_name.lower() if msg.sender.full_name is not None else ""
                    #username = msg.sender.username.lower() if msg.sender.username is not None else ""
                    text = msg.text.lower()

                    line = "TWEETID%dSTART\n" %(tweet_id)
                    f.write(line)

                    #line = "%s @%s" %(full_name, username)
                    #f.write(line)

                    line = "%s\n" %(text)
                    f.write(line)

                    line = "TWEETID%dEND\n" %(tweet_id)
                    f.write(line)

                except:
                    pass

        start += limit

def parse_tweets(tweet_parser_path, input_path, output_path):
    parser_cmd = "%s/runTagger.sh" %tweet_parser_path
    input_files = glob.glob("%s/dataset_*.txt" % input_path)

    for input_file in input_files:
        results = re.search('(dataset_.+)\.txt', input_file)
        filename = results.groups()[0]

        output_file = "%s/%s.out" % (output_path, filename)

        with codecs.open(output_file, encoding='utf-8', mode='w') as f:
            cmd = "%s --output-format conll %s" %(parser_cmd, input_file)
            print cmd
            try:
                subprocess.call(cmd.split(" "), stdout=f, stderr=subprocess.PIPE)
            except:
                pass

def lemmatize_tweets(input_path, output_path):
    wordnet_lemmatizer = WordNetLemmatizer()

    input_files = glob.glob("%s/dataset_*.out" % input_path)

    for input_file in input_files:
        results = re.search('(dataset_.+)\.out', input_file)
        filename = results.groups()[0]

        output_file = "%s/%s_converted.out" % (output_path, filename)
        output_file2 = "%s/%s_converted.out.id" % (output_path, filename)

        with codecs.open(output_file, encoding='utf-8', mode='w') as out:
            with codecs.open(output_file2, encoding='utf-8', mode='w') as out2:
                print >>out, "<doc>"
                with codecs.open(input_file, encoding='utf-8', mode='r') as f:
                    for line in f:
                        if re.search('TWEETID(\d+)START', line):
                            results = re.match('TWEETID(\d+)START', line)
                            groups = results.groups()
                            print >>out, "<p>"
                            print >>out2, "ID=%d" %(int(groups[0]))

                        elif re.search('TWEETID(\d+)END', line):
                            print >>out, "<\p>"
                        elif re.search("(.+)\t(.+)\t(.+)\n", line):
                            results = re.match("(.+)\t(.+)\t(.+)\n", line)
                            groups = results.groups()
                            word = groups[0]
                            pos = groups[1]
                            lemma = wordnet_lemmatizer.lemmatize(word)
                            print >>out, "%s\t%s\t%s" %(word, pos, lemma)
                            print >>out2, "%s\t%s\t%s" %(word, pos, lemma)

                print >>out, "</doc>"
<commit_msg>get rid of useless model<commit_after>import logging

from models import Dictionary, MessageWord, Word, MessageTopic, TweetWord, TweetWordMessageConnection
from msgvis.apps.corpus.models import Dataset, Message
import codecs
import re
from time import time
import subprocess
import os
import glob
from nltk.stem import WordNetLemmatizer
import sys

logger = logging.getLogger(__name__)

__all__ = ['get_twitter_context', 'get_chat_context', 'Dictionary']

_stoplist = None

def get_stoplist():
    global _stoplist
    if not _stoplist:
        from nltk.corpus import stopwords

        _stoplist = stopwords.words('english')
    return _stoplist

class DbTextIterator(object):
    def __init__(self, queryset):
        self.queryset = queryset
        self.current_position = 0
        self.current = None

    def __iter__(self):
        self.current_position = 0
        for msg in self.queryset.iterator():
            self.current = msg
            self.current_position += 1
            if self.current_position % 10000 == 0:
                logger.info("Iterating through database texts: item %d" % self.current_position)

            #yield msg.text
            yield msg

class DbWordVectorIterator(object):
    def __init__(self, dictionary, freq_field='tfidf'):
        self.dictionary = dictionary
        self.freq_field = freq_field
        self.current_message_id = None
        self.current_vector = None

    def __iter__(self):
        qset = MessageWord.objects.filter(dictionary=self.dictionary).order_by('message')
        self.current_message_id = None
        self.current_vector = []
        current_position = 0
        for mw in qset.iterator():
            message_id = mw.message_id
            word_idx = mw.word_index
            freq = getattr(mw, self.freq_field)

            if self.current_message_id is None:
                self.current_message_id = message_id
                self.current_vector = []

            if self.current_message_id != message_id:
                yield self.current_vector
                self.current_vector = []
                self.current_message_id = message_id
                current_position += 1

                if current_position % 10000 == 0:
                    logger.info("Iterating through database word-vectors: item %d" % current_position)

            self.current_vector.append((word_idx, freq))

        # one more extra one
        yield self.current_vector

    def __len__(self):
        from django.db.models import Count

        count = MessageWord.objects \
            .filter(dictionary=self.dictionary) \
            .aggregate(Count('message', distinct=True))

        if count:
            return count['message__count']

class Tokenizer(object):
    def __init__(self, texts, lemmatizer, *filters):
        """
        Filters is a list of objects which can be used like sets
        to determine if a word should be removed: if word in filter, then word will
        be ignored.
        """
        self.texts = texts
        self.lemmatizer = lemmatizer
        self.filters = filters
        self.max_length = Word._meta.get_field('text').max_length

    def __iter__(self):
        if self.texts is None:
            raise RuntimeError("Tokenizer can only iterate if given texts")

        for text in self.texts:
            tokenized_words = self.tokenize(text)
            if self.lemmatizer:
                tokenized_words = map(lambda x: self.lemmatizer.lemmatize(x), tokenized_words)
            yield tokenized_words

    def tokenize(self, text):
        words = []

        for word in self.split(text.lower()):

            if self.lemmatizer:
                word = self.lemmatizer.lemmatize(word)

            filter_out = False
            for f in self.filters:
                if word in f:
                    filter_out = True
                    break

            if filter_out:
                # skip this word
                continue

            if len(word) >= self.max_length:
                word = word[:self.max_length - 1]

            words.append(word)

        return words

    def split(self, text):
        return text.split()

class WordTokenizer(Tokenizer):
    def __init__(self, *args, **kwargs):
        super(WordTokenizer, self).__init__(*args, **kwargs)

        import nltk

        self._tokenize = nltk.word_tokenize

    def split(self, text):
        return self._tokenize(text)

class SimpleTokenizer(Tokenizer):
    def __init__(self, *args, **kwargs):
        super(SimpleTokenizer, self).__init__(*args, **kwargs)

        from nltk.tokenize import sent_tokenize, wordpunct_tokenize
        self._sent_tokenize = sent_tokenize
        self._tokenize = wordpunct_tokenize

        import re
        self._strip_punct = re.compile(r'[^\w\s]')

    def split(self, text):
        # split into sentence, then remove all punctuation
        sents = (self._strip_punct.sub('', sent) for sent in self._sent_tokenize(text))
        # then split on non-words
        return [token for sent in sents for token in self._tokenize(sent)]

class TweetParserTokenizer(Tokenizer):
    def __init__(self, *args, **kwargs):
        super(TweetParserTokenizer, self).__init__(*args, **kwargs)

    def tokenize(self, message):

        words = []

        for word in message.tweet_words.all():
            word = word.text

            filter_out = False
            for f in self.filters:
                if word in f:
                    filter_out = True
                    break

            if filter_out:
                # skip this word
                continue

            if len(word) >= self.max_length:
                word = word[:self.max_length - 1]

            words.append(word)

        return words

    def split(self, message):
        return self.tokenize(message)

class TopicContext(object):
    def __init__(self, name, queryset, tokenizer, lemmatizer, filters, minimum_frequency=4):
        self.name = name
        self.queryset = queryset
        self.tokenizer = tokenizer
        self.lemmatizer = lemmatizer
        self.filters = filters
        self.minimum_frequency = minimum_frequency

    def queryset_str(self):
        return str(self.queryset.query)

    def get_dict_settings(self):
        settings = dict(
            name=self.name,
            tokenizer=self.tokenizer.__name__,
            dataset=self.queryset_str(),
            filters=repr(self.filters),
            minimum_frequency=self.minimum_frequency
        )

        import json

        return json.dumps(settings, sort_keys=True)

    # This emthod doesn't work in a different environment as the instances
    # of filters have a different string representation
    def find_dictionary(self):
        results = Dictionary.objects.filter(settings=self.get_dict_settings())
        return results.last()

    def find_dictionary_by_name_dataset(self, name, dataset_id):
        results = Dictionary.objects.filter(name=name, dataset_id=dataset_id)
        return results.last()

    def build_dictionary(self, dataset_id):
        texts = DbTextIterator(self.queryset)

        tokenized_texts = self.tokenizer(texts, self.lemmatizer, *self.filters)
        dataset = Dataset.objects.get(pk=dataset_id)
        return Dictionary._create_from_texts(tokenized_texts=tokenized_texts,
                                             name=self.name,
                                             minimum_frequency=self.minimum_frequency,
                                             dataset=dataset,
                                             settings=self.get_dict_settings())

    def build_features(self, dictionary):
        texts = DbTextIterator(self.queryset)
        tokenized_texts = self.tokenizer(texts, self.lemmatizer, *self.filters)
        tokenizer = self.tokenizer(texts, self.lemmatizer, *self.filters)

        dictionary._create_features_from_texts(dict_model=dictionary,
                                             tokenized_texts=tokenized_texts,
                                             name=self.name,
                                             minimum_frequency=self.minimum_frequency,
                                             queryset=self.queryset)

    def bows_exist(self, dictionary):
        return MessageWord.objects.filter(dictionary=dictionary).exists()

    def build_bows(self, dictionary):
        texts = DbTextIterator(self.queryset)
        tokenized_texts = self.tokenizer(texts, self.lemmatizer, *self.filters)

        dictionary._vectorize_corpus(queryset=self.queryset,
                                     tokenizer=tokenized_texts)

    def build_lda(self, dictionary, num_topics=30, **kwargs):
        corpus = DbWordVectorIterator(dictionary)
        return dictionary._build_lda(self.name, corpus, num_topics=num_topics, **kwargs)

    def apply_lda(self, dictionary, model, lda=None):
        corpus = DbWordVectorIterator(dictionary)
        return dictionary._apply_lda(model, corpus, lda=lda)

    def evaluate_lda(self, dictionary, model, lda=None):
        corpus = DbWordVectorIterator(dictionary)
        return dictionary._evaluate_lda(model, corpus, lda=lda)

class LambdaWordFilter(object):
    def __init__(self, fn):
        self.fn = fn

    def __contains__(self, item):
        return self.fn(item)

def standard_features_pipeline(context, name, dataset_id):

    dictionary = context.find_dictionary_by_name_dataset(name, dataset_id)
    print dictionary
    context.build_features(dictionary)

def standard_topic_pipeline(context, dataset_id, num_topics, **kwargs):
    dictionary = context.find_dictionary()
    if dictionary is None:
        dictionary = context.build_dictionary(dataset_id=dataset_id)

    if not context.bows_exist(dictionary):
        context.build_bows(dictionary)

    model, lda = context.build_lda(dictionary, num_topics=num_topics, **kwargs)
    context.apply_lda(dictionary, model, lda)
    context.evaluate_lda(dictionary, model, lda)

def default_topic_context(name, dataset_id):
    dataset = Dataset.objects.get(pk=dataset_id)
    queryset = dataset.message_set.all()#filter(language__code='en')

    filters = [
        set(get_stoplist()),
        ['ive', 'wasnt', 'didnt', 'dont'],
        LambdaWordFilter(lambda word: word == 'rt' or len(word) <= 2),
        LambdaWordFilter(lambda word: word.startswith('http') and len(word) > 4)
    ]

    return TopicContext(name=name, queryset=queryset,
                        tokenizer=TweetParserTokenizer,
                        lemmatizer=None,#WordNetLemmatizer(),
                        filters=filters,
                        minimum_frequency=4)

def import_from_tweet_parser_results(dataset_id, filename):
    current_msg_id = -1
    current_msg = None
    word_list = []
    count = 0
    order = 0
    with codecs.open(filename, encoding='utf-8', mode='r') as f:
        print "Reading file %s" % filename

        start = time()
        for line in f:
            if re.search("ID=(\d+)", line):
                # save the previous word list
                if len(word_list) > 0:
                    TweetWordMessageConnection.objects.bulk_create(word_list)
                    word_list = []
                    count += 1
                    order = 0
                    if count % 1000 == 0:
                        print "Processed %d messages" % count
                        print "Time: %.2fs" % (time() - start)
                        start = time()

                results = re.match('ID=(\d+)', line)
                groups = results.groups()
                current_msg_id = int(groups[0])
                #print "current msg id = %d" %(current_msg_id)
                current_msg = Message.objects.get(id=current_msg_id)

            elif re.search("(.+)\t(.+)\t(.+)", line):
                results = re.match('(.+)\t(.+)\t(.+)', line)
                groups = results.groups()
                original_text = groups[0]
                pos = groups[1]
                text = groups[2]
                if re.search('[,~U]', pos):
                    continue
                else:
                    word_obj, created = TweetWord.objects.get_or_create(dataset_id=dataset_id, original_text=original_text, pos=pos, text=text)

                    order += 1
                    word_list.append(TweetWordMessageConnection(message=current_msg, tweet_word=word_obj, order=order))
        # save the previous word list
        if len(word_list) > 0:
            TweetWordMessageConnection.objects.bulk_create(word_list)
            count += 1
            word_list = []
        print "Processed %d messages" % count
        print "Time: %.2fs" % (time() - start)

def dump_tweets(dataset_id, save_path):
    dataset = Dataset.objects.get(id=dataset_id)
    total_count = dataset.message_set.count()
    messages = dataset.message_set.all()#.exclude(time__isnull=True)

    start = 0
    limit = 10000

    while start < total_count:
        filename = "%s/dataset_%d_message_%d_%d.txt" %(save_path, dataset_id, start, start + limit)
        with codecs.open(filename, encoding='utf-8', mode='w') as f:
            for msg in messages[start:start + limit]:

                try:
                    tweet_id = msg.id
                    #full_name = msg.sender.full_name.lower() if msg.sender.full_name is not None else ""
                    #username = msg.sender.username.lower() if msg.sender.username is not None else ""
                    text = msg.text.lower()

                    line = "TWEETID%dSTART\n" %(tweet_id)
                    f.write(line)

                    #line = "%s @%s" %(full_name, username)
                    #f.write(line)

                    line = "%s\n" %(text)
                    f.write(line)

                    line = "TWEETID%dEND\n" %(tweet_id)
                    f.write(line)

                except:
                    pass

        start += limit

def parse_tweets(tweet_parser_path, input_path, output_path):
    parser_cmd = "%s/runTagger.sh" %tweet_parser_path
    input_files = glob.glob("%s/dataset_*.txt" % input_path)

    for input_file in input_files:
        results = re.search('(dataset_.+)\.txt', input_file)
        filename = results.groups()[0]

        output_file = "%s/%s.out" % (output_path, filename)

        with codecs.open(output_file, encoding='utf-8', mode='w') as f:
            cmd = "%s --output-format conll %s" %(parser_cmd, input_file)
            print cmd
            try:
                subprocess.call(cmd.split(" "), stdout=f, stderr=subprocess.PIPE)
            except:
                pass

def lemmatize_tweets(input_path, output_path):
    wordnet_lemmatizer = WordNetLemmatizer()

    input_files = glob.glob("%s/dataset_*.out" % input_path)

    for input_file in input_files:
        results = re.search('(dataset_.+)\.out', input_file)
        filename = results.groups()[0]

        output_file = "%s/%s_converted.out" % (output_path, filename)
        output_file2 = "%s/%s_converted.out.id" % (output_path, filename)

        with codecs.open(output_file, encoding='utf-8', mode='w') as out:
            with codecs.open(output_file2, encoding='utf-8', mode='w') as out2:
                print >>out, "<doc>"
                with codecs.open(input_file, encoding='utf-8', mode='r') as f:
                    for line in f:
                        if re.search('TWEETID(\d+)START', line):
                            results = re.match('TWEETID(\d+)START', line)
                            groups = results.groups()
                            print >>out, "<p>"
                            print >>out2, "ID=%d" %(int(groups[0]))

                        elif re.search('TWEETID(\d+)END', line):
                            print >>out, "<\p>"
                        elif re.search("(.+)\t(.+)\t(.+)\n", line):
                            results = re.match("(.+)\t(.+)\t(.+)\n", line)
                            groups = results.groups()
                            word = groups[0]
                            pos = groups[1]
                            lemma = wordnet_lemmatizer.lemmatize(word)
                            print >>out, "%s\t%s\t%s" %(word, pos, lemma)
                            print >>out2, "%s\t%s\t%s" %(word, pos, lemma)

                print >>out, "</doc>"
<|endoftext|>


<commit_before>import logging

import click

import guild.app
import guild.cli

guild.app.init()

class Args(object):

    def __init__(self, kw):
        for name in kw:
            setattr(self, name, kw[name])

def append_params(fn, params):
    fn.__click_params__ = getattr(fn, "__click_params__", [])
    fn.__click_params__.extend(reversed(params))

###################################################################
# Main
###################################################################

class CLIGroup(click.Group):

    def get_command(self, ctx, cmd_name):
        if cmd_name in ["operations", "ops"]:
            cmd_name = "operations, ops"
        elif cmd_name in ["tensorboard", "tb"]:
            cmd_name = "tensorboard, tb"
        return super(CLIGroup, self).get_command(ctx, cmd_name)

@click.group(cls=CLIGroup)
@click.version_option(
    version=guild.app.version(),
    prog_name="guild",
    message="%(prog)s %(version)s"
)
@click.option(
    "--info", "log_level",
    help="Log information during command.",
    flag_value=logging.INFO)
@click.option(
    "--debug", "log_level",
    help="Log more information during command.",
    flag_value=logging.DEBUG)

def cli(**kw):
    """Guild AI command line interface."""
    guild.cli.main(Args(kw))

def main():
    guild.cli.apply_main(cli)

###################################################################
# check command
###################################################################

@click.command()
@click.option(
    "-T", "--tests", "all_tests",
    help="Run Guild test suite.",
    is_flag=True)
@click.option(
    "-t", "--test", "tests",
    help="Run TEST (may be used multiple times).",
    metavar="TEST",
    multiple=True)
@click.option(
    "-s", "--no-info",
    help="Don't print info (useful when just running tests).",
    is_flag=True)
@click.option(
    "-v", "--verbose",
    help="Show check details.",
    is_flag=True)

def check(**kw):
    """Check the Guild setup.

    This command performs a number of checks and prints information
    about the Guild setup.

    You can also run the Guild test suite by specifying the --tests
    option.
    """
    import guild.check_cmd
    guild.check_cmd.main(Args(kw))

cli.add_command(check)

###################################################################
# models command
###################################################################

@click.command()
@click.option(
    "-p", "--project", "project_location",
    help="Project location (file system directory) for models.",
    metavar="LOCATION")
@click.option(
    "--installed",
    help="Show available installed packages. --project is ignore.",
    is_flag=True)
@click.option(
    "-v", "--verbose",
    help="Show model details.",
    is_flag=True)
@click.pass_context

def models(ctx, **kw):
    """Show available models.

    By default Guild will show models defined in the current directory
    (in a MODEL or MODELS file). You may use --project to specify an
    alternative project location.

    To show installed models, use the --installed option. Any location
    specified by --project, will be ignored if --installed is used.
    """
    import guild.models_cmd
    guild.models_cmd.main(Args(kw), ctx)

cli.add_command(models)

###################################################################
# operations command
###################################################################

@click.command(name="operations, ops")
@click.argument("model", required=False)
@click.option(
    "-p", "--project", "project_location",
    help="Project location (file system directory) for MODEL.",
    metavar="LOCATION")
@click.option(
    "-v", "--verbose",
    help="Show operation details.",
    is_flag=True)
@click.pass_context

def operations(ctx, **kw):
    """Show model operations.
    """
    import guild.operations_cmd
    guild.operations_cmd.main(Args(kw), ctx)

cli.add_command(operations)

###################################################################
# run command
###################################################################

def run_params(fn):
    append_params(fn, [
        click.Argument(("args",), metavar="[ARG...]", nargs=-1),
        click.Option(
            ("-p", "--project", "project_location"),
            help="Project location (file system directory) for MODEL.",
            metavar="LOCATION"),
        click.Option(
            ("--disable-plugins",),
            help="A comma separated list of plugin names to disable."),
        click.Option(
            ("-y", "--yes"),
            help="Do not prompt before running operation.",
            is_flag=True),
        click.Option(
            ("--print-command",),
            help="Show the operation command and exit (does not run).",
            is_flag=True),
        click.Option(
            ("--print-env",),
            help="Show the operation environment and exit (does not run).",
            is_flag=True),
    ])
    return fn

@click.command()
@click.argument("opspec", metavar="[MODEL:]OPERATION")
@run_params

def run(**kw):
    """Run a model operation.

    By default Guild will try to run OPERATION for the default model
    defined in a project. If a project location is not specified (see
    --project option below), Guild looks for a project in the current
    directory.

    If MODEL is specified, Guild will use it instead of the default
    model defined in a project.
    """
    import guild.run_cmd
    guild.run_cmd.main(Args(kw))

cli.add_command(run)

###################################################################
# runs command
###################################################################

class RunsGroup(click.Group):

    def get_command(self, ctx, cmd_name):
        if cmd_name in ["delete", "rm"]:
            cmd_name = "delete, rm"
        elif cmd_name in ["list", "ls"]:
            cmd_name = "list, ls"
        return super(RunsGroup, self).get_command(ctx, cmd_name)

def run_scope_options(fn):
    append_params(fn, [
        click.Option(
            ("-p", "--project", "project_location"),
            help=("Project location (file system directory) for filtering "
                  "runs."),
            metavar="LOCATION"),
        click.Option(
            ("-S", "--system"),
            help=("Include system wide runs rather than limit to runs "
                  "associated with a project location. Ignores LOCATION."),
            is_flag=True)
    ])
    return fn

def run_filters(fn):
    append_params(fn, [
        click.Option(
            ("-r", "--running", "status"),
            help="Include only runs that are still running.",
            flag_value="running"),
        click.Option(
            ("-c", "--completed", "status"),
            help="Include only completed runs.",
            flag_value="completed"),
        click.Option(
            ("-s", "--stopped", "status"),
            help=("Include only runs that exited with an error or were "
                  "terminated by the user."),
            flag_value="stopped"),
        click.Option(
            ("-e", "--error", "status"),
            help="Include only runs that exited with an error.",
            flag_value="error"),
        click.Option(
            ("-t", "--terminated", "status"),
            help="Include only runs terminated by the user.",
            flag_value="terminated"),
    ])
    return fn

def runs_list_options(fn):
    run_scope_options(fn)
    run_filters(fn)
    append_params(fn, [
        click.Option(
            ("-v", "--verbose"),
            help="Show run details.",
            is_flag=True),
        click.Option(
            ("-d", "--deleted"),
            help="Show deleted runs.",
            is_flag=True),
    ])
    return fn

@click.group(invoke_without_command=True, cls=RunsGroup)
@runs_list_options
@click.pass_context

def runs(ctx, **kw):
    """Show or manage runs.

    If COMMAND is omitted, lists run. Refer to 'guild runs list
    --help' for more information on the list command.
    """
    if not ctx.invoked_subcommand:
        ctx.invoke(list_runs, **kw)

cli.add_command(runs)

###################################################################
# runs list command
###################################################################

@click.command("list, ls")
@runs_list_options
@click.pass_context

def list_runs(ctx, **kw):
    """List runs.

    By default lists runs associated with models defined in the
    current directory, or LOCATION if specified. To list all runs, use
    the --system option.

    To list deleted runs, use the --deleted option. Note that runs are
    still limited to the specified project unless --system is
    specified.

    You may apply any of the filter options below to limit the runs
    listed.
    """
    import guild.runs_cmd
    guild.runs_cmd.list_runs(Args(kw), ctx)

runs.add_command(list_runs)

###################################################################
# runs delete command
###################################################################

RUN_ARG_HELP = """
RUN may be a run ID (or the unique start of a run ID) or a zero-based
index corresponding to a run returned by the list command. Indexes may
also be specified in ranges in the form START:END where START is the
start index and END is the end index. Either START or END may be
omitted. If START is omitted, all runs up to END are selected. If END
id omitted, all runs from START on are selected. If both START and END
are omitted (i.e. the ':' char is used by itself) all runs are selected.
"""

@click.command("delete, rm", help="""
Delete one or more runs.

%s
""" % RUN_ARG_HELP)
@click.argument("runs", metavar="RUN [RUN...]", nargs=-1, required=True)
@run_scope_options
@run_filters
@click.option(
    "-y", "--yes",
    help="Do not prompt before deleting.",
    is_flag=True)
@click.pass_context

def delete_runs(ctx, **kw):
    # Help defined in command decorator.
    import guild.runs_cmd
    guild.runs_cmd.delete_runs(Args(kw), ctx)

runs.add_command(delete_runs)

###################################################################
# runs restore command
###################################################################

@click.command("restore", help="""
Restores one or more deleted runs.

%s
""" % RUN_ARG_HELP)

@click.argument("runs", metavar="RUN [RUN...]", nargs=-1, required=True)
@run_scope_options
@run_filters
@click.option(
    "-y", "--yes",
    help="Do not prompt before restoring.",
    is_flag=True)
@click.pass_context

def restore_runs(ctx, **kw):
    # Help defined in command decorator.
    import guild.runs_cmd
    guild.runs_cmd.restore_runs(Args(kw), ctx)

runs.add_command(restore_runs)

###################################################################
# runs info command
###################################################################

@click.command("info")
@click.argument("run", required=False)
@run_scope_options
@run_filters
@click.option("--files", help="Include run files", is_flag=True)
@click.pass_context

def run_info(ctx, **kw):
    """Show run details.

    RUN must be a run ID (or the start of a run ID that uniquely
    identifies a run) or a zero-based index corresponding to the run
    as it appears in the list of filtered runs.

    By default the latest run is selected (index 0).

    EXAMPLES

    Show info for the latest run in the current project:

        guild runs info

    Show info for the latest run system wide:

        guild runs info -S

    Show info for the latest completed run in the current project:

        guild runs info -c

    Show info for run a64b1710:

        guild runs info a64b1710

    """
    import guild.runs_cmd
    guild.runs_cmd.run_info(Args(kw), ctx)

runs.add_command(run_info)

###################################################################
# shell command
###################################################################

@click.command()

def shell(**kw):
    """Start a Python shell for API experimentation.
    """
    import guild.shell_cmd
    guild.shell_cmd.main(Args(kw))

cli.add_command(shell)

###################################################################
# sources command
###################################################################

class SourcesGroup(click.Group):

    def get_command(self, ctx, cmd_name):
        if cmd_name in ["rm", "remove"]:
            cmd_name = "remove, rm"
        return super(SourcesGroup, self).get_command(ctx, cmd_name)

@click.group(invoke_without_command=True, cls=SourcesGroup)
@click.pass_context

def sources(ctx, **kw):
    """Show or manage package sources.
    """
    if not ctx.invoked_subcommand:
        import guild.sources_cmd
        guild.sources_cmd.list_sources(Args(kw))

cli.add_command(sources)

###################################################################
# sources add command
###################################################################

@click.command("add")
@click.argument("name")
@click.argument("source")
@click.pass_context

def add_source(ctx, **kw):
    """Add a package source.

    NAME must be a unique source identifier for this system. Use
    'guild sources' to show a list of configured package sources.

    SOURCE must be a GitHub repository URL or a local
    directory. GitHub repository URLs may be provided in shorthand
    using ACCOUNT/REPOSITORY. For example, the SOURCE
    'guildai/guild-packages' will be interpreted as
    'https://github.com/guildai/guild-packages'.

    Local directories must be absolute paths. Use a local directory
    will avoid having to sync with an upstream source each time a
    change is made to a package. This is useful for package
    development.
    """
    import guild.sources_cmd
    guild.sources_cmd.add_source(Args(kw), ctx)

sources.add_command(add_source)

###################################################################
# sources remove command
###################################################################

@click.command("remove, rm")
@click.argument("name")

def remove_source(**kw):
    """Remove a package source.

    Use 'guild sources' for a list of package sources.
    """
    import guild.sources_cmd
    guild.sources_cmd.remove_source(Args(kw))

sources.add_command(remove_source)

###################################################################
# train command
###################################################################

@click.command("train")
@click.argument("model", required=False)
@run_params

def train(**kw):
    """Train a model.

    Equivalent to running 'guild run [MODEL:]train [ARG...]'.

    By default MODEL is the default model for project in LOCATION.

    You may omit MODEL (i.e. for training the default model) while
    providing one or more ARG values provided the first ARG value
    contains an equals sign ('='). When specifying a switch (i.e. an
    argument that doesn't accept a value) as the first ARG, you must
    provide MODEL.

    Refer to help for the run command ('guild run --help') for more
    information.

    """
    import guild.run_cmd
    args = Args(kw)
    # MODEL is treated as an ARG if it contains an equal sign (see
    # help text above).
    if args.model and "=" in args.model:
        args.args = (args.model,) + args.args
        args.model = None
    args.opspec = "%s:train" % args.model if args.model else "train"
    guild.run_cmd.main(args)

cli.add_command(train)

###################################################################
# tensorboard command
###################################################################

@click.command("tensorboard, tb")
@run_scope_options
@click.option(
    "--host",
    help="Name of host interface to listen on.")
@click.option(
    "--port",
    help="Port to listen on.",
    type=click.IntRange(0, 65535))
@click.option(
    "--refresh-interval",
    help="TensorBoard refresh interval (defaults to 5 seconds).",
    type=click.IntRange(1, None),
    default=5)
@click.option(
    "-n", "--no-open",
    help="Don't open the TensorBoard URL in a brower.",
    is_flag=True)
@click.pass_context

def tensorboard(ctx, **kw):
    """Start TensorBoard to view runs.
    """
    import guild.tensorboard_cmd
    guild.tensorboard_cmd.main(Args(kw), ctx)

cli.add_command(tensorboard)
<commit_msg>Help fix<commit_after>import logging

import click

import guild.app
import guild.cli

guild.app.init()

class Args(object):

    def __init__(self, kw):
        for name in kw:
            setattr(self, name, kw[name])

def append_params(fn, params):
    fn.__click_params__ = getattr(fn, "__click_params__", [])
    fn.__click_params__.extend(reversed(params))

###################################################################
# Main
###################################################################

class CLIGroup(click.Group):

    def get_command(self, ctx, cmd_name):
        if cmd_name in ["operations", "ops"]:
            cmd_name = "operations, ops"
        elif cmd_name in ["tensorboard", "tb"]:
            cmd_name = "tensorboard, tb"
        return super(CLIGroup, self).get_command(ctx, cmd_name)

@click.group(cls=CLIGroup)
@click.version_option(
    version=guild.app.version(),
    prog_name="guild",
    message="%(prog)s %(version)s"
)
@click.option(
    "--info", "log_level",
    help="Log information during command.",
    flag_value=logging.INFO)
@click.option(
    "--debug", "log_level",
    help="Log more information during command.",
    flag_value=logging.DEBUG)

def cli(**kw):
    """Guild AI command line interface."""
    guild.cli.main(Args(kw))

def main():
    guild.cli.apply_main(cli)

###################################################################
# check command
###################################################################

@click.command()
@click.option(
    "-T", "--tests", "all_tests",
    help="Run Guild test suite.",
    is_flag=True)
@click.option(
    "-t", "--test", "tests",
    help="Run TEST (may be used multiple times).",
    metavar="TEST",
    multiple=True)
@click.option(
    "-s", "--no-info",
    help="Don't print info (useful when just running tests).",
    is_flag=True)
@click.option(
    "-v", "--verbose",
    help="Show check details.",
    is_flag=True)

def check(**kw):
    """Check the Guild setup.

    This command performs a number of checks and prints information
    about the Guild setup.

    You can also run the Guild test suite by specifying the --tests
    option.
    """
    import guild.check_cmd
    guild.check_cmd.main(Args(kw))

cli.add_command(check)

###################################################################
# models command
###################################################################

@click.command()
@click.option(
    "-p", "--project", "project_location",
    help="Project location (file system directory) for models.",
    metavar="LOCATION")
@click.option(
    "--installed",
    help="Show available installed packages. --project is ignore.",
    is_flag=True)
@click.option(
    "-v", "--verbose",
    help="Show model details.",
    is_flag=True)
@click.pass_context

def models(ctx, **kw):
    """Show available models.

    By default Guild will show models defined in the current directory
    (in a MODEL or MODELS file). You may use --project to specify an
    alternative project location.

    To show installed models, use the --installed option. Any location
    specified by --project, will be ignored if --installed is used.
    """
    import guild.models_cmd
    guild.models_cmd.main(Args(kw), ctx)

cli.add_command(models)

###################################################################
# operations command
###################################################################

@click.command(name="operations, ops")
@click.argument("model", required=False)
@click.option(
    "-p", "--project", "project_location",
    help="Project location (file system directory) for MODEL.",
    metavar="LOCATION")
@click.option(
    "-v", "--verbose",
    help="Show operation details.",
    is_flag=True)
@click.pass_context

def operations(ctx, **kw):
    """Show model operations.
    """
    import guild.operations_cmd
    guild.operations_cmd.main(Args(kw), ctx)

cli.add_command(operations)

###################################################################
# run command
###################################################################

def run_params(fn):
    append_params(fn, [
        click.Argument(("args",), metavar="[ARG...]", nargs=-1),
        click.Option(
            ("-p", "--project", "project_location"),
            help="Project location (file system directory) for MODEL.",
            metavar="LOCATION"),
        click.Option(
            ("--disable-plugins",),
            help="A comma separated list of plugin names to disable."),
        click.Option(
            ("-y", "--yes"),
            help="Do not prompt before running operation.",
            is_flag=True),
        click.Option(
            ("--print-command",),
            help="Show the operation command and exit (does not run).",
            is_flag=True),
        click.Option(
            ("--print-env",),
            help="Show the operation environment and exit (does not run).",
            is_flag=True),
    ])
    return fn

@click.command()
@click.argument("opspec", metavar="[MODEL:]OPERATION")
@run_params

def run(**kw):
    """Run a model operation.

    By default Guild will try to run OPERATION for the default model
    defined in a project. If a project location is not specified (see
    --project option below), Guild looks for a project in the current
    directory.

    If MODEL is specified, Guild will use it instead of the default
    model defined in a project.
    """
    import guild.run_cmd
    guild.run_cmd.main(Args(kw))

cli.add_command(run)

###################################################################
# runs command
###################################################################

class RunsGroup(click.Group):

    def get_command(self, ctx, cmd_name):
        if cmd_name in ["delete", "rm"]:
            cmd_name = "delete, rm"
        elif cmd_name in ["list", "ls"]:
            cmd_name = "list, ls"
        return super(RunsGroup, self).get_command(ctx, cmd_name)

def run_scope_options(fn):
    append_params(fn, [
        click.Option(
            ("-p", "--project", "project_location"),
            help=("Project location (file system directory) for filtering "
                  "runs."),
            metavar="LOCATION"),
        click.Option(
            ("-S", "--system"),
            help=("Include system wide runs rather than limit to runs "
                  "associated with a project location. Ignores LOCATION."),
            is_flag=True)
    ])
    return fn

def run_filters(fn):
    append_params(fn, [
        click.Option(
            ("-r", "--running", "status"),
            help="Include only runs that are still running.",
            flag_value="running"),
        click.Option(
            ("-c", "--completed", "status"),
            help="Include only completed runs.",
            flag_value="completed"),
        click.Option(
            ("-s", "--stopped", "status"),
            help=("Include only runs that exited with an error or were "
                  "terminated by the user."),
            flag_value="stopped"),
        click.Option(
            ("-e", "--error", "status"),
            help="Include only runs that exited with an error.",
            flag_value="error"),
        click.Option(
            ("-t", "--terminated", "status"),
            help="Include only runs terminated by the user.",
            flag_value="terminated"),
    ])
    return fn

def runs_list_options(fn):
    run_scope_options(fn)
    run_filters(fn)
    append_params(fn, [
        click.Option(
            ("-v", "--verbose"),
            help="Show run details.",
            is_flag=True),
        click.Option(
            ("-d", "--deleted"),
            help="Show deleted runs.",
            is_flag=True),
    ])
    return fn

@click.group(invoke_without_command=True, cls=RunsGroup)
@runs_list_options
@click.pass_context

def runs(ctx, **kw):
    """Show or manage runs.

    If COMMAND is omitted, lists run. Refer to 'guild runs list
    --help' for more information on the list command.
    """
    if not ctx.invoked_subcommand:
        ctx.invoke(list_runs, **kw)

cli.add_command(runs)

###################################################################
# runs list command
###################################################################

@click.command("list, ls")
@runs_list_options
@click.pass_context

def list_runs(ctx, **kw):
    """List runs.

    By default lists runs associated with models defined in the
    current directory, or LOCATION if specified. To list all runs, use
    the --system option.

    To list deleted runs, use the --deleted option. Note that runs are
    still limited to the specified project unless --system is
    specified.

    You may apply any of the filter options below to limit the runs
    listed.
    """
    import guild.runs_cmd
    guild.runs_cmd.list_runs(Args(kw), ctx)

runs.add_command(list_runs)

###################################################################
# runs delete command
###################################################################

RUN_ARG_HELP = """
RUN may be a run ID (or the unique start of a run ID) or a zero-based
index corresponding to a run returned by the list command. Indexes may
also be specified in ranges in the form START:END where START is the
start index and END is the end index. Either START or END may be
omitted. If START is omitted, all runs up to END are selected. If END
id omitted, all runs from START on are selected. If both START and END
are omitted (i.e. the ':' char is used by itself) all runs are selected.
"""

@click.command("delete, rm", help="""
Delete one or more runs.

%s
""" % RUN_ARG_HELP)
@click.argument("runs", metavar="RUN [RUN...]", nargs=-1, required=True)
@run_scope_options
@run_filters
@click.option(
    "-y", "--yes",
    help="Do not prompt before deleting.",
    is_flag=True)
@click.pass_context

def delete_runs(ctx, **kw):
    # Help defined in command decorator.
    import guild.runs_cmd
    guild.runs_cmd.delete_runs(Args(kw), ctx)

runs.add_command(delete_runs)

###################################################################
# runs restore command
###################################################################

@click.command("restore", help="""
Restore one or more deleted runs.

%s
""" % RUN_ARG_HELP)

@click.argument("runs", metavar="RUN [RUN...]", nargs=-1, required=True)
@run_scope_options
@run_filters
@click.option(
    "-y", "--yes",
    help="Do not prompt before restoring.",
    is_flag=True)
@click.pass_context

def restore_runs(ctx, **kw):
    # Help defined in command decorator.
    import guild.runs_cmd
    guild.runs_cmd.restore_runs(Args(kw), ctx)

runs.add_command(restore_runs)

###################################################################
# runs info command
###################################################################

@click.command("info")
@click.argument("run", required=False)
@run_scope_options
@run_filters
@click.option("--files", help="Include run files", is_flag=True)
@click.pass_context

def run_info(ctx, **kw):
    """Show run details.

    RUN must be a run ID (or the start of a run ID that uniquely
    identifies a run) or a zero-based index corresponding to the run
    as it appears in the list of filtered runs.

    By default the latest run is selected (index 0).

    EXAMPLES

    Show info for the latest run in the current project:

        guild runs info

    Show info for the latest run system wide:

        guild runs info -S

    Show info for the latest completed run in the current project:

        guild runs info -c

    Show info for run a64b1710:

        guild runs info a64b1710

    """
    import guild.runs_cmd
    guild.runs_cmd.run_info(Args(kw), ctx)

runs.add_command(run_info)

###################################################################
# shell command
###################################################################

@click.command()

def shell(**kw):
    """Start a Python shell for API experimentation.
    """
    import guild.shell_cmd
    guild.shell_cmd.main(Args(kw))

cli.add_command(shell)

###################################################################
# sources command
###################################################################

class SourcesGroup(click.Group):

    def get_command(self, ctx, cmd_name):
        if cmd_name in ["rm", "remove"]:
            cmd_name = "remove, rm"
        return super(SourcesGroup, self).get_command(ctx, cmd_name)

@click.group(invoke_without_command=True, cls=SourcesGroup)
@click.pass_context

def sources(ctx, **kw):
    """Show or manage package sources.
    """
    if not ctx.invoked_subcommand:
        import guild.sources_cmd
        guild.sources_cmd.list_sources(Args(kw))

cli.add_command(sources)

###################################################################
# sources add command
###################################################################

@click.command("add")
@click.argument("name")
@click.argument("source")
@click.pass_context

def add_source(ctx, **kw):
    """Add a package source.

    NAME must be a unique source identifier for this system. Use
    'guild sources' to show a list of configured package sources.

    SOURCE must be a GitHub repository URL or a local
    directory. GitHub repository URLs may be provided in shorthand
    using ACCOUNT/REPOSITORY. For example, the SOURCE
    'guildai/guild-packages' will be interpreted as
    'https://github.com/guildai/guild-packages'.

    Local directories must be absolute paths. Use a local directory
    will avoid having to sync with an upstream source each time a
    change is made to a package. This is useful for package
    development.
    """
    import guild.sources_cmd
    guild.sources_cmd.add_source(Args(kw), ctx)

sources.add_command(add_source)

###################################################################
# sources remove command
###################################################################

@click.command("remove, rm")
@click.argument("name")

def remove_source(**kw):
    """Remove a package source.

    Use 'guild sources' for a list of package sources.
    """
    import guild.sources_cmd
    guild.sources_cmd.remove_source(Args(kw))

sources.add_command(remove_source)

###################################################################
# train command
###################################################################

@click.command("train")
@click.argument("model", required=False)
@run_params

def train(**kw):
    """Train a model.

    Equivalent to running 'guild run [MODEL:]train [ARG...]'.

    By default MODEL is the default model for project in LOCATION.

    You may omit MODEL (i.e. for training the default model) while
    providing one or more ARG values provided the first ARG value
    contains an equals sign ('='). When specifying a switch (i.e. an
    argument that doesn't accept a value) as the first ARG, you must
    provide MODEL.

    Refer to help for the run command ('guild run --help') for more
    information.

    """
    import guild.run_cmd
    args = Args(kw)
    # MODEL is treated as an ARG if it contains an equal sign (see
    # help text above).
    if args.model and "=" in args.model:
        args.args = (args.model,) + args.args
        args.model = None
    args.opspec = "%s:train" % args.model if args.model else "train"
    guild.run_cmd.main(args)

cli.add_command(train)

###################################################################
# tensorboard command
###################################################################

@click.command("tensorboard, tb")
@run_scope_options
@click.option(
    "--host",
    help="Name of host interface to listen on.")
@click.option(
    "--port",
    help="Port to listen on.",
    type=click.IntRange(0, 65535))
@click.option(
    "--refresh-interval",
    help="TensorBoard refresh interval (defaults to 5 seconds).",
    type=click.IntRange(1, None),
    default=5)
@click.option(
    "-n", "--no-open",
    help="Don't open the TensorBoard URL in a brower.",
    is_flag=True)
@click.pass_context

def tensorboard(ctx, **kw):
    """Start TensorBoard to view runs.
    """
    import guild.tensorboard_cmd
    guild.tensorboard_cmd.main(Args(kw), ctx)

cli.add_command(tensorboard)
<|endoftext|>


<commit_before>#/*******************************************************************************
# * Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *       http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.
# *******************************************************************************/

__author__ = 'elip'

PLUGIN_INSTALLER_VERSION = 'develop'
COSMO_CELERY_COMMON_VERSION = 'develop'
KV_STORE_VERSION = 'develop'
RIEMANN_CONFIGURER_VERSION = 'develop'
AGENT_INSTALLER_VERSION = 'develop'
OPENSTACK_PROVISIONER_VERSION = 'develop'
VAGRANT_PROVISIONER_VERSION = 'develop'
<commit_msg>CLOUDIFY-2475 change worker installer plugin version<commit_after>#/*******************************************************************************
# * Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *       http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.
# *******************************************************************************/

__author__ = 'elip'

PLUGIN_INSTALLER_VERSION = 'develop'
COSMO_CELERY_COMMON_VERSION = 'develop'
KV_STORE_VERSION = 'develop'
RIEMANN_CONFIGURER_VERSION = 'develop'
AGENT_INSTALLER_VERSION = 'feature/CLOUDIFY-2475-dynamic-management-plugins-and-worker-per-deployment'
OPENSTACK_PROVISIONER_VERSION = 'develop'
VAGRANT_PROVISIONER_VERSION = 'develop'
<|endoftext|>


<commit_before>import unittest

import rotational_cipher

# Tests adapted from `problem-specifications//canonical-data.json` @ v1.1.0

class RotationalCipher(unittest.TestCase):
    def test_rotate_a_by_0(self):
        self.assertEqual(rotational_cipher.rotate('a', 0), 'a')

    def test_rotate_a_by_1(self):
        self.assertEqual(rotational_cipher.rotate('a', 1), 'b')

    def test_rotate_a_by_26(self):
        self.assertEqual(rotational_cipher.rotate('a', 26), 'a')

    def test_rotate_m_by_13(self):
        self.assertEqual(rotational_cipher.rotate('m', 13), 'z')

    def test_rotate_n_by_13_with_wrap_around_alphabet(self):
        self.assertEqual(rotational_cipher.rotate('n', 13), 'a')

    def test_rotate_capital_letters(self):
        self.assertEqual(rotational_cipher.rotate('OMG', 5), 'TRL')

    def test_rotate_spaces(self):
        self.assertEqual(rotational_cipher.rotate('O M G', 5), 'T R L')

    def test_rotate_numbers(self):
        self.assertEqual(
            rotational_cipher.rotate('Testing 1 2 3 testing', 4),
            'Xiwxmrk 1 2 3 xiwxmrk')

    def test_rotate_punctuation(self):
        self.assertEqual(
            rotational_cipher.rotate("Let's eat, Grandma!", 21),
            "Gzo'n zvo, Bmviyhv!")

    def test_rotate_all_letters(self):
        self.assertEqual(
            rotational_cipher.rotate("The quick brown fox jumps"
                                     " over the lazy dog.", 13),
            "Gur dhvpx oebja sbk whzcf bire gur ynml qbt.")

if __name__ == '__main__':
    unittest.main()
<commit_msg>rotational-cipher: update tests to v1.2.0 (#1276)<commit_after>import unittest

import rotational_cipher

# Tests adapted from `problem-specifications//canonical-data.json` @ v1.2.0

class RotationalCipher(unittest.TestCase):
    def test_rotate_a_by_0(self):
        self.assertEqual(rotational_cipher.rotate('a', 0), 'a')

    def test_rotate_a_by_1(self):
        self.assertEqual(rotational_cipher.rotate('a', 1), 'b')

    def test_rotate_a_by_26(self):
        self.assertEqual(rotational_cipher.rotate('a', 26), 'a')

    def test_rotate_m_by_13(self):
        self.assertEqual(rotational_cipher.rotate('m', 13), 'z')

    def test_rotate_n_by_13_with_wrap_around_alphabet(self):
        self.assertEqual(rotational_cipher.rotate('n', 13), 'a')

    def test_rotate_capital_letters(self):
        self.assertEqual(rotational_cipher.rotate('OMG', 5), 'TRL')

    def test_rotate_spaces(self):
        self.assertEqual(rotational_cipher.rotate('O M G', 5), 'T R L')

    def test_rotate_numbers(self):
        self.assertEqual(
            rotational_cipher.rotate('Testing 1 2 3 testing', 4),
            'Xiwxmrk 1 2 3 xiwxmrk')

    def test_rotate_punctuation(self):
        self.assertEqual(
            rotational_cipher.rotate("Let's eat, Grandma!", 21),
            "Gzo'n zvo, Bmviyhv!")

    def test_rotate_all_letters(self):
        self.assertEqual(
            rotational_cipher.rotate("The quick brown fox jumps"
                                     " over the lazy dog.", 13),
            "Gur dhvpx oebja sbk whzcf bire gur ynml qbt.")

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from ..node import OP_ADD, OP_MUL, OP_DIV, OP_POW
from .poly import match_combine_polynomes, match_expand
from .powers import match_add_exponents, match_subtract_exponents, \
        match_multiply_exponents, match_duplicate_exponent, \
        match_remove_negative_exponent, match_exponent_to_root

RULES = {
        OP_ADD: [match_combine_polynomes],
        OP_MUL: [match_expand, match_add_exponents],
        OP_DIV: [match_subtract_exponents],
        OP_POW: [match_multiply_exponents, match_duplicate_exponent, \
                 match_remove_negative_exponent, match_exponent_to_root],
        }
<commit_msg>Added match_divide_numerics to main execution rules.<commit_after>from ..node import OP_ADD, OP_MUL, OP_DIV, OP_POW
from .poly import match_combine_polynomes, match_expand
from .powers import match_add_exponents, match_subtract_exponents, \
        match_multiply_exponents, match_duplicate_exponent, \
        match_remove_negative_exponent, match_exponent_to_root
from .numerics import match_divide_numerics

RULES = {
        OP_ADD: [match_combine_polynomes],
        OP_MUL: [match_expand, match_add_exponents],
        OP_DIV: [match_subtract_exponents, match_divide_numerics],
        OP_POW: [match_multiply_exponents, match_duplicate_exponent, \
                 match_remove_negative_exponent, match_exponent_to_root],
        }
<|endoftext|>