<commit_before>#!/Users/siudeja/anaconda/bin/python
""" Coarse Ricci flow for a point cloud. """
import numpy as np
import numexpr as ne
from numba import jit  # , vectorize
import timeit

n = 9
k = 4
l = 4
runs = 10000  # how many iterations
show = 1000  # how frequently we show the result
eta = 0.0002  # factor of Ricci that is added to distance squared
# 'min' rescales the distance squared function so minimum is 1.   'L1' rescales it so the sum of distance squared stays the same (perhaps this is a misgnomer and it should be 'L2' but whatever)
rescale = 'L1'
t = 0.1  # should not be integer to avaoid division problems
noise = 0.2  # noise coefficient
CLIP = 60  # value at which we clip distance function

import gmpy2 as mp
mp.get_context().precision = 200
exp = np.frompyfunc(mp.exp, 1, 1)
expm1 = np.frompyfunc(mp.expm1, 1, 1)
log = np.frompyfunc(mp.log, 1, 1)
is_finite = np.frompyfunc(mp.is_finite, 1, 1)
to_mpfr = np.frompyfunc(mp.mpfr, 1, 1)
to_double = np.frompyfunc(float, 1, 1)

# treat some numpy warnings as errors
np.seterr(all="print")  # divide='raise', invalid='raise')

# Note dist is always the distance squared matrix.

@jit("void(f8[:,:], f8)", nopython=True, nogil=True)
def symmetric(A, sigma):
    """
    Symmetric random normal matrix with -1 on the diagonal.

    Compiled using numba jit!
    """
    n = len(A) / 2
    # blocks around diagonal (symmetric, 0 diagonal at first)
    for i in range(n):
        for j in range(i + 1, n):
            A[i, j] = A[j, i] = A[i + n, j + n] = A[j + n, i + n] = \
                np.random.normal(1.0, sigma)
    # off diagonal blocks 4*(diag block)+noise
    for i in range(n):
        for j in range(n):
            A[i, j + n] = A[j + n, i] = 4 * \
                A[i, j] + np.random.normal(1.0, sigma)

def logsumexp(a):
    """ mpfr compatible minimal logsumexp version. """
    m = np.max(a, axis=1)
    return log(np.sum(exp(a - m[:, None]), axis=1)) + m

def computeLaplaceMatrix(dMatrix, t, logeps=mp.mpfr("-10")):
    """
    Compute heat approximation to Laplacian matrix using logarithms.

    Use gmpy2 mpfr to gain more precision.

    This is slow, but more accurate.

    Cutoff for really small values, and row elimination if degenerate.
    """
    # cutoff ufunc
    cutoff = np.frompyfunc((lambda x: mp.inf(-1) if x < logeps else x), 1, 1)

    t2 = mp.mpfr(t)
    lt = mp.log(2 / t2)
    d = to_mpfr(dMatrix)
    L = d * d
    L /= -2 * t2
    cutoff(L, out=L)
    logdensity = logsumexp(L)
    L = exp(L - logdensity[:, None] + lt)
    L[np.diag_indices(len(L))] -= 2 / t2
    L = np.array(to_double(L), dtype=float)
    # if just one nonzero element, then erase row and column
    degenerate = np.sum(L != 0.0, axis=1) <= 1
    L[:, degenerate] = 0
    L[degenerate, :] = 0
    return L

def computeLaplaceMatrix2(dMatrix, t, logeps=mp.mpfr("-10")):
    """
    Compute heat approximation to Laplacian matrix using logarithms.

    Use gmpy2 mpfr to gain more precision.

    This is slow, but more accurate.

    Cutoff for really small values, and row elimination if degenerate.
    """
    lt = np.log(2 / t)
    L = np.sqrt(dMatrix * dMatrix)
    L /= -2.0 * t
    # numpy floating point errors likely below
    logdensity = logsumexp(L, axis=1)
    # logdensity = logaddexp.reduce(np.sort(L, axis=1), axis=1)
    # compute log(density-1):
    # np.log(np.expm1(logdensity))
    # logdensity + np.log1p(-exp(logdensity))

    # sum in rows must be 1
    L = np.exp(L - logdensity[:, None] + lt)
    # fix diagonal to account for -f(x)?
    # L_t matrix is the unajusted one - scaled identity
    L[np.diag_indices(len(L))] -= 2.0 / t
    # alternatively L_t could be computed using unadjusted matrix
    # applied to f - f at point
    return L

def coarseRicci(L, dMatrix):
    """ numexpr parallelized Ricci. """
    Ric = np.zeros_like(dMatrix)
    for i in xrange(len(L)):
        # now f_i and cdc are matrices
        f_i = ne.evaluate("ci-dMatrix",
                          global_dict={'ci': dMatrix[:, i, None]})
        # first CDC
        cdc = L.dot(ne.evaluate("f_i*f_i"))
        Lf = L.dot(f_i)
        # end of CDC1 combined with CDC2
        ne.evaluate("cdc/2.0-2.0*f_i*Lf", out=cdc)
        cdc = L.dot(cdc)
        ne.evaluate("cdc+f_i*LLf+Lf*Lf",
                    global_dict={'LLf': L.dot(Lf)}, out=cdc)
        Ric[i, :] = cdc[i] / 2.0
    return Ric

def CDC1(L, f, g):
    """ Compute carre-du-champ for L. """
    u = f * g
    cdc = L.dot(u) - f * L.dot(g) - g * L.dot(f)
    return cdc / 2

# for test purposes.   Tested - and it at least for simple examples the
# Ricci's are all the same.
def coarseRicciold(L, dMatrix):
    Ric = np.zeros((len(dMatrix), len(dMatrix)))
    for i in range(len(L)):
        for j in range(len(L)):
            # pay close attention to this if it becomes asymmetric
            f_ij = dMatrix[:, i] - dMatrix[:, j]
            cdc1 = CDC1(L, f_ij, f_ij)
            cdc2 = L.dot(cdc1) - 2 * CDC1(L, f_ij, L.dot(f_ij))
            Ric[i, j] = cdc2[i] / 2

    return Ric

def coarseRicci3(L, dMatrix):
    """
    Precompute Ld first and try to avoid mat-mat multiplications.

    This one is about 3x faster.
    """
    Ric = np.zeros_like(dMatrix)
    Ld = L.dot(dMatrix)
    for i in xrange(len(L)):
        # now f_i and cdc are matrices
        f_i = ne.evaluate("di-dMatrix",
                          global_dict={'di': dMatrix[:, i, None]})
        # first CDC
        # FIXME how to compute L(di*d) quickly ??
        # this one is the only matrix-matrix multiplication
        cdc = L.dot(ne.evaluate("f_i*f_i"))
        # L is linear so Lf = Ld[:, i, None] - Ld
        Lf = ne.evaluate("Ldi-Ld", global_dict={'Ldi': Ld[:, i, None]})
        # end of CDC1 combined with CDC2
        ne.evaluate("cdc-4.0*f_i*Lf", out=cdc)
        # we are using one row from cdc in Ric, so we can use one row from L
        cdc = L[i].dot(cdc)
        # we can also use one row from the rest too
        ne.evaluate("(cdc/2.0+f_ii*LLfi+Lfi*Lfi)/2.0",
                    global_dict={
                        'LLfi': L[i].dot(Lf),
                        'Lfi': Lf[i],
                        'f_ii': f_i[i]
                    }, out=cdc)
        Ric[i, :] = cdc
    return Ric

# def CDC1(L,f,g):
#     u=f*g
#     cdc = L.dot(u)-f*L.dot(g)-g*L.dot(f)
#     return cdc/2
#
# def coarseRicci2(L,dMatrix):
#     Ric = np.zeros((len(dMatrix),len(dMatrix)))
#     for i in range(len(L)):
#         for j in range(len(L)):
#             f_ij = dMatrix[:,i]-dMatrix[:,j]
#             cdc1 = CDC1(L, f_ij,f_ij)
#             cdc2 = L.dot(cdc1)-2*CDC1(L,f_ij,L.dot(f_ij))
#             Ric[i,j]=cdc2[i]/2
#     return Ric

def test(f, args_string):
    """ Test speed of a function. """
    print f.__name__
    t = timeit.repeat("%s(%s)" % (f.__name__, args_string),
                      repeat=5, number=1,
                      setup="from __main__ import %s, %s" % (f.__name__,
                                                             args_string))
    print min(t)

# This sections provides some simple test sets

# k,l are sizes of points.  sigma is about how far away points in the same
# cluster are.  Sigma must be positive
def onedimensionpair(k, l, sigma):
    X = np.random.normal(size=(k, 1))
    Y = np.random.normal(size=(l, 1)) + 2 / sigma
    Z = np.concatenate((X, Y))
    # print X
    # print Y
    print Z
    n = len(Z)
    dist = np.zeros((n, n))
    for i in range(n):
        for j in range(n):
            dist[i, j] = (Z[i] - Z[j]) * (Z[i] - Z[j])
    dist = sigma * dist
    return dist

# returns distance squared for cyclical graph with n points, with noise added
def cyclegraph(n, noise):
    dist = np.zeros((n, n))
    ndist = np.zeros((n, n))
    for i in range(n):
        for j in range(n):
            dist[i, j] = np.amin([(i - j) % n, (j - i) % n])
            ndist[i, j] = dist[i, j] * noise * np.random.randn(1)
    dist = dist * dist
    dist = dist + ndist + ndist.transpose()
    return dist

# returns distance squared.  Object is a pair of simplices with distance
# separation from each other, and internal distance 1.  Add some noise.
def closefarsimplices(n, noise, separation):
    dist = np.zeros((2 * n, 2 * n))
    symmetric(dist, noise)  # This isn't quite the object we want FIXME
    return dist

def metricize(dist):  # Only minimizes over two-stop paths not all
    dist = np.sqrt(dist)
    olddist = dist + 1
    d_ij = dist
    different = (olddist == dist).all()
    while(not different):
        # rint 'in loop'
        olddist = dist
        for i in range(len(dist)):
            for j in range(len(dist)):
                for k in range(len(dist)):
                    dijk = dist[i, k] + dist[k, j]
                    d_ij[i, j] = np.amin([d_ij[i, j], dijk])
                dist[i, j] = d_ij[i, j]
        different = (olddist == dist).all()
    return dist ** 2

dist = onedimensionpair(2, 3, noise)
dist = cyclegraph(6, noise)
#dist = closefarsimplices(n, 0.1, 1)

dist = metricize(dist)
L = computeLaplaceMatrix2(dist, t)

Ricci = coarseRicci3(L, dist)

print 'initial distance'
print dist
print 'initial Ricci'
print Ricci

ne.evaluate("dist-eta*Ricci", out=dist)

initial_L1 = dist.sum()

for i in range(runs + show + 3):
    L = computeLaplaceMatrix2(dist, t)
    Ricci = coarseRicci3(L, dist)
    ne.evaluate("dist-eta*Ricci", out=dist)
    dist = ne.evaluate("(dist + distT)/2",
                       global_dict={'distT': dist.transpose()})

    # total_distance = dist.sum()
    # dist = (total_distance0/total_distance)*dist
    nonzero = dist[np.nonzero(dist)]
    mindist = np.amin(nonzero)
    s1 = mindist
    s2 = dist.sum()
    # print t
    #ne.evaluate("dist/s", out=dist)

    dist = np.clip(dist, 0, CLIP)
    if rescale == 'L1':
        ne.evaluate("initial_L1*dist/s2", out=dist)
    if rescale == 'min':
        ne.evaluate("dist/s1", out=dist)
    dist = metricize(dist)
    if i % show == 2:
    # print Ricci
        print "dist for ", i, "  time"
        print dist
        print 't = ', t
        # print Ricci
        # print Ricci/dist, '<- Ricc/dist'
        print '---------'
<commit_msg>fixed merge mixups<commit_after>#!/Users/siudeja/anaconda/bin/python
""" Coarse Ricci flow for a point cloud. """
import numpy as np
import numexpr as ne
import scipy.misc as sm
from numba import jit  # , vectorize
import timeit

n = 9
k = 4
l = 4
runs = 10000  # how many iterations
show = 1000  # how frequently we show the result
eta = 0.0002  # factor of Ricci that is added to distance squared
# 'min' rescales the distance squared function so minimum is 1.   'L1' rescales it so the sum of distance squared stays the same (perhaps this is a misgnomer and it should be 'L2' but whatever)
rescale = 'L1'
t = 0.1  # should not be integer to avaoid division problems
noise = 0.2  # noise coefficient
CLIP = 60  # value at which we clip distance function

import gmpy2 as mp
mp.get_context().precision = 200
exp = np.frompyfunc(mp.exp, 1, 1)
expm1 = np.frompyfunc(mp.expm1, 1, 1)
log = np.frompyfunc(mp.log, 1, 1)
is_finite = np.frompyfunc(mp.is_finite, 1, 1)
to_mpfr = np.frompyfunc(mp.mpfr, 1, 1)
to_double = np.frompyfunc(float, 1, 1)

# treat some numpy warnings as errors
np.seterr(all="print")  # divide='raise', invalid='raise')

# Note dist is always the distance squared matrix.

@jit("void(f8[:,:], f8)", nopython=True, nogil=True)
def symmetric(A, sigma):
    """
    Symmetric random normal matrix with -1 on the diagonal.

    Compiled using numba jit!
    """
    n = len(A) / 2
    # blocks around diagonal (symmetric, 0 diagonal at first)
    for i in range(n):
        for j in range(i + 1, n):
            A[i, j] = A[j, i] = A[i + n, j + n] = A[j + n, i + n] = \
                np.random.normal(1.0, sigma)
    # off diagonal blocks 4*(diag block)+noise
    for i in range(n):
        for j in range(n):
            A[i, j + n] = A[j + n, i] = 4 * \
                A[i, j] + np.random.normal(1.0, sigma)

def logsumexp(a):
    """ mpfr compatible minimal logsumexp version. """
    m = np.max(a, axis=1)
    return log(np.sum(exp(a - m[:, None]), axis=1)) + m

def computeLaplaceMatrix(dMatrix, t, logeps=mp.mpfr("-10")):
    """
    Compute heat approximation to Laplacian matrix using logarithms.

    Use gmpy2 mpfr to gain more precision.

    This is slow, but more accurate.

    Cutoff for really small values, and row elimination if degenerate.
    """
    # cutoff ufunc
    cutoff = np.frompyfunc((lambda x: mp.inf(-1) if x < logeps else x), 1, 1)

    t2 = mp.mpfr(t)
    lt = mp.log(2 / t2)
    d = to_mpfr(dMatrix)
    L = d * d
    L /= -2 * t2
    cutoff(L, out=L)
    logdensity = logsumexp(L)
    L = exp(L - logdensity[:, None] + lt)
    L[np.diag_indices(len(L))] -= 2 / t2
    L = np.array(to_double(L), dtype=float)
    # if just one nonzero element, then erase row and column
    degenerate = np.sum(L != 0.0, axis=1) <= 1
    L[:, degenerate] = 0
    L[degenerate, :] = 0
    return L

def computeLaplaceMatrix2(dMatrix, t, logeps=mp.mpfr("-10")):
    """
    Compute heat approximation to Laplacian matrix using logarithms.

    Use gmpy2 mpfr to gain more precision.

    This is slow, but more accurate.

    Cutoff for really small values, and row elimination if degenerate.
    """
    lt = np.log(2 / t)
    L = np.sqrt(dMatrix * dMatrix)
    L /= -2.0 * t
    # numpy floating point errors likely below
    logdensity = sm.logsumexp(L, axis=1)
    # logdensity = logaddexp.reduce(np.sort(L, axis=1), axis=1)
    # compute log(density-1):
    # np.log(np.expm1(logdensity))
    # logdensity + np.log1p(-exp(logdensity))

    # sum in rows must be 1
    L = np.exp(L - logdensity[:, None] + lt)
    # fix diagonal to account for -f(x)?
    # L_t matrix is the unajusted one - scaled identity
    L[np.diag_indices(len(L))] -= 2.0 / t
    # alternatively L_t could be computed using unadjusted matrix
    # applied to f - f at point
    return L

def coarseRicci(L, dMatrix):
    """ numexpr parallelized Ricci. """
    Ric = np.zeros_like(dMatrix)
    for i in xrange(len(L)):
        # now f_i and cdc are matrices
        f_i = ne.evaluate("ci-dMatrix",
                          global_dict={'ci': dMatrix[:, i, None]})
        # first CDC
        cdc = L.dot(ne.evaluate("f_i*f_i"))
        Lf = L.dot(f_i)
        # end of CDC1 combined with CDC2
        ne.evaluate("cdc/2.0-2.0*f_i*Lf", out=cdc)
        cdc = L.dot(cdc)
        ne.evaluate("cdc+f_i*LLf+Lf*Lf",
                    global_dict={'LLf': L.dot(Lf)}, out=cdc)
        Ric[i, :] = cdc[i] / 2.0
    return Ric

def CDC1(L, f, g):
    """ Compute carre-du-champ for L. """
    u = f * g
    cdc = L.dot(u) - f * L.dot(g) - g * L.dot(f)
    return cdc / 2

# for test purposes.   Tested - and it at least for simple examples the
# Ricci's are all the same.
def coarseRicciold(L, dMatrix):
    Ric = np.zeros((len(dMatrix), len(dMatrix)))
    for i in range(len(L)):
        for j in range(len(L)):
            # pay close attention to this if it becomes asymmetric
            f_ij = dMatrix[:, i] - dMatrix[:, j]
            cdc1 = CDC1(L, f_ij, f_ij)
            cdc2 = L.dot(cdc1) - 2 * CDC1(L, f_ij, L.dot(f_ij))
            Ric[i, j] = cdc2[i] / 2

    return Ric

def coarseRicci3(L, dMatrix):
    """
    Precompute Ld first and try to avoid mat-mat multiplications.

    This one is about 3x faster.
    """
    Ric = np.zeros_like(dMatrix)
    Ld = L.dot(dMatrix)
    for i in xrange(len(L)):
        # now f_i and cdc are matrices
        f_i = ne.evaluate("di-dMatrix",
                          global_dict={'di': dMatrix[:, i, None]})
        # first CDC
        # FIXME how to compute L(di*d) quickly ??
        # this one is the only matrix-matrix multiplication
        cdc = L.dot(ne.evaluate("f_i*f_i"))
        # L is linear so Lf = Ld[:, i, None] - Ld
        Lf = ne.evaluate("Ldi-Ld", global_dict={'Ldi': Ld[:, i, None]})
        # end of CDC1 combined with CDC2
        ne.evaluate("cdc-4.0*f_i*Lf", out=cdc)
        # we are using one row from cdc in Ric, so we can use one row from L
        cdc = L[i].dot(cdc)
        # we can also use one row from the rest too
        ne.evaluate("(cdc/2.0+f_ii*LLfi+Lfi*Lfi)/2.0",
                    global_dict={
                        'LLfi': L[i].dot(Lf),
                        'Lfi': Lf[i],
                        'f_ii': f_i[i]
                    }, out=cdc)
        Ric[i, :] = cdc
    return Ric

# def CDC1(L,f,g):
#     u=f*g
#     cdc = L.dot(u)-f*L.dot(g)-g*L.dot(f)
#     return cdc/2
#
# def coarseRicci2(L,dMatrix):
#     Ric = np.zeros((len(dMatrix),len(dMatrix)))
#     for i in range(len(L)):
#         for j in range(len(L)):
#             f_ij = dMatrix[:,i]-dMatrix[:,j]
#             cdc1 = CDC1(L, f_ij,f_ij)
#             cdc2 = L.dot(cdc1)-2*CDC1(L,f_ij,L.dot(f_ij))
#             Ric[i,j]=cdc2[i]/2
#     return Ric

def test(f, args_string):
    """ Test speed of a function. """
    print f.__name__
    t = timeit.repeat("%s(%s)" % (f.__name__, args_string),
                      repeat=5, number=1,
                      setup="from __main__ import %s, %s" % (f.__name__,
                                                             args_string))
    print min(t)

# This sections provides some simple test sets

# k,l are sizes of points.  sigma is about how far away points in the same
# cluster are.  Sigma must be positive
def onedimensionpair(k, l, sigma):
    X = np.random.normal(size=(k, 1))
    Y = np.random.normal(size=(l, 1)) + 2 / sigma
    Z = np.concatenate((X, Y))
    # print X
    # print Y
    print Z
    n = len(Z)
    dist = np.zeros((n, n))
    for i in range(n):
        for j in range(n):
            dist[i, j] = (Z[i] - Z[j]) * (Z[i] - Z[j])
    dist = sigma * dist
    return dist

# returns distance squared for cyclical graph with n points, with noise added
def cyclegraph(n, noise):
    dist = np.zeros((n, n))
    ndist = np.zeros((n, n))
    for i in range(n):
        for j in range(n):
            dist[i, j] = np.amin([(i - j) % n, (j - i) % n])
            ndist[i, j] = dist[i, j] * noise * np.random.randn(1)
    dist = dist * dist
    dist = dist + ndist + ndist.transpose()
    return dist

# returns distance squared.  Object is a pair of simplices with distance
# separation from each other, and internal distance 1.  Add some noise.
def closefarsimplices(n, noise, separation):
    dist = np.zeros((2 * n, 2 * n))
    symmetric(dist, noise)  # This isn't quite the object we want FIXME
    return dist

def metricize(dist):  # Only minimizes over two-stop paths not all
    dist = np.sqrt(dist)
    olddist = dist + 1
    d_ij = dist
    different = (olddist == dist).all()
    while(not different):
        # rint 'in loop'
        olddist = dist
        for i in range(len(dist)):
            for j in range(len(dist)):
                for k in range(len(dist)):
                    dijk = dist[i, k] + dist[k, j]
                    d_ij[i, j] = np.amin([d_ij[i, j], dijk])
                dist[i, j] = d_ij[i, j]
        different = (olddist == dist).all()
    return dist ** 2

dist = onedimensionpair(2, 3, noise)
dist = cyclegraph(6, noise)
#dist = closefarsimplices(n, 0.1, 1)

dist = metricize(dist)
L = computeLaplaceMatrix2(dist, t)

Ricci = coarseRicci3(L, dist)

print 'initial distance'
print dist
print 'initial Ricci'
print Ricci

ne.evaluate("dist-eta*Ricci", out=dist)

initial_L1 = dist.sum()

for i in range(runs + show + 3):
    L = computeLaplaceMatrix2(dist, t)
    Ricci = coarseRicci3(L, dist)
    ne.evaluate("dist-eta*Ricci", out=dist)
    dist = ne.evaluate("(dist + distT)/2",
                       global_dict={'distT': dist.transpose()})

    # total_distance = dist.sum()
    # dist = (total_distance0/total_distance)*dist
    nonzero = dist[np.nonzero(dist)]
    mindist = np.amin(nonzero)
    s1 = mindist
    s2 = dist.sum()
    # print t
    #ne.evaluate("dist/s", out=dist)

    dist = np.clip(dist, 0, CLIP)
    if rescale == 'L1':
        ne.evaluate("initial_L1*dist/s2", out=dist)
    if rescale == 'min':
        ne.evaluate("dist/s1", out=dist)
    dist = metricize(dist)
    if i % show == 2:
    # print Ricci
        print "dist for ", i, "  time"
        print dist
        print 't = ', t
        # print Ricci
        # print Ricci/dist, '<- Ricc/dist'
        print '---------'
<|endoftext|>


<commit_before>from celery.task import task
from timetable.courses.bridge import import_courses

import_courses = task(import_courses)<commit_msg>Updated tasks.py file<commit_after>from timetable.courses.bridge import import_courses
#from celery.task import task

#import_courses = task(import_courses)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import codecs

from pyparsing import White, Word, alphanums, CharsNotIn
from pyparsing import Forward, Group, OneOrMore
from pyparsing import pythonStyleComment

class CronJob(object):

    def __init__(self, command, minute, user="root", hour="*", dom="*",
                 month="*", dow="*"):
        self.command = command
        self.minute = minute
        self.user = user
        self.hour = hour
        self.dom = dom
        self.month = month
        self.dow = dow

    @classmethod
    def from_time(cls, command, user, time):
        """
        Alternative method to instantiate CronJob.
        :param command: like in ``CronJob.__init__``
        :param user: like in ``CronJob.__init__``
        :param time: a list or tuple of at most five strings. Missing elements are set to "*".
        :return: a ``CronJob`` object
        """
        assert len(time) <= 5
        padded_time = tuple(time) + ('*',) * (5 - len(time))
        assert len(padded_time) == 5
        return CronJob(command, time[0], user, padded_time[1], padded_time[2], padded_time[3], padded_time[4])

    @property
    def time(self):
        """ Return the cronjob time as a tuple of five strings """
        return (self.minute,
                self.hour,
                self.dom,
                self.month,
                self.dow)

    def __str__(self):
        """
        This simply returns the CronJob so that it can be added to the crontab
        """
        cronjob = "%s\t%s\t%s\t%s\t%s\t%s\t%s" % (self.minute, self.hour,
                                                  self.dom, self.month,
                                                  self.dow, self.user,
                                                  self.command)
        return cronjob

    def get_time_comment(self):
        """
        Return a human-readable comment describing the cronjob time, if possible.
        :return: a string, e.g. "hourly", "monthly", which may be empty if no description can be inferred.
        """
        comment = ""
        if self.minute != "*":
            comment = "hourly"
        if self.hour != "*":
            comment = "daily"
        if self.dow != "*":
            comment = "weekly"
        if self.dom != "*":
            comment = "monthly"
        if self.month != "*":
            comment = "yearly"
        return comment

    def get_time_summary(self):
        """
        Get a human-readable summary of the cronjob time.
        :return: a string of the form "Y at time hh:mm, day of month:x, month:x, day of week:x",
        where as Y is the comment returned by ``get_time_comment``. If no comment can be found,
        the "Y at" is omitted.
        """
        comment = self.get_time_comment()
        if comment:
            full_comment = "{} at ".format(comment)
        else:
            full_comment = ""
        hour = self.hour
        if hour != "*":
            hour = hour.zfill(2)
        minute = self.minute
        if minute != "*":
            minute = minute.zfill(2)
        return "{}time {}:{}, day of month: {}, month: {}, day of week: {}".format(
            full_comment,
            hour,
            minute,
            self.dom,
            self.month,
            self.dow
        )

class CronJobParser(object):

    dtime = Word("0123456789-*")
    command = CharsNotIn("{}\n#,")
    username = Word(alphanums)
    key = Word(alphanums)
    value = command
    space = White().suppress()
    comment = Word("#")
    equals = Word("=")
    assignment = Forward()
    assignment << Group(key
                        + equals.suppress()
                        + value)
    entry_block = Forward()
    entry_block << Group(dtime
                         + space.suppress()
                         + dtime
                         + space.suppress()
                         + dtime
                         + space.suppress()
                         + dtime
                         + space.suppress()
                         + dtime
                         + space.suppress()
                         + username
                         + space.suppress()
                         + command
                         )
    cron_file = OneOrMore(entry_block | assignment).ignore(pythonStyleComment)

    file_header = """# File parsed and saved by crontabparser.\n\n"""

    def __init__(self, infile="/etc/crontab"):
        self.file = infile
        self.read()

    def read(self):
        f = codecs.open(self.file, "r", "utf-8")
        self.content = f.read()
        f.close()
        data = self.get()
        self.assignments = {}
        self.cronjobs = []
        for entry in data:
            if len(entry) == 2:
                self.assignments[entry[0]] = entry[1]
            elif len(entry) == 7:
                self.cronjobs.append(CronJob(entry[6], entry[0],
                                        user=entry[5], hour=entry[1],
                                        dom=entry[2], month=entry[3],
                                        dow=entry[4]))

    @property
    def config(self):
        return {"assignments": self.assignments,
                "cronjobs": self.cronjobs}

    def get(self):
        """
        return the grouped config

        """
        # reread the file from dist
        if self.file:
            f = codecs.open(self.file, "r", "utf-8")
            self.content = f.read()
            f.close()
        config = self.cron_file.parseString(self.content)
        return config

    def format(self):
        """
        :return: The formatted data as it would be written to a file
        """
        output = ""
        output += self.file_header
        # write the assignments
        assignments = self.assignments
        for assignment in assignments:
            output += "%s=%s\n" % (assignment,
                                   assignments.get(assignment))
        # write the cronjobs
        output += "\n#m\th\tdom\tmon\tdow\tuser\tcommand\n"
        for cronjob in self.cronjobs:
            output += "%s\n" % cronjob
        return output

    def save(self, outfile):
        output = self.format()
        f = codecs.open(outfile, 'w', 'utf-8')
        for line in output.splitlines():
            f.write(line + "\n")
        f.close()
<commit_msg>Actually use a @classmethod<commit_after># -*- coding: utf-8 -*-
import codecs

from pyparsing import White, Word, alphanums, CharsNotIn
from pyparsing import Forward, Group, OneOrMore
from pyparsing import pythonStyleComment

class CronJob(object):

    def __init__(self, command, minute, user="root", hour="*", dom="*",
                 month="*", dow="*"):
        self.command = command
        self.minute = minute
        self.user = user
        self.hour = hour
        self.dom = dom
        self.month = month
        self.dow = dow

    @classmethod
    def from_time(cls, command, user, time):
        """
        Alternative method to instantiate CronJob.
        :param command: like in ``CronJob.__init__``
        :param user: like in ``CronJob.__init__``
        :param time: a list or tuple of at most five strings. Missing elements are set to "*".
        :return: a ``CronJob`` object
        """
        assert len(time) <= 5
        padded_time = tuple(time) + ('*',) * (5 - len(time))
        assert len(padded_time) == 5
        return cls(command, time[0], user, padded_time[1], padded_time[2], padded_time[3], padded_time[4])

    @property
    def time(self):
        """ Return the cronjob time as a tuple of five strings """
        return (self.minute,
                self.hour,
                self.dom,
                self.month,
                self.dow)

    def __str__(self):
        """
        This simply returns the CronJob so that it can be added to the crontab
        """
        cronjob = "%s\t%s\t%s\t%s\t%s\t%s\t%s" % (self.minute, self.hour,
                                                  self.dom, self.month,
                                                  self.dow, self.user,
                                                  self.command)
        return cronjob

    def get_time_comment(self):
        """
        Return a human-readable comment describing the cronjob time, if possible.
        :return: a string, e.g. "hourly", "monthly", which may be empty if no description can be inferred.
        """
        comment = ""
        if self.minute != "*":
            comment = "hourly"
        if self.hour != "*":
            comment = "daily"
        if self.dow != "*":
            comment = "weekly"
        if self.dom != "*":
            comment = "monthly"
        if self.month != "*":
            comment = "yearly"
        return comment

    def get_time_summary(self):
        """
        Get a human-readable summary of the cronjob time.
        :return: a string of the form "Y at time hh:mm, day of month:x, month:x, day of week:x",
        where as Y is the comment returned by ``get_time_comment``. If no comment can be found,
        the "Y at" is omitted.
        """
        comment = self.get_time_comment()
        if comment:
            full_comment = "{} at ".format(comment)
        else:
            full_comment = ""
        hour = self.hour
        if hour != "*":
            hour = hour.zfill(2)
        minute = self.minute
        if minute != "*":
            minute = minute.zfill(2)
        return "{}time {}:{}, day of month: {}, month: {}, day of week: {}".format(
            full_comment,
            hour,
            minute,
            self.dom,
            self.month,
            self.dow
        )

class CronJobParser(object):

    dtime = Word("0123456789-*")
    command = CharsNotIn("{}\n#,")
    username = Word(alphanums)
    key = Word(alphanums)
    value = command
    space = White().suppress()
    comment = Word("#")
    equals = Word("=")
    assignment = Forward()
    assignment << Group(key
                        + equals.suppress()
                        + value)
    entry_block = Forward()
    entry_block << Group(dtime
                         + space.suppress()
                         + dtime
                         + space.suppress()
                         + dtime
                         + space.suppress()
                         + dtime
                         + space.suppress()
                         + dtime
                         + space.suppress()
                         + username
                         + space.suppress()
                         + command
                         )
    cron_file = OneOrMore(entry_block | assignment).ignore(pythonStyleComment)

    file_header = """# File parsed and saved by crontabparser.\n\n"""

    def __init__(self, infile="/etc/crontab"):
        self.file = infile
        self.read()

    def read(self):
        f = codecs.open(self.file, "r", "utf-8")
        self.content = f.read()
        f.close()
        data = self.get()
        self.assignments = {}
        self.cronjobs = []
        for entry in data:
            if len(entry) == 2:
                self.assignments[entry[0]] = entry[1]
            elif len(entry) == 7:
                self.cronjobs.append(CronJob(entry[6], entry[0],
                                        user=entry[5], hour=entry[1],
                                        dom=entry[2], month=entry[3],
                                        dow=entry[4]))

    @property
    def config(self):
        return {"assignments": self.assignments,
                "cronjobs": self.cronjobs}

    def get(self):
        """
        return the grouped config

        """
        # reread the file from dist
        if self.file:
            f = codecs.open(self.file, "r", "utf-8")
            self.content = f.read()
            f.close()
        config = self.cron_file.parseString(self.content)
        return config

    def format(self):
        """
        :return: The formatted data as it would be written to a file
        """
        output = ""
        output += self.file_header
        # write the assignments
        assignments = self.assignments
        for assignment in assignments:
            output += "%s=%s\n" % (assignment,
                                   assignments.get(assignment))
        # write the cronjobs
        output += "\n#m\th\tdom\tmon\tdow\tuser\tcommand\n"
        for cronjob in self.cronjobs:
            output += "%s\n" % cronjob
        return output

    def save(self, outfile):
        output = self.format()
        f = codecs.open(outfile, 'w', 'utf-8')
        for line in output.splitlines():
            f.write(line + "\n")
        f.close()
<|endoftext|>


<commit_before>__version__ = "0.3.0"
<commit_msg>bump version to 0.3.1<commit_after>__version__ = "0.3.1"
<|endoftext|>


<commit_before>"""
Methods for converting Mediawiki content to the Dokuwiki format.

Uses mwlib to parse the Mediawiki markup.

Copyright (C) 2014 Angus Gratton
Licensed under New BSD License as described in the file LICENSE.
"""
from __future__ import print_function, unicode_literals, absolute_import, division
import re, string, dokuwiki, visitor
from mwlib.parser import *
from mwlib import uparser

# Regex to match any known File: namespace (can be updated based on the mediawiki installation language)
mw_file_namespace_aliases = re.compile("^(Image|File):", re.IGNORECASE)
dw_file_namespace = "File:"

def set_file_namespaces(canonical_alias, aliases):
    """
    Allow the mediawiki parser to match localised namespaces for files/images

    Arguments:
    canonical_alias is the single namespace that dokuwiki will use (default File:)
    aliases is a list of alternative namespace names that will be converted to the canonical alias
    """
    global mw_file_namespace_aliases
    global dw_file_namespace
    dw_file_namespace = canonical_alias + ":"
    mw_file_namespace_aliases = re.compile("^(%s):" % "|".join(aliases), re.IGNORECASE)

def is_file_namespace(target):
    """
    Is this target URL part of a known File or Image path?
    """
    return re.match(mw_file_namespace_aliases, target)

def canonicalise_file_namespace(target):
    """
    Convert any known File: or Image: (or alias) namespace link to be File:
    ... mediawiki stores all these under a common namespace, so dokuwiki has no choice but to import
    them all under a single canonical
    """
    return re.sub(mw_file_namespace_aliases, dw_file_namespace, target)

def convert_pagecontent(title, content):
    """
    Convert a string in Mediawiki content format to a string in
    Dokuwiki content format.
    """
    # wrap the "magic" marker tag <__mw_nowiki> around <nowiki>, as
    # as mwlib just discards it otherwise and we can't detect it within the parser.
    # We keep the inner <nowiki> so the mwlib parser still skips that content
    content = re.sub(r"<nowiki>.+</nowiki>", lambda e: "<__mw_nowiki>"+e.group(0)+"</__mw_nowiki>", content)

    root = uparser.parseString(title, content) # create parse tree
    return convert(root, False)

def convert_children(node):
    """Walk the children of this parse node and call convert() on each.
    """
    result = ""
    for child in node.children:
        res = convert(child, result.endswith("\n"))
        if type(res) is str:
            res = unicode(res)
        if type(res) is not unicode:
            print("Got invalid response '%s' when processing '%s'" % (res,child))
        result += res
    return result

@visitor.when(Article)
def convert(node, trailing_newline):
    return convert_children(node)

@visitor.when(Paragraph)
def convert(node, trailing_newline):
    return convert_children(node) + "\n"

@visitor.when(Text)
def convert(text, trailing_newline):
    magic_nw_tag = "<__mw_nowiki>"
    if text._text == magic_nw_tag:
        has_trailing_newline = (text.caption[-1] == '\n')
        suffix = ""
        if has_trailing_newline:
            text.caption = text.caption[:-1]
            suffix = "\n"
        return "<nowiki>" + text.caption[len(magic_nw_tag):-1-len(magic_nw_tag)] + "</nowiki>" + suffix
    else:
        return text.caption

@visitor.when(Section)
def convert(section, trailing_newline):
    result = ""
    if section.tagname == "p":
        pass
    elif section.tagname == "@section":
        level = section.level
        heading = convert(section.children.pop(0), trailing_newline).strip()
        heading_boundary = "="*(6-level)
        result = "\n%s %s %s\n" % (heading_boundary, heading, heading_boundary)
    else:
        print("Unknown tagname %s" % section.tagname)

    return result + convert_children(section)

@visitor.when(Style)
def convert(style, trailing_newline):
    formatter = {
        ";" :  ("**", r"**\\"),     # definition (essentially boldface)
        "''" : ("//", "//"),        # italics
        "'''" :("**", "**"),        # boldface
        ":"   : ("", ""),           # other end of a definition???
        "sub" : ("",""),
        "sup" : ("",""),
        "big" : ("**", "**"),        # <big> not in dokuwiki so use bold
        }.get(style.caption, None)
    if formatter is None:
        print("WARNING: Ignoring unknown formatter %s" % style.caption)
        formatter = ("","")
    return formatter[0] + convert_children(style) + formatter[1]

@visitor.when(NamedURL)
def convert(url, trailing_newline):
    text = convert_children(url).strip(" ")
    url = url.caption
    if len(text):
        return "[[%s|%s]]" % (url, text)
    else:
        return "%s" % (url)

@visitor.when(URL)
def convert(url, trailing_newline):
    return url.caption

@visitor.when(ImageLink)
def convert(link, trailing_newline):
    suffix = ""
    if link.width is not None:
        if link.height is None:
            suffix = "?%s" % link.width
        else:
            suffix = "?%sx%s" % (link.width, link.height)
    else:
        try:
            if link.in_gallery: # see below for Tag->gallery handling
                suffix = "?160" # gallery images should be thumbnailed
        except AttributeError:
            pass # not in a gallery
    prealign = " " if link.align in [ "center", "right" ] else ""
    postalign = " " if link.align in [ "center", "left" ] else ""
    target = canonicalise_file_namespace(link.target)
    target = convert_internal_link(target)
    return "{{%s%s%s%s}}" % (prealign, target, suffix, postalign)

@visitor.when(ArticleLink)
def convert(link, trailing_newline):
    text = convert_children(link).strip(" ")
    pagename = convert_internal_link(link.target)
    if len(text):
        return "[[%s|%s]]" % (pagename, text)
    else:
        return "[[%s]]" % pagename

@visitor.when(CategoryLink)
def convert(link, trailing_newline):
    # Category functionality can be implemented with plugin:tag, but not used here
    return ""

@visitor.when(NamespaceLink)
def convert(link, trailing_newline):
    if is_file_namespace(link.target): # is a link to a file or image
        filename = dokuwiki.make_dokuwiki_pagename(canonicalise_file_namespace(link.target))
        caption = convert_children(link).strip()
        if len(caption) > 0:
            return "{{%s%s}}" % (filename, caption)
        else:
            return "{{%s}}" % filename

    print("WARNING: Ignoring namespace link to " + link.target)
    return convert_children(link)

@visitor.when(ItemList)
def convert(itemlist, trailing_newline):
    def apply_itemlist_properties(node):
        # ItemLists are used for depth/style tracking - applies those attributes to all its children
        for child in node.children:
            try:
                child.list_style = "* " if itemlist.tagname == "ul" else "- "
                child.list_depth += 1
            except AttributeError:
                child.list_depth = 1
            apply_itemlist_properties(child)
    apply_itemlist_properties(itemlist)
    return convert_children(itemlist)

@visitor.when(Item)
def convert(item, trailing_newline):
    item_content = convert_children(item)
    return "  "*item.list_depth + item.list_style + item_content

@visitor.when(Table)
def convert(table, trailing_newline):
    # we ignore the actual Table tags, instead convert each Row & Cell individually
    return convert_children(table)

@visitor.when(Cell)
def convert(cell, trailing_newline):
    marker = "^" if cell.tagname == "th" else "|"
    result = "%s %s" % (marker, convert_children(cell).replace('\n','').strip())
    return result

@visitor.when(Row)
def convert(row, trailing_newline):
    return convert_children(row) + " |\n"

@visitor.when(PreFormatted)
def convert(pre, trailing_newline):
    try:
        in_list = item.list_depth > 0
    except:
        in_list = False
    if trailing_newline and not in_list: # in its own paragraph, use a two space indent
        return "  " + convert_children(pre).replace("\n","\n  ").strip(" ")
    else: # inline in a list or a paragraph body, use  tags
        return "" + convert_children(pre) + ""

@visitor.when(TagNode)
def convert(tag, trailing_newline):
    # dict maps mediawiki tag name to tuple of starting, ending dokuwiki tag
    simple_tagitems = {
        "tt" : ("''", "''"),
        "ref" : ("((","))"), # references converted to footnotes
        "code" : ("",""),
    }
    if tag.tagname in simple_tagitems:
        pre,post = simple_tagitems[tag.tagname]
        return pre + convert_children(tag) + post
    elif tag._text is not None:
        if tag._text.replace(" ","").replace("/","") == "
":
            return "\n" # this is a oneoff hack for one wiki page covered in

        return tag._text # may not work for non-selfclosing tags
    elif tag.tagname == "gallery":
        # with a lot of cleverness we could use the gallery plugin for this,
        # but this should do. We flag each child image as in the gallery, then
        # deal with the ImageLinks above
        for child in tag.children:
            child.in_gallery = True
    elif tag.tagname == "references":
        print("WARNING: <references> tag has no equivalent in Dokuwiki, ignoring...")

    return convert_children(tag)

# catchall for Node, which is the parent class of everything above
@visitor.when(Node)
def convert(node, trailing_newline):
    if node.__class__ != Node:
        print("WARNING: Unsupported node type: %s" % (node.__class__))
    return convert_children(node)

def convert_internal_link(mw_target):
    """
    Convert an internal Mediawiki link, with or without an anchor # in the middle.

    Same as converting a plain pagename, only we want to preserve any #s in the target text.
    """
    if "#" in mw_target:
        page,anchor = mw_target.split("#",1)
    else:
        page = mw_target
        anchor = None
    if len(page):
        page = dokuwiki.make_dokuwiki_pagename(page)
    if anchor is not None:
        page = page + "#" + dokuwiki.make_dokuwiki_heading_id(anchor)
    return page
<commit_msg>Use non-greedy matching when looking for pairs of <nowiki> tags<commit_after>"""
Methods for converting Mediawiki content to the Dokuwiki format.

Uses mwlib to parse the Mediawiki markup.

Copyright (C) 2014 Angus Gratton
Licensed under New BSD License as described in the file LICENSE.
"""
from __future__ import print_function, unicode_literals, absolute_import, division
import re, string, dokuwiki, visitor
from mwlib.parser import *
from mwlib import uparser

# Regex to match any known File: namespace (can be updated based on the mediawiki installation language)
mw_file_namespace_aliases = re.compile("^(Image|File):", re.IGNORECASE)
dw_file_namespace = "File:"

def set_file_namespaces(canonical_alias, aliases):
    """
    Allow the mediawiki parser to match localised namespaces for files/images

    Arguments:
    canonical_alias is the single namespace that dokuwiki will use (default File:)
    aliases is a list of alternative namespace names that will be converted to the canonical alias
    """
    global mw_file_namespace_aliases
    global dw_file_namespace
    dw_file_namespace = canonical_alias + ":"
    mw_file_namespace_aliases = re.compile("^(%s):" % "|".join(aliases), re.IGNORECASE)

def is_file_namespace(target):
    """
    Is this target URL part of a known File or Image path?
    """
    return re.match(mw_file_namespace_aliases, target)

def canonicalise_file_namespace(target):
    """
    Convert any known File: or Image: (or alias) namespace link to be File:
    ... mediawiki stores all these under a common namespace, so dokuwiki has no choice but to import
    them all under a single canonical
    """
    return re.sub(mw_file_namespace_aliases, dw_file_namespace, target)

def convert_pagecontent(title, content):
    """
    Convert a string in Mediawiki content format to a string in
    Dokuwiki content format.
    """
    # wrap the "magic" marker tag <__mw_nowiki> around <nowiki>, as
    # as mwlib just discards it otherwise and we can't detect it within the parser.
    # We keep the inner <nowiki> so the mwlib parser still skips that content
    content = re.sub(r"<nowiki>.+?</nowiki>", lambda e: "<__mw_nowiki>"+e.group(0)+"</__mw_nowiki>", content)

    root = uparser.parseString(title, content) # create parse tree
    return convert(root, False)

def convert_children(node):
    """Walk the children of this parse node and call convert() on each.
    """
    result = ""
    for child in node.children:
        res = convert(child, result.endswith("\n"))
        if type(res) is str:
            res = unicode(res)
        if type(res) is not unicode:
            print("Got invalid response '%s' when processing '%s'" % (res,child))
        result += res
    return result

@visitor.when(Article)
def convert(node, trailing_newline):
    return convert_children(node)

@visitor.when(Paragraph)
def convert(node, trailing_newline):
    return convert_children(node) + "\n"

@visitor.when(Text)
def convert(text, trailing_newline):
    magic_nw_tag = "<__mw_nowiki>"
    if text._text == magic_nw_tag:
        has_trailing_newline = (text.caption[-1] == '\n')
        suffix = ""
        if has_trailing_newline:
            text.caption = text.caption[:-1]
            suffix = "\n"
        return "<nowiki>" + text.caption[len(magic_nw_tag):-1-len(magic_nw_tag)] + "</nowiki>" + suffix
    else:
        return text.caption

@visitor.when(Section)
def convert(section, trailing_newline):
    result = ""
    if section.tagname == "p":
        pass
    elif section.tagname == "@section":
        level = section.level
        heading = convert(section.children.pop(0), trailing_newline).strip()
        heading_boundary = "="*(6-level)
        result = "\n%s %s %s\n" % (heading_boundary, heading, heading_boundary)
    else:
        print("Unknown tagname %s" % section.tagname)

    return result + convert_children(section)

@visitor.when(Style)
def convert(style, trailing_newline):
    formatter = {
        ";" :  ("**", r"**\\"),     # definition (essentially boldface)
        "''" : ("//", "//"),        # italics
        "'''" :("**", "**"),        # boldface
        ":"   : ("", ""),           # other end of a definition???
        "sub" : ("",""),
        "sup" : ("",""),
        "big" : ("**", "**"),        # <big> not in dokuwiki so use bold
        }.get(style.caption, None)
    if formatter is None:
        print("WARNING: Ignoring unknown formatter %s" % style.caption)
        formatter = ("","")
    return formatter[0] + convert_children(style) + formatter[1]

@visitor.when(NamedURL)
def convert(url, trailing_newline):
    text = convert_children(url).strip(" ")
    url = url.caption
    if len(text):
        return "[[%s|%s]]" % (url, text)
    else:
        return "%s" % (url)

@visitor.when(URL)
def convert(url, trailing_newline):
    return url.caption

@visitor.when(ImageLink)
def convert(link, trailing_newline):
    suffix = ""
    if link.width is not None:
        if link.height is None:
            suffix = "?%s" % link.width
        else:
            suffix = "?%sx%s" % (link.width, link.height)
    else:
        try:
            if link.in_gallery: # see below for Tag->gallery handling
                suffix = "?160" # gallery images should be thumbnailed
        except AttributeError:
            pass # not in a gallery
    prealign = " " if link.align in [ "center", "right" ] else ""
    postalign = " " if link.align in [ "center", "left" ] else ""
    target = canonicalise_file_namespace(link.target)
    target = convert_internal_link(target)
    return "{{%s%s%s%s}}" % (prealign, target, suffix, postalign)

@visitor.when(ArticleLink)
def convert(link, trailing_newline):
    text = convert_children(link).strip(" ")
    pagename = convert_internal_link(link.target)
    if len(text):
        return "[[%s|%s]]" % (pagename, text)
    else:
        return "[[%s]]" % pagename

@visitor.when(CategoryLink)
def convert(link, trailing_newline):
    # Category functionality can be implemented with plugin:tag, but not used here
    return ""

@visitor.when(NamespaceLink)
def convert(link, trailing_newline):
    if is_file_namespace(link.target): # is a link to a file or image
        filename = dokuwiki.make_dokuwiki_pagename(canonicalise_file_namespace(link.target))
        caption = convert_children(link).strip()
        if len(caption) > 0:
            return "{{%s%s}}" % (filename, caption)
        else:
            return "{{%s}}" % filename

    print("WARNING: Ignoring namespace link to " + link.target)
    return convert_children(link)

@visitor.when(ItemList)
def convert(itemlist, trailing_newline):
    def apply_itemlist_properties(node):
        # ItemLists are used for depth/style tracking - applies those attributes to all its children
        for child in node.children:
            try:
                child.list_style = "* " if itemlist.tagname == "ul" else "- "
                child.list_depth += 1
            except AttributeError:
                child.list_depth = 1
            apply_itemlist_properties(child)
    apply_itemlist_properties(itemlist)
    return convert_children(itemlist)

@visitor.when(Item)
def convert(item, trailing_newline):
    item_content = convert_children(item)
    return "  "*item.list_depth + item.list_style + item_content

@visitor.when(Table)
def convert(table, trailing_newline):
    # we ignore the actual Table tags, instead convert each Row & Cell individually
    return convert_children(table)

@visitor.when(Cell)
def convert(cell, trailing_newline):
    marker = "^" if cell.tagname == "th" else "|"
    result = "%s %s" % (marker, convert_children(cell).replace('\n','').strip())
    return result

@visitor.when(Row)
def convert(row, trailing_newline):
    return convert_children(row) + " |\n"

@visitor.when(PreFormatted)
def convert(pre, trailing_newline):
    try:
        in_list = item.list_depth > 0
    except:
        in_list = False
    if trailing_newline and not in_list: # in its own paragraph, use a two space indent
        return "  " + convert_children(pre).replace("\n","\n  ").strip(" ")
    else: # inline in a list or a paragraph body, use  tags
        return "" + convert_children(pre) + ""

@visitor.when(TagNode)
def convert(tag, trailing_newline):
    # dict maps mediawiki tag name to tuple of starting, ending dokuwiki tag
    simple_tagitems = {
        "tt" : ("''", "''"),
        "ref" : ("((","))"), # references converted to footnotes
        "code" : ("",""),
    }
    if tag.tagname in simple_tagitems:
        pre,post = simple_tagitems[tag.tagname]
        return pre + convert_children(tag) + post
    elif tag._text is not None:
        if tag._text.replace(" ","").replace("/","") == "
":
            return "\n" # this is a oneoff hack for one wiki page covered in

        return tag._text # may not work for non-selfclosing tags
    elif tag.tagname == "gallery":
        # with a lot of cleverness we could use the gallery plugin for this,
        # but this should do. We flag each child image as in the gallery, then
        # deal with the ImageLinks above
        for child in tag.children:
            child.in_gallery = True
    elif tag.tagname == "references":
        print("WARNING: <references> tag has no equivalent in Dokuwiki, ignoring...")

    return convert_children(tag)

# catchall for Node, which is the parent class of everything above
@visitor.when(Node)
def convert(node, trailing_newline):
    if node.__class__ != Node:
        print("WARNING: Unsupported node type: %s" % (node.__class__))
    return convert_children(node)

def convert_internal_link(mw_target):
    """
    Convert an internal Mediawiki link, with or without an anchor # in the middle.

    Same as converting a plain pagename, only we want to preserve any #s in the target text.
    """
    if "#" in mw_target:
        page,anchor = mw_target.split("#",1)
    else:
        page = mw_target
        anchor = None
    if len(page):
        page = dokuwiki.make_dokuwiki_pagename(page)
    if anchor is not None:
        page = page + "#" + dokuwiki.make_dokuwiki_heading_id(anchor)
    return page
<|endoftext|>


<commit_before>import asyncio
import time
import unittest
from test_sync import setup
from pyappbase import Appbase

## this test is more of a demo of actually how the async get stream works

async def hello_world():
    while True:
        await asyncio.sleep(0.1)
        print("hello world")

class AsnycStreamTests(unittest.TestCase):

    def setUp(self):
        self.data = {
            "type": "Books",
            "id": "X2",
        }
        self.appbase = setup(Appbase)
        self.appbase.set_async()
        self.sync_appbase = setup(Appbase)

        self.sync_appbase.update({
            "type": "Books",
            "id": "X2",
            "body": {
                "department_id": 1,
                "department_name": "Books",
                "name": "A Fake Book on Network Routing",
                "price": 5295
            }
        })

    def test_stream(self):

        loop = asyncio.get_event_loop()
        loop.create_task(self.appbase.get_stream({
                "type":"Books",
                "id":"1"
            },lambda word: print(word)))

        loop.run_until_complete(hello_world())<commit_msg>making the test not run for ever<commit_after>import asyncio
import time
import unittest
from test_sync import setup
from pyappbase import Appbase

## this test is more of a demo of actually how the async get stream works

async def hello_world():
    while True:
        await asyncio.sleep(0.1)
        print("hello world")

class AsnycStreamTests(unittest.TestCase):

    def setUp(self):
        self.data = {
            "type": "Books",
            "id": "X2",
        }
        self.appbase = setup(Appbase)
        self.appbase.set_async()
        self.sync_appbase = setup(Appbase)

        self.sync_appbase.update({
            "type": "Books",
            "id": "X2",
            "body": {
                "department_id": 1,
                "department_name": "Books",
                "name": "A Fake Book on Network Routing",
                "price": 5295
            }
        })

    def test_stream(self):

        loop = asyncio.get_event_loop()
        loop.create_task(self.appbase.get_stream({
                "type":"Books",
                "id":"1"
            },lambda word: print(word)))

        loop.create_task(hello_world())
        loop.run_until_complete(asyncio.sleep(5))<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
#Copyright (C) 2009 sp00b, sebnapi, RaNaN
#
#This program is free software; you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation; either version 3 of the License,
#or (at your option) any later version.
#
#This program is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#See the GNU General Public License for more details.
#
#You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.
#
###
CURRENT_VERSION = '0.1'

import gettext

#python imports
import ConfigParser
import logging
import logging.handlers
import time
import urllib2
from glob import glob
from os import mkdir
from os import sep
from os.path import basename
from os.path import exists
from sys import exit
from sys import path
from sys import stdout
from time import sleep

from module.Py_Load_File import PyLoadFile
from module.remote.SocketServer import ServerThread
from module.thread_list import Thread_List

class Core(object):
    """ pyLoad main
    """
    def __init__(self):
        self.config = {}
        self.config['plugin_folder'] = "Plugins"
        self.config['link_file'] = "links.txt"
        self.plugins_avaible = {}
        self.search_updates = False

        self.read_config()

        translation = gettext.translation("pyLoad", "locale", languages=[self.config['language']])
        translation.install(unicode=True)

        self.init_logger(logging.DEBUG) # logging level

        self.check_update()

        self.logger.info(_("Downloadtime: %s") % self.is_dltime()) # debug only

        self.thread_list = Thread_List(self)

        #self.check_create(self.config['log_folder'], _("folder for logs"))
        self.check_create(self.config['download_folder'], _("folder for downloads"))
        self.check_create(self.config['link_file'], _("file for links"), False)

        path.append(self.config['plugin_folder'])
        self.create_plugin_index()

    def read_config(self):
        """ read config and sets preferences
        """
        config = ConfigParser.SafeConfigParser()
        config.read('config')

        for section in config.sections():
            for option in config.options(section):
                self.config[option] = config.get(section, option)
                self.config[option] = False if self.config[option].lower() == 'false' else self.config[option]

        self.config['language'] = config.get('general', 'language')
        self.config['download_folder'] = config.get('general', 'download_folder')
        self.config['link_file'] = config.get('general', 'link_file')
        self.config['search_updates'] = config.getboolean('updates', 'search_updates')
        self.config['log_folder'] = config.get('log', 'log_folder')
        self.config['reconnectMethod'] = config.get('general', 'reconnect_method')

    def create_plugin_index(self):
        for file_handler in glob(self.config['plugin_folder'] + sep + '*.py'):
            if file_handler != self.config['plugin_folder'] + sep + "Plugin.py":
                plugin_pattern = ""
                plugin_file = basename(file_handler).replace('.py', '')
                for line in open(file_handler, "r").readlines():
                    if "props['pattern']" in line:
                        plugin_pattern = line.split("r\"")[1].split("\"")[0]
                if plugin_pattern != "":
                    self.plugins_avaible[plugin_file] = plugin_pattern
                    self.logger.debug(plugin_file + _(" added"))
        self.logger.info(_("created index of plugins"))

##    def check_needed_plugins(self):
##        links = open(self.link_file, 'r').readlines()
##        plugins_indexed = pickle.load(open(self.plugin_index, "r"))
##        for link in links:
##            link = link.replace("\n", "")
##            for plugin_file in plugins_indexed.keys():
##                if re.search(plugins_indexed[plugin_file], link) != None:
##                    self.plugins_needed[plugin_file] = plugins_indexed[plugin_file]
##        print "Benoetigte Plugins: " + str(self.plugins_needed.keys())
##
##    def import_needed_plugins(self):
##        for needed_plugin in self.plugins_needed.keys():
##            self.import_plugin(needed_plugin)
##
##    def import_plugin(self, needed_plugin):
##        try:
##            new_plugin = __import__(needed_plugin)
##            self.plugins_dict[new_plugin] = self.plugins_needed[needed_plugin]
##            #if new_plugin.plugin_type in "hoster" or new_plugin.plugin_type in "container":
##            #   print "Plugin geladen: " + new_plugin.plugin_name
##            #plugins[plugin_file] = __import__(plugin_file)
##        except:
##            print "Fehlerhaftes Plugin: " + needed_plugin
##

    def _get_links(self, link_file):
        """ funktion nur zum testen ohne gui bzw. tui
        """
        links = open(link_file, 'r').readlines()
        self.extend_links(links)

    def append_link(self, link):
        if link not in self.thread_list.get_loaded_urls():
            self.__new_py_load_file(link.replace("\n",""))

    def extend_links(self, links):
        for link in links:
            self.append_link(link)

    def check_update(self):
        """checks newst version
        """
        newst_version = urllib2.urlopen("http://pyload.nady.biz/files/version.txt").readline().strip()
        if CURRENT_VERSION < newst_version:
            self.logger.info(_("new update %s on pyload.org") % newst_version) #newer version out
        elif CURRENT_VERSION == newst_version:
            self.logger.info(_("newst version %s in use:") % CURRENT_VERSION) #using newst version
        else:
            self.logger.info( _("beta version %s in use:") % CURRENT_VERSION) #using beta version

    def check_create(self, check_name, legend, folder=True):
        if not exists(check_name):
            try:
                if folder:
                    mkdir(check_name)
                else:
                    open(check_name, "w")
                print _("%s created") % legend
            except:
                print _("could %s not create ") % legend
                exit()

    #def addLinks(self, newLinks, atTheBeginning):
        #pass

#    def get_hoster(self, url):
#        """ searches the right plugin for an url
#        """
#        for plugin, plugin_pattern in self.plugins_avaible.items():
#            if re.match(plugin_pattern, url) != None: #guckt ob übergebende url auf muster des plugins passt
#                return plugin
#        #logger: kein plugin gefunden
#        return None

    def __new_py_load_file(self, url):
        new_file = PyLoadFile(self, url)
        new_file.download_folder = self.config['download_folder']
        self.thread_list.append_py_load_file(new_file)
        return True

    def init_logger(self, level):
        handler = logging.handlers.RotatingFileHandler(self.config['log_folder'] + sep + 'log.txt', maxBytes=12800, backupCount=10) #100 kb
        console = logging.StreamHandler(stdout)
        #handler = logging.FileHandler('Logs/log.txt')
        frm = logging.Formatter("%(asctime)s: %(levelname)-8s  %(message)s", "%d.%m.%Y %H:%M:%S")
        handler.setFormatter(frm)
        console.setFormatter(frm)

        self.logger = logging.getLogger() # settable in config
        self.logger.addHandler(handler)
        self.logger.addHandler(console) #if console logging
        self.logger.setLevel(level)

    def is_dltime(self):
        start_h, start_m = self.config['start'].split(":")
        end_h, end_m = self.config['end'].split(":")

        #@todo: little bug, when start and end time in same hour
        hour, minute  = time.localtime()[3:5]

        if hour > int(start_h) and hour < int(end_h):
            return True
        elif hour == int(start_h) and minute >= int(start_m):
            return True
        elif hour == int(end_h) and minute <= int(end_m):
            return True
        else:
            return False

    def get_downloads(self): #only for debuging?!?
        list = []
        for pyfile in self.thread_list.py_downloading:
            download = {}
            download['name'] = pyfile.status.filename
            download['speed'] = pyfile.status.get_speed()
            download['eta'] = pyfile.status.get_ETA()
            list.append(download)

        return list

    def _test_print_status(self):
        if self.thread_list.py_downloading:

            for pyfile in self.thread_list.py_downloading:
                if pyfile.status.type == 'downloading':
                    print pyfile.status.filename + ": speed is", int(pyfile.status.get_speed()), "kb/s"
                    print pyfile.status.filename + ": finished in", int(pyfile.status.get_ETA()), "seconds"
                elif pyfile.status.type == 'waiting':
                    print pyfile.status.filename + ": wait", int(pyfile.status.waituntil - time.time()), "seconds"

    def start(self):
        """ starts the machine
        """
        self._get_links(self.config['link_file'])
        while True:
            #self.thread_list.status()
            self._test_print_status()
            sleep(2)
            if len(self.thread_list.threads) == 0:
                pass #break

if __name__ == "__main__":

    testLoader = Core()
    #if testLoader.config['remote_activated']:
    #@spoob: lass den server an
    print _("Server Mode")
    server = ServerThread(testLoader)
    server.start()

    testLoader.start()
<commit_msg>fixed no log folder bug<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
#Copyright (C) 2009 sp00b, sebnapi, RaNaN
#
#This program is free software; you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation; either version 3 of the License,
#or (at your option) any later version.
#
#This program is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#See the GNU General Public License for more details.
#
#You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.
#
###
CURRENT_VERSION = '0.1'

import gettext

#python imports
import ConfigParser
import logging
import logging.handlers
import time
import urllib2
from glob import glob
from os import mkdir
from os import sep
from os.path import basename
from os.path import exists
from sys import exit
from sys import path
from sys import stdout
from time import sleep

from module.Py_Load_File import PyLoadFile
from module.remote.SocketServer import ServerThread
from module.thread_list import Thread_List

class Core(object):
    """ pyLoad main
    """
    def __init__(self):
        self.config = {}
        self.config['plugin_folder'] = "Plugins"
        self.config['link_file'] = "links.txt"
        self.plugins_avaible = {}
        self.search_updates = False

        self.read_config()

        translation = gettext.translation("pyLoad", "locale", languages=[self.config['language']])
        translation.install(unicode=True)

        self.check_create(self.config['log_folder'], _("folder for logs"))
        self.init_logger(logging.DEBUG) # logging level

        self.check_update()

        self.logger.info(_("Downloadtime: %s") % self.is_dltime()) # debug only

        self.thread_list = Thread_List(self)

        self.check_create(self.config['download_folder'], _("folder for downloads"))
        self.check_create(self.config['link_file'], _("file for links"), False)

        path.append(self.config['plugin_folder'])
        self.create_plugin_index()

    def read_config(self):
        """ read config and sets preferences
        """
        config = ConfigParser.SafeConfigParser()
        config.read('config')

        for section in config.sections():
            for option in config.options(section):
                self.config[option] = config.get(section, option)
                self.config[option] = False if self.config[option].lower() == 'false' else self.config[option]

        self.config['language'] = config.get('general', 'language')
        self.config['download_folder'] = config.get('general', 'download_folder')
        self.config['link_file'] = config.get('general', 'link_file')
        self.config['search_updates'] = config.getboolean('updates', 'search_updates')
        self.config['log_folder'] = config.get('log', 'log_folder')
        self.config['reconnectMethod'] = config.get('general', 'reconnect_method')

    def create_plugin_index(self):
        for file_handler in glob(self.config['plugin_folder'] + sep + '*.py'):
            if file_handler != self.config['plugin_folder'] + sep + "Plugin.py":
                plugin_pattern = ""
                plugin_file = basename(file_handler).replace('.py', '')
                for line in open(file_handler, "r").readlines():
                    if "props['pattern']" in line:
                        plugin_pattern = line.split("r\"")[1].split("\"")[0]
                if plugin_pattern != "":
                    self.plugins_avaible[plugin_file] = plugin_pattern
                    self.logger.debug(plugin_file + _(" added"))
        self.logger.info(_("created index of plugins"))

##    def check_needed_plugins(self):
##        links = open(self.link_file, 'r').readlines()
##        plugins_indexed = pickle.load(open(self.plugin_index, "r"))
##        for link in links:
##            link = link.replace("\n", "")
##            for plugin_file in plugins_indexed.keys():
##                if re.search(plugins_indexed[plugin_file], link) != None:
##                    self.plugins_needed[plugin_file] = plugins_indexed[plugin_file]
##        print "Benoetigte Plugins: " + str(self.plugins_needed.keys())
##
##    def import_needed_plugins(self):
##        for needed_plugin in self.plugins_needed.keys():
##            self.import_plugin(needed_plugin)
##
##    def import_plugin(self, needed_plugin):
##        try:
##            new_plugin = __import__(needed_plugin)
##            self.plugins_dict[new_plugin] = self.plugins_needed[needed_plugin]
##            #if new_plugin.plugin_type in "hoster" or new_plugin.plugin_type in "container":
##            #   print "Plugin geladen: " + new_plugin.plugin_name
##            #plugins[plugin_file] = __import__(plugin_file)
##        except:
##            print "Fehlerhaftes Plugin: " + needed_plugin
##

    def _get_links(self, link_file):
        """ funktion nur zum testen ohne gui bzw. tui
        """
        links = open(link_file, 'r').readlines()
        self.extend_links(links)

    def append_link(self, link):
        if link not in self.thread_list.get_loaded_urls():
            self.__new_py_load_file(link.replace("\n",""))

    def extend_links(self, links):
        for link in links:
            self.append_link(link)

    def check_update(self):
        """checks newst version
        """
        newst_version = urllib2.urlopen("http://pyload.nady.biz/files/version.txt").readline().strip()
        if CURRENT_VERSION < newst_version:
            self.logger.info(_("new update %s on pyload.org") % newst_version) #newer version out
        elif CURRENT_VERSION == newst_version:
            self.logger.info(_("newst version %s in use:") % CURRENT_VERSION) #using newst version
        else:
            self.logger.info( _("beta version %s in use:") % CURRENT_VERSION) #using beta version

    def check_create(self, check_name, legend, folder=True):
        if not exists(check_name):
            try:
                if folder:
                    mkdir(check_name)
                else:
                    open(check_name, "w")
                print _("%s created") % legend
            except:
                print _("could %s not create ") % legend
                exit()

    #def addLinks(self, newLinks, atTheBeginning):
        #pass

#    def get_hoster(self, url):
#        """ searches the right plugin for an url
#        """
#        for plugin, plugin_pattern in self.plugins_avaible.items():
#            if re.match(plugin_pattern, url) != None: #guckt ob übergebende url auf muster des plugins passt
#                return plugin
#        #logger: kein plugin gefunden
#        return None

    def __new_py_load_file(self, url):
        new_file = PyLoadFile(self, url)
        new_file.download_folder = self.config['download_folder']
        self.thread_list.append_py_load_file(new_file)
        return True

    def init_logger(self, level):
        handler = logging.handlers.RotatingFileHandler(self.config['log_folder'] + sep + 'log.txt', maxBytes=12800, backupCount=10) #100 kb
        console = logging.StreamHandler(stdout)
        #handler = logging.FileHandler('Logs/log.txt')
        frm = logging.Formatter("%(asctime)s: %(levelname)-8s  %(message)s", "%d.%m.%Y %H:%M:%S")
        handler.setFormatter(frm)
        console.setFormatter(frm)

        self.logger = logging.getLogger() # settable in config
        self.logger.addHandler(handler)
        self.logger.addHandler(console) #if console logging
        self.logger.setLevel(level)

    def is_dltime(self):
        start_h, start_m = self.config['start'].split(":")
        end_h, end_m = self.config['end'].split(":")

        #@todo: little bug, when start and end time in same hour
        hour, minute  = time.localtime()[3:5]

        if hour > int(start_h) and hour < int(end_h):
            return True
        elif hour == int(start_h) and minute >= int(start_m):
            return True
        elif hour == int(end_h) and minute <= int(end_m):
            return True
        else:
            return False

    def get_downloads(self): #only for debuging?!?
        list = []
        for pyfile in self.thread_list.py_downloading:
            download = {}
            download['name'] = pyfile.status.filename
            download['speed'] = pyfile.status.get_speed()
            download['eta'] = pyfile.status.get_ETA()
            list.append(download)

        return list

    def _test_print_status(self):
        if self.thread_list.py_downloading:

            for pyfile in self.thread_list.py_downloading:
                if pyfile.status.type == 'downloading':
                    print pyfile.status.filename + ": speed is", int(pyfile.status.get_speed()), "kb/s"
                    print pyfile.status.filename + ": finished in", int(pyfile.status.get_ETA()), "seconds"
                elif pyfile.status.type == 'waiting':
                    print pyfile.status.filename + ": wait", int(pyfile.status.waituntil - time.time()), "seconds"

    def start(self):
        """ starts the machine
        """
        self._get_links(self.config['link_file'])
        while True:
            #self.thread_list.status()
            self._test_print_status()
            sleep(2)
            if len(self.thread_list.threads) == 0:
                pass #break

if __name__ == "__main__":

    testLoader = Core()
    #if testLoader.config['remote_activated']:
    #@spoob: lass den server an
    print _("Server Mode")
    server = ServerThread(testLoader)
    server.start()

    testLoader.start()
<|endoftext|>


<commit_before>from django.core.exceptions import ValidationError
from django.core.files.storage import DefaultStorage
from django.db.models.fields import TextField
from django.db.models.fields.files import ImageFieldFile
from imagekit.generators import SpecFileGenerator
from imagekit.processors.crop import Crop
import jsonfield
import os

def upload_to(instance, filename, crop_name):
    """
    Default function to specify a location to save crops to.

    :param instance: The model instance this crop field belongs to.
    :param filename: The image's filename this crop field operates on.
    :param crop_name: The crop name used when :attr:`CropFieldDescriptor.crop` was
        called.
    """
    filename, ext = os.path.splitext(os.path.split(filename)[-1])
    return os.path.join('crops', u'%s-%s%s' % (filename, crop_name, ext))

class CropFieldFile(ImageFieldFile):
    """
    :attr:`CropFieldFile` objects are attached to a model's crop descriptor for each
    specified crop spec.
    """

    def delete(self, save=True):
        """
        Overriding delete method to not reset the model instance with an empty
        field. This would override all our other crops and the JSON metadata.

        .. note:: This method should not be called directly, rather use
            :attr:`CropFieldDescriptor.delete`.

        :param save: When true, the instance is saved to the database after
            the crop was deleted.
        """
        if hasattr(self, '_file'):
            self.close()
            del self.file

        self.storage.delete(self.name)

        if save:
            self.instance.save()

        self._commited = False

class CropFieldDescriptor(object):
    """
    Crop descriptors are created by :attr:`CropField` and allow for creating,
    inspecting and deleting crops.
    """
    def __init__(self, instance, field, data):
        self.instance = instance
        self.field = field
        self.image = getattr(instance, field.image_field)

        self._data = {}
        self.data = data

    def create(self, name, spec, save=True):
        """
        Create a new crop with the provided spec. For example the following code
        creates a crop of the original image starting at the X/Y coordinates of
        0/0 and having a width of 100 pixels and a height of 150 pixels::

            >>> image.crops.create('thumbnail', (0, 0, 100, 150))

        :param name: Crop name. This must be unique and is also used to generate
            the filename.
        :param spec: 4-tuple containing ``(x, y, width, height)``
        :param save: Boolean, if specified the model is saved back to DB after the crop.
        """

        self.validate_name(name)
        (x, y, width, height) = spec
        spec = {name: dict(x=x, y=y, width=width, height=height)}

        processors = [Crop(**spec[name])]

        filename = self.get_filename(name)

        spec[name]['filename'] = filename

        generator = SpecFileGenerator(processors, storage=self.field.storage)

        generator.generate_file(filename, self.image)

        self.data = dict(self.data, **spec)

        if save:
            self.instance.save()

    def delete(self, name, save=True):
        """
        Delete the named crop. If save is specified, the model instance is saved
        after the deletion.

        :param name: Crop name
        :param save: Boolean, whether to save the model instance or not after
            deleting the file.
        """
        crop = getattr(self, name)
        delattr(self, name)
        del self._data[name]
        crop.delete(save)

    def validate_name(self, name):
        """
        Makes sure that the crop name does not clash with any methods or
        attributes on :attr:`CropFieldDescriptor`.

        Raises :attr:`django.core.exceptions.ValidationError` in case there is
        a clash.
        """

        if hasattr(self, name) and not isinstance(getattr(self, name), CropFieldFile):
            raise ValidationError(
                "Cannot override existing attribute '%s' with crop file." % name
            )
    def get_filename(self, name):
        """
        Delegate filename creation to :attr:`field.upload_to`.
        """
        return self.field.upload_to(self.instance, self.image.name, name)

    @property
    def data(self):
        """
        Return the raw picture data as a dictionary including path, coordinates, e.g.:

            >>> model.crops.data
            {
                'thumbnail': {
                    'x': 0, 'y': 0,
                    'width': 100, 'height': 100,
                    'name': 'crops/image_thumbnail.png'
                },
                'skyscraper': {
                    'x': 0, 'y': 0,
                    'width': 100, height: 600,
                    'name': 'crops/image_skyscraper.png'
                }
            }

        """
        return self._data

    @data.setter
    def data(self, value):
        """
        Sets the data attribute and generates :attr:`CropFieldFiles`
        for convenience methods and attributes like :attr:`CropFieldFiles.delete`,
        :attr:`CropFieldFiles.url`, :attr:`CropFieldFiles.path`, etc.
        """
        for name, spec in value.iteritems():
            self.validate_name(name)
            self._data[name] = spec

            if hasattr(self, name):
                continue

            setattr(self, name, CropFieldFile(
                self.instance,
                self.field,
                spec['filename']))

    def __iter__(self):
        for key in self.data:
            yield getattr(self, key)

    def __len__(self):
        return len(self.data.keys())

class CropFieldCreator(object):
    """
    Instead of using a metaclass for our custom field, we kind of
    follow :attr:`django.db.fields.files` in how to set up custom field
    attributes on model objects.
    """

    def __init__(self, field):
        self.field = field

    def __get__(self, instance, cls):
        if instance is None:
            return self.field
        return instance.__dict__[self.field.name]

    def __set__(self, instance, data):
        """
        Turn data from string into Python and store the CropFieldDescriptor
        on the instance
        """
        instance.__dict__[self.field.name] = CropFieldDescriptor(instance, self.field,
            self.field.to_python(data))

class CropField(TextField):
    """
    Creates custom crops from a specified model field::

        class MyModel(models.Model):
            my_image = models.ImageField()
            my_crops = CropField('my_image')

    """

    def __init__(self, image_field=None, storage=DefaultStorage(),
        upload_to=upload_to, editable=False, *args, **kwargs):
        """
        Custom field to generate crops of custom sizes in custom locations.

        :param image_field: The name of the image field this cropper operates on.
        :param storage: A custom storage backend to use.
        :param upload_to: A custom function to generate crop filenames. Must take
            three attributes, ``instance``, ``image`` and ``crop_name``. See
            :attr:`upload_to`.
        """
        assert image_field is not None, "You must specify an 'image_field' parameter."

        self.image_field = image_field

        self.storage = storage
        self.upload_to = upload_to

        kwargs['editable'] = editable

        self.json_field = jsonfield.JSONField(*args, **kwargs)

        kwargs['default'] = self.json_field.default

        super(CropField, self).__init__(*args, **kwargs)

    def contribute_to_class(self, cls, name):
        super(CropField, self).contribute_to_class(cls, name)
        setattr(cls, name, CropFieldCreator(self))

    def get_db_prep_value(self, value, **kwargs):
        return self.json_field.get_db_prep_value(value.data, **kwargs)

    def to_python(self, value):
        return self.json_field.to_python(value)

try:
    from south.modelsinspector import add_introspection_rules

    rules = [
        (
            (CropField,),
            [],  # Args
            {  # Kwargs
                "image_field": ["image_field", {}],
            }
        ),
    ]
    add_introspection_rules(rules, ["^croppy\.fields\.CropField"])
except ImportError:
    pass
<commit_msg>CropFieldFile initialised with data about itself<commit_after>from django.core.exceptions import ValidationError
from django.core.files.storage import DefaultStorage
from django.db.models.fields import TextField
from django.db.models.fields.files import ImageFieldFile
from imagekit.generators import SpecFileGenerator
from imagekit.processors.crop import Crop
import jsonfield
import os

def upload_to(instance, filename, crop_name):
    """
    Default function to specify a location to save crops to.

    :param instance: The model instance this crop field belongs to.
    :param filename: The image's filename this crop field operates on.
    :param crop_name: The crop name used when :attr:`CropFieldDescriptor.crop` was
        called.
    """
    filename, ext = os.path.splitext(os.path.split(filename)[-1])
    return os.path.join('crops', u'%s-%s%s' % (filename, crop_name, ext))

class CropFieldFile(ImageFieldFile):
    """
    :attr:`CropFieldFile` objects are attached to a model's crop descriptor for each
    specified crop spec.
    """
    def __init__(self, crop_name, spec, *args, **kwargs):
        self.crop_name = crop_name
        self.spec = spec
        super(CropFieldFile, self).__init__(*args, **kwargs)

    def delete(self, save=True):
        """
        Overriding delete method to not reset the model instance with an empty
        field. This would override all our other crops and the JSON metadata.

        .. note:: This method should not be called directly, rather use
            :attr:`CropFieldDescriptor.delete`.

        :param save: When true, the instance is saved to the database after
            the crop was deleted.
        """
        if hasattr(self, '_file'):
            self.close()
            del self.file

        self.storage.delete(self.name)

        if save:
            self.instance.save()

        self._commited = False

class CropFieldDescriptor(object):
    """
    Crop descriptors are created by :attr:`CropField` and allow for creating,
    inspecting and deleting crops.
    """
    def __init__(self, instance, field, data):
        self.instance = instance
        self.field = field
        self.image = getattr(instance, field.image_field)

        self._data = {}
        self.data = data

    def create(self, name, spec, save=True):
        """
        Create a new crop with the provided spec. For example the following code
        creates a crop of the original image starting at the X/Y coordinates of
        0/0 and having a width of 100 pixels and a height of 150 pixels::

            >>> image.crops.create('thumbnail', (0, 0, 100, 150))

        :param name: Crop name. This must be unique and is also used to generate
            the filename.
        :param spec: 4-tuple containing ``(x, y, width, height)``
        :param save: Boolean, if specified the model is saved back to DB after the crop.
        """

        self.validate_name(name)
        (x, y, width, height) = spec
        spec = {name: dict(x=x, y=y, width=width, height=height)}

        processors = [Crop(**spec[name])]

        filename = self.get_filename(name)

        spec[name]['filename'] = filename

        generator = SpecFileGenerator(processors, storage=self.field.storage)

        generator.generate_file(filename, self.image)

        self.data = dict(self.data, **spec)

        if save:
            self.instance.save()

    def delete(self, name, save=True):
        """
        Delete the named crop. If save is specified, the model instance is saved
        after the deletion.

        :param name: Crop name
        :param save: Boolean, whether to save the model instance or not after
            deleting the file.
        """
        crop = getattr(self, name)
        delattr(self, name)
        del self._data[name]
        crop.delete(save)

    def validate_name(self, name):
        """
        Makes sure that the crop name does not clash with any methods or
        attributes on :attr:`CropFieldDescriptor`.

        Raises :attr:`django.core.exceptions.ValidationError` in case there is
        a clash.
        """

        if hasattr(self, name) and not isinstance(getattr(self, name), CropFieldFile):
            raise ValidationError(
                "Cannot override existing attribute '%s' with crop file." % name
            )
    def get_filename(self, name):
        """
        Delegate filename creation to :attr:`field.upload_to`.
        """
        return self.field.upload_to(self.instance, self.image.name, name)

    @property
    def data(self):
        """
        Return the raw picture data as a dictionary including path, coordinates, e.g.:

            >>> model.crops.data
            {
                'thumbnail': {
                    'x': 0, 'y': 0,
                    'width': 100, 'height': 100,
                    'name': 'crops/image_thumbnail.png'
                },
                'skyscraper': {
                    'x': 0, 'y': 0,
                    'width': 100, height: 600,
                    'name': 'crops/image_skyscraper.png'
                }
            }

        """
        return self._data

    @data.setter
    def data(self, value):
        """
        Sets the data attribute and generates :attr:`CropFieldFiles`
        for convenience methods and attributes like :attr:`CropFieldFiles.delete`,
        :attr:`CropFieldFiles.url`, :attr:`CropFieldFiles.path`, etc.
        """
        for name, spec in value.iteritems():
            self.validate_name(name)
            self._data[name] = spec

            if hasattr(self, name):
                continue

            setattr(self, name, CropFieldFile(
                name, spec,
                self.instance,
                self.field,
                spec['filename']))

    def __iter__(self):
        for key in self.data:
            yield getattr(self, key)

    def __len__(self):
        return len(self.data.keys())

class CropFieldCreator(object):
    """
    Instead of using a metaclass for our custom field, we kind of
    follow :attr:`django.db.fields.files` in how to set up custom field
    attributes on model objects.
    """

    def __init__(self, field):
        self.field = field

    def __get__(self, instance, cls):
        if instance is None:
            return self.field
        return instance.__dict__[self.field.name]

    def __set__(self, instance, data):
        """
        Turn data from string into Python and store the CropFieldDescriptor
        on the instance
        """
        instance.__dict__[self.field.name] = CropFieldDescriptor(instance, self.field,
            self.field.to_python(data))

class CropField(TextField):
    """
    Creates custom crops from a specified model field::

        class MyModel(models.Model):
            my_image = models.ImageField()
            my_crops = CropField('my_image')

    """

    def __init__(self, image_field=None, storage=DefaultStorage(),
        upload_to=upload_to, editable=False, *args, **kwargs):
        """
        Custom field to generate crops of custom sizes in custom locations.

        :param image_field: The name of the image field this cropper operates on.
        :param storage: A custom storage backend to use.
        :param upload_to: A custom function to generate crop filenames. Must take
            three attributes, ``instance``, ``image`` and ``crop_name``. See
            :attr:`upload_to`.
        """
        assert image_field is not None, "You must specify an 'image_field' parameter."

        self.image_field = image_field

        self.storage = storage
        self.upload_to = upload_to

        kwargs['editable'] = editable

        self.json_field = jsonfield.JSONField(*args, **kwargs)

        kwargs['default'] = self.json_field.default

        super(CropField, self).__init__(*args, **kwargs)

    def contribute_to_class(self, cls, name):
        super(CropField, self).contribute_to_class(cls, name)
        setattr(cls, name, CropFieldCreator(self))

    def get_db_prep_value(self, value, **kwargs):
        return self.json_field.get_db_prep_value(value.data, **kwargs)

    def to_python(self, value):
        return self.json_field.to_python(value)

try:
    from south.modelsinspector import add_introspection_rules

    rules = [
        (
            (CropField,),
            [],  # Args
            {  # Kwargs
                "image_field": ["image_field", {}],
            }
        ),
    ]
    add_introspection_rules(rules, ["^croppy\.fields\.CropField"])
except ImportError:
    pass
<|endoftext|>