<commit_before># updates all DeVIDE dependencies except ITK and VTK
# i.e. do a cvs update, a cmake and a build of:
# vtkdevide, vtktud
# TODO: ConnectVTKITK
# $Id: updateAll.py,v 1.7 2005/05/12 16:00:09 cpbotha Exp $
# authored by Charl P. Botha http://cpbotha.net/

import getopt
import os
import sys

# There should be an updateAllDefaults.py file in the same directory
# as this script.  An example would be:
# --- BEGIN updateAllDefaults.py
# devideTopLevel = '/home/cpbotha/work/code/devide'
# vtkdevideSource = '/home/cpbotha/work/code/vtkdevide'
# vtkdevideBinary = '/home/cpbotha/work/code/vtkdevide'
# vtktudSource = '/home/cpbotha/work/code/vtktud'
# vtktudBinary = '/home/cpbotha/work/code/vtktud'
# wrapitkSource = '/home/cpbotha/work/code/wrapitk'
# wrapitkBinary = '/home/cpbotha/work/code/wrapitk-gcc'
# --- END updateAllDefaults
try:
    import updateAllDefaults as defaults
except ImportError, e:
    print 'Error loading updateAllDefaults.py: %s' % (str(e),)
    sys.exit()

# buildcommands for posix and MSVS 7.1 on Windows
if os.name == 'posix':
    buildCommand = 'make'
else:
    # first %s is to be replaced by the .sln file
    buildCommand = 'devenv %s /project ALL_BUILD /projectconfig ' \
                   '"RelWithDebInfo|Win32" /build RelWithDebInfo'

cvsUpdateCommand = 'cvs -z3 update -dAP'

def standardUpdate(source):
    print '\nCVS updating %s\n' % (source,)
    os.chdir(source)
    os.system(cvsUpdateCommand)

def standardBuild(binary, slnFile):
    print '\nUpdating build config in %s' % (binary,)
    os.chdir(binary)
    os.system('cmake .')

    if os.name == 'posix':
        theBuildCommand = buildCommand
    else:
        theBuildCommand = buildCommand % (slnFile,)

    print 'Building with "%s"\n' % (theBuildCommand,)

    os.system(theBuildCommand)

def doUpdates(itkRequested):
    standardUpdate(defaults.devideTopLevel)
    standardUpdate(defaults.vtkdevideSource)
    standardUpdate(defaults.vtktudSource)
    if itkRequested:
        #standardUpdate(defaults.wrapitkSource)
        pass

def doBuilds(itkRequested):
    standardBuild(defaults.vtkdevideBinary, 'vtkdevide.sln')
    standardBuild(defaults.vtktudBinary, 'vtktud.sln')
    if itkRequested:
        #standardBuild(defaults.wrapitkBinary, 'wrapitk.sln')
        pass

def dispUsage():
    print "-h or --help               : Display this message."
    print "--noUpdates                : Do not do any CVS updates."
    print "--noBuilds                 : Do not do any builds."
    print "--noItk or --no-itk        : Do not build any itk-related source."

def main():
    updatesRequested = True
    buildsRequested = True
    itkRequested = True

    try:
        # 'p:' means -p with something after
        optlist, args = getopt.getopt(
            sys.argv[1:], 'h',
            ['help', 'noUpdates', 'noBuilds', 'noItk', 'no-itk'])

    except getopt.GetoptError,e:
        dispUsage()
        sys.exit(1)

    for o, a in optlist:
        if o in ('-h', '--help'):
            dispUsage()
            sys.exit(1)

        elif o in ('--noUpdates'):
            updatesRequested = False

        elif o in ('--noBuilds'):
            buildsRequested = False

        elif o in ('--noItk', '--no-itk'):
            itkRequested = False

    if updatesRequested:
        doUpdates(itkRequested)

    if buildsRequested:
        doBuilds(itkRequested)

if __name__ == '__main__':
    main()

<commit_msg>ERR: Fixed bug with in (tuple,) checking.<commit_after># updates all DeVIDE dependencies except ITK and VTK
# i.e. do a cvs update, a cmake and a build of:
# vtkdevide, vtktud
# TODO: ConnectVTKITK
# $Id: updateAll.py,v 1.8 2005/07/04 23:29:06 cpbotha Exp $
# authored by Charl P. Botha http://cpbotha.net/

import getopt
import os
import sys

# There should be an updateAllDefaults.py file in the same directory
# as this script.  An example would be:
# --- BEGIN updateAllDefaults.py
# devideTopLevel = '/home/cpbotha/work/code/devide'
# vtkdevideSource = '/home/cpbotha/work/code/vtkdevide'
# vtkdevideBinary = '/home/cpbotha/work/code/vtkdevide'
# vtktudSource = '/home/cpbotha/work/code/vtktud'
# vtktudBinary = '/home/cpbotha/work/code/vtktud'
# wrapitkSource = '/home/cpbotha/work/code/wrapitk'
# wrapitkBinary = '/home/cpbotha/work/code/wrapitk-gcc'
# --- END updateAllDefaults
try:
    import updateAllDefaults as defaults
except ImportError, e:
    print 'Error loading updateAllDefaults.py: %s' % (str(e),)
    sys.exit()

# buildcommands for posix and MSVS 7.1 on Windows
if os.name == 'posix':
    buildCommand = 'make'
else:
    # first %s is to be replaced by the .sln file
    buildCommand = 'devenv %s /project ALL_BUILD /projectconfig ' \
                   '"RelWithDebInfo|Win32" /build RelWithDebInfo'

cvsUpdateCommand = 'cvs -z3 update -dAP'

def standardUpdate(source):
    print '\nCVS updating %s\n' % (source,)
    os.chdir(source)
    os.system(cvsUpdateCommand)

def standardBuild(binary, slnFile):
    print '\nUpdating build config in %s' % (binary,)
    os.chdir(binary)
    os.system('cmake .')

    if os.name == 'posix':
        theBuildCommand = buildCommand
    else:
        theBuildCommand = buildCommand % (slnFile,)

    print 'Building with "%s"\n' % (theBuildCommand,)

    os.system(theBuildCommand)

def doUpdates(itkRequested):
    standardUpdate(defaults.devideTopLevel)
    standardUpdate(defaults.vtkdevideSource)
    standardUpdate(defaults.vtktudSource)
    if itkRequested:
        #standardUpdate(defaults.wrapitkSource)
        pass

def doBuilds(itkRequested):
    standardBuild(defaults.vtkdevideBinary, 'vtkdevide.sln')
    standardBuild(defaults.vtktudBinary, 'vtktud.sln')
    if itkRequested:
        #standardBuild(defaults.wrapitkBinary, 'wrapitk.sln')
        pass

def dispUsage():
    print "-h or --help               : Display this message."
    print "--noUpdates                : Do not do any CVS updates."
    print "--noBuilds                 : Do not do any builds."
    print "--noItk or --no-itk        : Do not build any itk-related source."

def main():
    updatesRequested = True
    buildsRequested = True
    itkRequested = True

    try:
        # 'p:' means -p with something after
        optlist, args = getopt.getopt(
            sys.argv[1:], 'h',
            ['help', 'noUpdates', 'noBuilds', 'noItk', 'no-itk'])

    except getopt.GetoptError,e:
        dispUsage()
        sys.exit(1)

    for o, a in optlist:
        if o in ('-h', '--help'):
            dispUsage()
            sys.exit(1)

        elif o in ('--noUpdates',):
            updatesRequested = False

        elif o in ('--noBuilds',):
            buildsRequested = False

        elif o in ('--noItk', '--no-itk'):
            itkRequested = False

    if updatesRequested:
        doUpdates(itkRequested)

    if buildsRequested:
        doBuilds(itkRequested)

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>"""
Rank all available models

@copyright: The Broad Institute of MIT and Harvard 2015
"""

import argparse
import os, csv
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns

'''
src_file = "./data/sources.txt"
range_file = "./data/ranges.txt"
ignore_file = "./data/ignore.txt"

def load_data(var_file):
    input_file = ""
    with open(src_file, "rb") as sfile:
        for line in sfile.readlines():
            input_file = os.path.abspath(line.strip())

    model_variables = []
    with open(var_file, "rb") as vfile:
        for line in vfile.readlines():
            line = line.strip()
            if not line: continue
            model_variables.append(line.split()[0])

    dvar = model_variables[0]
    ivar = model_variables[1:]

    range_variables = []
    with open(range_file, "rb") as rfile:
        for line in rfile.readlines():
            line = line.strip()
            if not line: continue
            parts = line.strip().split()
            if 2 < len(parts):
                range_variables.append({"name":parts[0], "type":parts[1], "range":parts[2].split(",")})

    ignore_records = []
    with open(ignore_file, "rb") as rfile:
        for line in rfile.readlines():
            line = line.strip()
            if not line: continue
            ignore_records.append(line)

    idx_info = []
    all_data = []
    with open(input_file, "rb") as ifile:
        reader = csv.reader(ifile)
        titles = reader.next()
        model_idx = [titles.index(var) for var in model_variables]
        r0 = 0
        r = 0
        for row in reader:
            if row[0] in ignore_records: continue

            r0 += 1 # Starts at 1, because of titles
            all_missing = True
            some_missing = False
            missing_dvar = row[model_idx[0]] == "\\N"
            for i in range(1, len(model_variables)):
                var_idx = model_idx[i]
                if row[var_idx] == "\\N":
                    some_missing = True
                else:
                    all_missing = False

            inside_range = True
            for var in range_variables:
                idx = titles.index(var["name"])
                val = row[idx]
                if val == "\\N": continue
                vtype = var["type"]
                vrang = var["range"]
                test = True
                if vtype == "category":
                    test = val in vrang
                else:
                    test = float(vrang[0]) <= float(val) and float(val) < float(vrang[1])
                inside_range = inside_range and test

            if not all_missing and not missing_dvar and inside_range:
                idx_info.append([r0, row[0], row[model_idx[0]]])
                all_data.append([row[idx].replace("\\N", "?") for idx in model_idx])
                r += 1

    print all_data
'''

parser = argparse.ArgumentParser()
parser.add_argument("-v", "--var_file", nargs=1, default=["./data/variables-master.txt"],
                    help="File containing list of variables to consider in the test")
args = parser.parse_args()

var_file = args.var_file[0]
# load_data(var_file)

depvar = None
numvar = []
catvar = []
with open(var_file, "rb") as vfile:
    for line in vfile.readlines():
        line = line.strip()
        if not line: continue
        parts = line.split()
        if not depvar: depvar = parts[0]
        else:
            if parts[1] == "category": catvar.append(parts[0])
            else: numvar.append(parts[0])
#
#         model_variables.append([0])

# dvar = model_variables[0]
# ivar = model_variables[1:]
# print dvar
# print ivar

src_file = "./data/sources.txt"
input_file = ""
with open(src_file, "rb") as sfile:
    for line in sfile.readlines():
        input_file = os.path.abspath(line.strip())
df = pd.read_csv(input_file, delimiter=",", na_values="\\N")

plt.clf()
fig = plt.figure()

# print df[dvar]
# color = dict(boxes='DarkGreen', whiskers='DarkOrange', medians='DarkBlue', caps='Gray')

range_file = "./data/ranges.txt"
range_variables = []
with open(range_file, "rb") as rfile:
    for line in rfile.readlines():
       line = line.strip()
       if not line: continue
       parts = line.strip().split()
       if 2 < len(parts):
           range_variables.append({"name":parts[0], "type":parts[1], "range":parts[2].split(",")})
ranges = (df["DIAG"] == 1) & (10 <= df["AGE"]) & (df["AGE"] <= 50) & ((df["OUT"] == 1) | (df["OUT"] == 0))

# sns.set(style="whitegrid")

for var in numvar:
    #df[ranges].boxplot(var, by=depvar)
    g = sns.factorplot("OUT", var, data=df[ranges], kind="box", ci=0, palette="coolwarm")
    g.despine(offset=10, trim=True)
    g.set_xticklabels(['No', 'Yes'])
    plt.savefig("./out/" + var + ".pdf")
    plt.clf()

for var in catvar:
	with mpl.rc_context({"lines.linewidth": 0.5}):
# 		g = sns.barplot(df[ranges][var], df[ranges]["OUT"], palette="coolwarm")
		g = sns.FacetGrid(df[ranges], aspect=1)
		g.map(sns.barplot, "OUT", var, palette="coolwarm");
		g.set_xticklabels(['No', 'Yes'])
		g.set(ylim=(0.0, 1.0))
#     df[ranges].hist(var, by=depvar)
#     df[ranges].plot(kind='area', var, by=depvar)
#
    	plt.savefig("./out/" + var + ".pdf")
    	plt.clf()

# df[ranges & (df["OUT"] == 0)]["ALT_1"].plot(kind='box', color=color, sym='r+')
# df[ranges & (df["OUT"] == 1)]["ALT_1"].plot(kind='box', color=color, sym='r+')

#

# df[criterion]["ALT_1"].plot(kind='box', color=color, sym='r+')

# sns.factorplot("ALT_1", df, kind="bar");<commit_msg>some tweaks in the variable plots<commit_after>"""
Rank all available models

@copyright: The Broad Institute of MIT and Harvard 2015
"""

import argparse
import os, csv
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns

var_labels = { "OUT": "Outcome",
               "PCR": "PCR",
               "WEAK": "Weakness",
               "VOMIT": "Vomit",
               "AST_1": "AST",
               "Ca_1": "Ca",
               "AlkPhos_1": "ALK",
               "EDEMA": "Edema",
               "CONF": "Confussion",
               "Cl_1": "Cl",
               "TEMP": "Temperature",
               "RRATE": "Respiratory rate",
               "PBACK": "Back pain",
               "DIZZI": "Dizziness",
               "ALT_1": "ALT",
               "Cr_1": "CRE",
               "TCo2_1": "tCO2",
               "PRETROS": "Retrosternal pain",
               "DIARR": "Diarrhea",
               "HRATE": "Hearth rate",
               "Alb_1": "Alb",
               "BUN_1": "BUN",
               "TP_1": "TP",
               "PDIAST": "Diastolic pressure",
               "PABD": "Abdominal pain" }

parser = argparse.ArgumentParser()
parser.add_argument("-v", "--var_file", nargs=1, default=["./data/variables-master.txt"],
                    help="File containing list of variables to consider in the test")
args = parser.parse_args()

var_file = args.var_file[0]
# load_data(var_file)

depvar = None
numvar = []
catvar = []
with open(var_file, "rb") as vfile:
    for line in vfile.readlines():
        line = line.strip()
        if not line: continue
        parts = line.split()
        if not depvar: depvar = parts[0]
        else:
            if parts[1] == "category": catvar.append(parts[0])
            else: numvar.append(parts[0])
#
#         model_variables.append([0])

# dvar = model_variables[0]
# ivar = model_variables[1:]
# print dvar
# print ivar

src_file = "./data/sources.txt"
input_file = ""
with open(src_file, "rb") as sfile:
    for line in sfile.readlines():
        input_file = os.path.abspath(line.strip())
df = pd.read_csv(input_file, delimiter=",", na_values="\\N")

plt.clf()
fig = plt.figure()

# print df[dvar]
# color = dict(boxes='DarkGreen', whiskers='DarkOrange', medians='DarkBlue', caps='Gray')

range_file = "./data/ranges.txt"
range_variables = []
with open(range_file, "rb") as rfile:
    for line in rfile.readlines():
       line = line.strip()
       if not line: continue
       parts = line.strip().split()
       if 2 < len(parts):
           range_variables.append({"name":parts[0], "type":parts[1], "range":parts[2].split(",")})
ranges = (df["DIAG"] == 1) & (10 <= df["AGE"]) & (df["AGE"] <= 50) & ((df["OUT"] == 1) | (df["OUT"] == 0))

# sns.set(style="whitegrid")

for var in numvar:
    #df[ranges].boxplot(var, by=depvar)
    g = sns.factorplot("OUT", var, data=df[ranges], kind="box", ci=0, palette="coolwarm")
    g.despine(offset=10, trim=True)
    g.set_xticklabels(['Discharged', 'Died'])
    plt.xlabel('Outcome')
    plt.ylabel(var_labels[var])
    plt.savefig("./out/" + var + ".pdf")
    plt.clf()

for var in catvar:
	with mpl.rc_context({"lines.linewidth": 0.5}):
# 		g = sns.barplot(df[ranges][var], df[ranges]["OUT"], palette="coolwarm")
		g = sns.FacetGrid(df[ranges], aspect=1)
		g.map(sns.barplot, "OUT", var, palette="coolwarm");
		g.set_xticklabels(['Discharged', 'Died'])
		g.set(ylim=(0.0, 1.0))
        plt.xlabel('Outcome')
        plt.ylabel(var_labels[var])
#     df[ranges].hist(var, by=depvar)
#     df[ranges].plot(kind='area', var, by=depvar)
#
    	plt.savefig("./out/" + var + ".pdf")
    	plt.clf()
<|endoftext|>


<commit_before>import sys
import cv2
import numpy as np
import math

def open_image_and_separate_bg(fname):
    img = cv2.imread(fname)

    # Let's build a simple mask to separate pieces from background
    # just by checking if pixel is in range of some colours
    mask = cv2.inRange(img,
                       np.array([0, 0, 165], dtype=img.dtype),
                       np.array([200, 100, 255], dtype=img.dtype))
    mask = cv2.bitwise_not(mask)

    # Init kernel for dilate/erode
    kernel = np.ones((5, 5), np.uint8)
    # optional blur of mask
    # mask = cv2.medianBlur(mask, 5)

    # Clean noise on background
    mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
    # Clean noise inside pieces
    mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)

    # Apply mask to image
    img = cv2.bitwise_and(img, img, mask=mask)

    # Add some borders (just in case)
    img = cv2.copyMakeBorder(img, 10, 10, 10, 10, cv2.BORDER_CONSTANT,
                             value=[0, 0, 0])
    mask = cv2.copyMakeBorder(mask, 10, 10, 10, 10, cv2.BORDER_CONSTANT,
                              value=[0, 0, 0])

    # Write original image with no background for debug purposes
    cv2.imwrite("debug/mask.tif", mask)

    return img, mask

def extract_piece_and_features(img, c, name):
    height, width, channels = img.shape

    # bounding rect of currrent contour
    r_x, r_y, r_w, r_h = cv2.boundingRect(c)
    area = cv2.contourArea(c)

    # filter out too small fragments
    if r_w <= 10 or r_h <= 10 or area < 100:
        print("Skipping piece #%s as too small" % name)
        return

    hull = cv2.convexHull(c)
    hull_area = cv2.contourArea(hull)
    solidity = float(area) / hull_area
    if solidity < 0.75:
        print("Piece #%s looks suspicious" % name)

    # position of rect of min area.
    # this will provide us angle to straighten image
    box_center, bbox, angle = cv2.minAreaRect(c)

    # We want our pieces to be "vertical"
    if bbox[0] > bbox[1]:
        angle += 90
        bbox = (bbox[1], bbox[0])

    # Coords of region of interest using which we should crop piece after
    # rotation
    y1 = math.floor(box_center[1] - bbox[1] / 2)
    x1 = math.floor(box_center[0] - bbox[0] / 2)
    bbox = tuple(map(int, map(math.ceil, bbox)))

    # A mask we use to show only piece we are currently working on
    mask = np.zeros([height, width, 1], dtype=np.uint8)
    cv2.drawContours(mask, [c], -1, 255, cv2.cv.CV_FILLED)

    # apply mask to original image
    img_crp = img[r_y:r_y + r_h, r_x:r_x + r_w]
    mask = mask[r_y:r_y + r_h, r_x:r_x + r_w]
    img_roi = cv2.bitwise_and(img_crp, img_crp, mask=mask)

    # Add alpha layer and set it to the mask
    img_roi = cv2.cvtColor(img_roi, cv2.cv.CV_BGR2BGRA)
    img_roi[:, :, 3] = mask[:, :, 0]

    # Straighten it
    # Because we crop original image before rotation we save us some memory
    # and a lot of time but we need to adjust coords of the center of
    # new min area rect
    M = cv2.getRotationMatrix2D((box_center[0] - r_x,
                                 box_center[1] - r_y), angle, 1)

    # And translate an image a bit to make it fit to the bbox again.
    # This is done with direct editing of the transform matrix.
    # (Wooohoo, I know matrix-fu)
    M[0][2] += r_x - x1
    M[1][2] += r_y - y1

    # Apply rotation/transform/crop
    img_roi = cv2.warpAffine(img_roi, M, bbox)
    cv2.imwrite("pieces/%s.tif" % name, img_roi)

    # FEATURES MAGIC BELOW
    #
    # Get our mask/contour back after the trasnform
    _, _, _, mask = cv2.split(img_roi)
    contours, _ = cv2.findContours(mask.copy(), cv2.RETR_TREE,
                                   cv2.CHAIN_APPROX_SIMPLE)

    if len(contours) != 1:
        print("Piece #%s has strange contours after transform" % name)

    # Let's also detect 25 of most outstanding corners of the contour.
    corners = cv2.goodFeaturesToTrack(mask, 25, 0.01, 10)
    corners = np.int0(corners)

    # Draw contours for debug purposes
    mask = cv2.cvtColor(mask, cv2.cv.CV_GRAY2BGR)
    cv2.drawContours(mask, contours, -1, (255, 0, 255), 2)

    # And put corners on top
    for i in corners:
        x, y = i.ravel()
        cv2.circle(mask, (x, y), 3, [0, 255, 0], -1)

    cnt = contours[0]

    # Let's find features that will help us to determine top side of the piece
    if mask.shape[0] / float(mask.shape[1]) >= 2.7:
        hull = cv2.convexHull(cnt, returnPoints=False)
        defects = cv2.convexityDefects(cnt, hull)

        # Check for convex defects to see if we can find a trace of a
        # shredder cut which is usually on top of the piece.
        for i in range(defects.shape[0]):
            s, e, f, d = defects[i, 0]
            far = tuple(cnt[f][0])

            # if convex defect is big enough
            if d / 256. > 7:
                # And lays at the top or bottom of the piece
                y_dist = min(abs(0 - far[1]), abs(mask.shape[0] - far[1]))
                if float(y_dist) / mask.shape[0] < 0.1:
                    # and more or less is in the center
                    if abs(far[0] - mask.shape[1] / 2.) / mask.shape[1] < 0.25:
                        cv2.circle(mask, far, 5, [0, 0, 255], -1)

        # Also top and bottom points on the contour
        topmost = tuple(cnt[cnt[:, :, 1].argmin()][0])
        bottommost = tuple(cnt[cnt[:, :, 1].argmax()][0])
        cv2.circle(mask, topmost, 3, [0, 255, 255], -1)
        cv2.circle(mask, bottommost, 3, [0, 255, 255], -1)

    cv2.imwrite("pieces/%s_mask.tif" % name, mask)

if __name__ == '__main__':
    # Open an image here
    fname = "src/puzzle_small.tif" if len(sys.argv) == 1 else sys.argv[1]

    img, mask = open_image_and_separate_bg(fname)

    # Find contours of pieces
    contours, _ = cv2.findContours(mask, cv2.RETR_TREE,
                                   cv2.CHAIN_APPROX_SIMPLE)

    # Walk through contrours to extract pieces and unify the rotation
    for i, c in enumerate(contours):
        extract_piece_and_features(img, c, i)

    # Another useful debug: drawing contours and their min area boxes
    cv2.drawContours(img, contours, -1, (0, 255, 0), 2)
    boxes = map(cv2.minAreaRect, contours)
    boxes2draw = map(lambda b: np.int0(cv2.cv.BoxPoints(b)), boxes)
    cv2.drawContours(img, boxes2draw, -1, (0, 0, 255), 2)
    cv2.imwrite("debug/out_with_contours.tif", img)
<commit_msg>WIP: generalizing BG removal<commit_after>import sys
import cv2
import numpy as np
import math
# import random

backgrounds = [
    # DARPA SHRED task #1
    [[np.array([0, 0, 165]), np.array([200, 100, 255])]],
    # Pink
    [[np.array([160, 120, 230]), np.array([200, 210, 255])],
     [np.array([210, 185, 245]), np.array([235, 195, 255])],
     ],
]

def replace_scanner_background(img):
    # Here we are trying to find scanner background
    # (gray borders around colored sheet where shreds are glued)
    # Problem here is that colored sheet might have borders of different sizes
    # on different sides of it and we don't know the color of the sheet.
    # Also sheets isn't always has perfectly straight edges or rotated slightly
    # against edges of the scanner
    # Idea is relatively simple:

    # Convert image to LAB and grab A and B channels.
    fimg = cv2.cvtColor(img, cv2.cv.CV_BGR2Lab)
    _, a_channel, b_channel = cv2.split(fimg)

    def try_method(fimg, border, aggressive=True):
        fimg = cv2.copyMakeBorder(fimg, 5, 5, 5, 5, cv2.BORDER_CONSTANT,
                                  value=border)

        if aggressive:
            cv2.floodFill(fimg, None, (10, 10), 255, 1, 1)
        else:
            cv2.floodFill(fimg, None, (10, 10), 255, 2, 2,
                          cv2.FLOODFILL_FIXED_RANGE)

        _, fimg = cv2.threshold(fimg, 254, 255, cv2.THRESH_BINARY)

        hist = cv2.calcHist([fimg], [0], None, [2], [0, 256])
        return fimg, hist

    options = [
        [a_channel, 127],
        [b_channel, 134],
        [a_channel, 127, False],
        [b_channel, 134, False],
    ]

    # And then try to add a border of predefined color around each channel
    # and flood fill scanner scanner background starting from most
    # aggressive flood fill settings to least aggressive.
    for i, opt in enumerate(options):
        fimg, hist = try_method(*opt)
        # First setting that doesn't flood that doesn't hurt colored sheet
        # to badly wins.
        if hist[1] < (hist[0] + hist[1]) * 0.2:
            break

    # Then we dilate it a bit.
    fimg = cv2.morphologyEx(fimg, cv2.MORPH_DILATE, np.ones((5, 5), np.uint8),
                            iterations=2)
    fimg = fimg[5:-5, 5:-5]

    # Searching for a biggest outter contour
    contours, _ = cv2.findContours(cv2.bitwise_not(fimg), cv2.RETR_EXTERNAL,
                                   cv2.CHAIN_APPROX_SIMPLE)
    main_contour = np.array(map(cv2.contourArea, contours)).argmax()

    # build a convex hull for it
    hull = cv2.convexHull(contours[main_contour])

    # And make a mask out of it.
    cv2.fillPoly(fimg, [hull], 0)

    # Done. Piece of cake.
    return fimg
    # Cannot say I'm satisfied with algo but using grabcut here seems too
    # expensive. Also it was a lot of fun to build it.

def find_appropriate_mask(img):
    # Let's build a simple mask to separate pieces from background
    # just by checking if pixel is in range of some colours
    m = 0
    res = None

    # Here we calculate mask to separate background of the scanner
    scanner_bg = replace_scanner_background(img)

    # And here we are trying to check different ranges for different
    # background to find the winner.
    for bg in backgrounds:
        mask = np.zeros(img.shape[:2], np.uint8)

        for rng in bg:
            # As each pre-defined background is described by a list of
            # color ranges we are summing up their masks
            mask = cv2.bitwise_or(mask, cv2.inRange(img, rng[0], rng[1]))

        hist = cv2.calcHist([mask], [0], None, [2], [0, 256])
        # And here we are searching for a biggest possible mask across all
        # possible predefined backgrounds
        if hist[1] > m:
            m = hist[1]
            res = mask

    # then we remove scanner background
    res = cv2.bitwise_or(scanner_bg, res)

    bgdModel = np.zeros((1, 65), np.float64)
    fgdModel = np.zeros((1, 65), np.float64)

    # and run grabcut algo against the seed mask to refine background
    # separation
    res = cv2.medianBlur(res, 5)
    mask = np.where((res == 255), cv2.GC_BGD, cv2.GC_PR_FGD).astype('uint8')
    cv2.grabCut(img, mask, None, bgdModel, fgdModel, 2, cv2.GC_INIT_WITH_MASK)
    res = np.where((mask == 2) | (mask == 0), 255, 0).astype('uint8')

    return cv2.bitwise_not(res)

def open_image_and_separate_bg(fname):
    img = cv2.imread(fname)

    mask = find_appropriate_mask(img)

    # Init kernel for dilate/erode
    kernel = np.ones((5, 5), np.uint8)
    # optional blur of mask
    # mask = cv2.medianBlur(mask, 5)

    # Clean noise on background
    mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
    # Clean noise inside pieces
    mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)

    # Apply mask to image
    img = cv2.bitwise_and(img, img, mask=mask)

    # Add some borders (just in case)
    img = cv2.copyMakeBorder(img, 10, 10, 10, 10, cv2.BORDER_CONSTANT,
                             value=[0, 0, 0])
    mask = cv2.copyMakeBorder(mask, 10, 10, 10, 10, cv2.BORDER_CONSTANT,
                              value=[0, 0, 0])

    # Write original image with no background for debug purposes
    cv2.imwrite("debug/mask.tif", mask)

    return img, mask

def extract_piece_and_features(img, c, name):
    height, width, channels = img.shape

    # bounding rect of currrent contour
    r_x, r_y, r_w, r_h = cv2.boundingRect(c)
    area = cv2.contourArea(c)

    # filter out too small fragments
    if r_w <= 10 or r_h <= 10 or area < 100:
        print("Skipping piece #%s as too small" % name)
        return

    hull = cv2.convexHull(c)
    hull_area = cv2.contourArea(hull)
    solidity = float(area) / hull_area
    if solidity < 0.75:
        print("Piece #%s looks suspicious" % name)

    # position of rect of min area.
    # this will provide us angle to straighten image
    box_center, bbox, angle = cv2.minAreaRect(c)

    # We want our pieces to be "vertical"
    if bbox[0] > bbox[1]:
        angle += 90
        bbox = (bbox[1], bbox[0])

    # Coords of region of interest using which we should crop piece after
    # rotation
    y1 = math.floor(box_center[1] - bbox[1] / 2)
    x1 = math.floor(box_center[0] - bbox[0] / 2)
    bbox = tuple(map(int, map(math.ceil, bbox)))

    # A mask we use to show only piece we are currently working on
    mask = np.zeros([height, width, 1], dtype=np.uint8)
    cv2.drawContours(mask, [c], -1, 255, cv2.cv.CV_FILLED)

    # apply mask to original image
    img_crp = img[r_y:r_y + r_h, r_x:r_x + r_w]
    mask = mask[r_y:r_y + r_h, r_x:r_x + r_w]
    img_roi = cv2.bitwise_and(img_crp, img_crp, mask=mask)

    # Add alpha layer and set it to the mask
    img_roi = cv2.cvtColor(img_roi, cv2.cv.CV_BGR2BGRA)
    img_roi[:, :, 3] = mask[:, :, 0]

    # Straighten it
    # Because we crop original image before rotation we save us some memory
    # and a lot of time but we need to adjust coords of the center of
    # new min area rect
    M = cv2.getRotationMatrix2D((box_center[0] - r_x,
                                 box_center[1] - r_y), angle, 1)

    # And translate an image a bit to make it fit to the bbox again.
    # This is done with direct editing of the transform matrix.
    # (Wooohoo, I know matrix-fu)
    M[0][2] += r_x - x1
    M[1][2] += r_y - y1

    # Apply rotation/transform/crop
    img_roi = cv2.warpAffine(img_roi, M, bbox)
    cv2.imwrite("pieces/%s.tif" % name, img_roi)

    # FEATURES MAGIC BELOW
    #
    # Get our mask/contour back after the trasnform
    _, _, _, mask = cv2.split(img_roi)
    contours, _ = cv2.findContours(mask.copy(), cv2.RETR_TREE,
                                   cv2.CHAIN_APPROX_SIMPLE)

    if len(contours) != 1:
        print("Piece #%s has strange contours after transform" % name)

    # Let's also detect 25 of most outstanding corners of the contour.
    corners = cv2.goodFeaturesToTrack(mask, 25, 0.01, 10)
    corners = np.int0(corners) if corners is not None else []

    edges = cv2.Canny(img_roi[:, :, 2], 100, 200)
    cv2.imwrite("pieces/%s_edges.tif" % name, edges)

    # Draw contours for debug purposes
    mask = cv2.cvtColor(mask, cv2.cv.CV_GRAY2BGR)
    cv2.drawContours(mask, contours, -1, (255, 0, 255), 2)

    # And put corners on top
    for i in corners:
        x, y = i.ravel()
        cv2.circle(mask, (x, y), 3, [0, 255, 0], -1)

    cnt = contours[0]

    # Let's find features that will help us to determine top side of the piece
    if mask.shape[0] / float(mask.shape[1]) >= 2.7:
        hull = cv2.convexHull(cnt, returnPoints=False)
        defects = cv2.convexityDefects(cnt, hull)

        # Check for convex defects to see if we can find a trace of a
        # shredder cut which is usually on top of the piece.
        for i in range(defects.shape[0]):
            s, e, f, d = defects[i, 0]
            far = tuple(cnt[f][0])

            # if convex defect is big enough
            if d / 256. > 7:
                # And lays at the top or bottom of the piece
                y_dist = min(abs(0 - far[1]), abs(mask.shape[0] - far[1]))
                if float(y_dist) / mask.shape[0] < 0.1:
                    # and more or less is in the center
                    if abs(far[0] - mask.shape[1] / 2.) / mask.shape[1] < 0.25:
                        cv2.circle(mask, far, 5, [0, 0, 255], -1)

        # Also top and bottom points on the contour
        topmost = tuple(cnt[cnt[:, :, 1].argmin()][0])
        bottommost = tuple(cnt[cnt[:, :, 1].argmax()][0])
        cv2.circle(mask, topmost, 3, [0, 255, 255], -1)
        cv2.circle(mask, bottommost, 3, [0, 255, 255], -1)

    cv2.imwrite("pieces/%s_mask.tif" % name, mask)

if __name__ == '__main__':
    # Open an image here
    fname = "src/puzzle_small.tif" if len(sys.argv) == 1 else sys.argv[1]

    img, mask = open_image_and_separate_bg(fname)

    # # Find contours of pieces
    contours, _ = cv2.findContours(mask, cv2.RETR_TREE,
                                   cv2.CHAIN_APPROX_SIMPLE)

    # Walk through contrours to extract pieces and unify the rotation
    for i, c in enumerate(contours):
        extract_piece_and_features(img, c, i)

    # Another useful debug: drawing contours and their min area boxes
    cv2.drawContours(img, contours, -1, (0, 255, 0), 2)
    boxes = map(cv2.minAreaRect, contours)
    boxes2draw = map(lambda b: np.int0(cv2.cv.BoxPoints(b)), boxes)
    cv2.drawContours(img, boxes2draw, -1, (0, 0, 255), 2)
    cv2.imwrite("debug/out_with_contours.tif", img)
<|endoftext|>


<commit_before>#!/bin/env python
"""
Authors: David Kent, Tim Erwin, Tim Bedin, Damien Irving

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Convert a CDAT xml catalogue to a single netCDF file and/or crop
temporal and spatial dimensions.

"""

import os, sys, pdb
import subprocess
import argparse

import numpy as np
import cdms2
if hasattr(cdms2, 'setNetcdfDeflateFlag'):
    cdms2.setNetcdfDeflateFlag(0)
    cdms2.setNetcdfDeflateLevelFlag(0)
    cdms2.setNetcdfShuffleFlag(0)
import cdtime

def list_nobounds(cf, ids=False):
    """Get the names of all the variables in a netCDF file."""

    bnds = [v.bounds for v in cf.variables.values() if 'bounds' in v.attributes]
    bnds += [a.bounds for a in cf.axes.values() if 'bounds' in a.attributes]
    nodim = [vid for vid in cf.variables.keys() if cf[vid].getOrder() == '']
    nonvar = bnds + nodim
    if ids:
        return [v for v, k in cf.variables.iteritems() if v not in nonvar]
    else:
        return [k for v, k in cf.variables.iteritems() if v not in nonvar]

def check_time_axis(tvar):
    """Check that the time-axis is monotonically increasing.

    When cdo functions fail mid-process they often output a file
    that has a non-sensical, non-monotonic time axis.

    """

    assert type(tvar) == cdms2.tvariable.TransientVariable

    time_values = tvar.getTime()[:]
    dt = np.diff(time_values)
    check = np.all(dt > 0)

    if not check:
        print "Time axis not monotonically increasing, skipping %s" %(tvar.id)

    return check

def check_valid_range(tvar):
    """Check if the variable data falls within its valid range."""

    assert type(tvar) == cdms2.tvariable.TransientVariable

    if 'valid_range' in tvar.attributes and isinstance(tvar.valid_range, basestring):
        try:
            tvar.valid_range = np.fromstring(tvar.valid_range.strip('[]'),
                                             dtype=vtar.dtype,
                                             sep=' ')
        except:
            pass

def main(var, infile, outfile,
         time_bounds=':',
         lon_bounds=':',
         lat_bounds=':',
         level_bounds=':'):
    """Run the program."""

    cf = cdms2.open(infile)

    if var == 'None':
        vars = list_nobounds(cf, ids=True)
    else:
        vars = [var]

    cfout = cdms2.createDataset(outfile)
    nwritten = 0
    for var in vars:
        v = cf(var, time=time_bounds, longitude=lon_bounds,
               latitude=lat_bounds, level=level_bounds)

        time_check = check_time_axis(v)
        check_valid_range(v)

        if time_check:
            vout = cfout.write(v, axes=v.getAxisList(), id=v.id)
            if hasattr(v, 'name') and 'variable' not in v.name:
                vout.name = v.name
            nwritten += 1

    if nwritten == 0:
        cfout.close()
        os.remove(outfile)
        sys.exit(1)

    for att in cf.listglobal():
        setattr(cfout, att, cf.attributes[att])

    cfout.close()

if __name__ == '__main__':

    extra_info = """  The edges of specified bounds are included.
  To select a single time, lon, lat or level, set both bounds to the same value.
  """

    description = 'Convert a CDAT xml catalogue to a single netCDF file and/or crop temporal and spatial dimensions'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("variable", type=str, help="""Variable to extract. If 'None', all variables will be extracted.""")
    parser.add_argument("infile", type=str, help="Name of input netCDF file or cdscan xml catalogue file")
    parser.add_argument("outfile", type=str, help="Name of output netCDF file")

    parser.add_argument("--time_bounds", type=str, nargs=2, metavar=('START_DATE', 'END_DATE'),
                        help="Bounds of the time period to extract from infile [default = all times]. Date format is YYYY-MM-DD.")
    parser.add_argument("--lon_bounds", type=float, nargs=2, metavar=('WEST_LON', 'EAST_LON'),
                        help="Longitude bounds of the region to extract from infile [default = all longitudes].")
    parser.add_argument("--lat_bounds", type=float, nargs=2, metavar=('SOUTH_LAT', 'NORTH_LAT'),
                        help="Latitude bounds of the region to extract from infile [default = all latitudes].")
    parser.add_argument("--level_bounds", type=float, nargs=2, metavar=('BOTTOM_LEVEL', 'TOP_LEVEL'),
                        help="Vertical level bounds of the region to extract from infile [default = all vetical levels].")

    args = parser.parse_args()

    args.time_bounds = ':' if not args.time_bounds else args.time_bounds
    args.lon_bounds = ':' if not args.lon_bounds else args.lon_bounds
    args.lat_bounds = ':' if not args.lat_bounds else args.lat_bounds
    args.level_bounds = ':' if not args.level_bounds else args.level_bounds

    main(args.variable, args.infile, args.outfile,
         time_bounds=args.time_bounds,
         lon_bounds=args.lon_bounds,
         lat_bounds=args.lat_bounds,
         level_bounds=args.level_bounds)

<commit_msg>Added tuple conversion for xml_to_nc cropping<commit_after>#!/bin/env python
"""
Authors: David Kent, Tim Erwin, Tim Bedin, Damien Irving

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Convert a CDAT xml catalogue to a single netCDF file and/or crop
temporal and spatial dimensions.

"""

import os
import sys
import argparse

import numpy as np
import cdms2
if hasattr(cdms2, 'setNetcdfDeflateFlag'):
    cdms2.setNetcdfDeflateFlag(0)
    cdms2.setNetcdfDeflateLevelFlag(0)
    cdms2.setNetcdfShuffleFlag(0)
import cdtime

def list_nobounds(cf, ids=False):
    """Get the names of all the variables in a netCDF file."""

    bnds = [v.bounds for v in cf.variables.values() if 'bounds' in v.attributes]
    bnds += [a.bounds for a in cf.axes.values() if 'bounds' in a.attributes]
    nodim = [vid for vid in cf.variables.keys() if cf[vid].getOrder() == '']
    nonvar = bnds + nodim
    if ids:
        return [v for v, k in cf.variables.iteritems() if v not in nonvar]
    else:
        return [k for v, k in cf.variables.iteritems() if v not in nonvar]

def check_time_axis(tvar):
    """Check that the time-axis is monotonically increasing.

    When cdo functions fail mid-process they often output a file
    that has a non-sensical, non-monotonic time axis.

    """

    assert type(tvar) == cdms2.tvariable.TransientVariable

    time_values = tvar.getTime()[:]
    dt = np.diff(time_values)
    check = np.all(dt > 0)

    if not check:
        print "Time axis not monotonically increasing, skipping %s" %(tvar.id)

    return check

def check_valid_range(tvar):
    """Check if the variable data falls within its valid range."""

    assert type(tvar) == cdms2.tvariable.TransientVariable

    if 'valid_range' in tvar.attributes and isinstance(tvar.valid_range, basestring):
        try:
            tvar.valid_range = np.fromstring(tvar.valid_range.strip('[]'),
                                             dtype=vtar.dtype,
                                             sep=' ')
        except:
            pass

def main(var, infile, outfile,
         time_bounds, lon_bounds,
         lat_bounds, level_bounds):
    """Run the program."""

    cf = cdms2.open(infile)

    if var == 'None':
        vars = list_nobounds(cf, ids=True)
    else:
        vars = [var]

    cfout = cdms2.createDataset(outfile)
    nwritten = 0

    for var in vars:
        v = cf(var, time=time_bounds, longitude=lon_bounds,
               latitude=lat_bounds, level=level_bounds)

        time_check = check_time_axis(v)
        check_valid_range(v)

        if time_check:
            vout = cfout.write(v, axes=v.getAxisList(), id=v.id)
            if hasattr(v, 'name') and 'variable' not in v.name:
                vout.name = v.name
            nwritten += 1

    if nwritten == 0:
        cfout.close()
        os.remove(outfile)
        sys.exit(1)

    for att in cf.listglobal():
        setattr(cfout, att, cf.attributes[att])

    cfout.close()

if __name__ == '__main__':

    extra_info = """  The edges of specified bounds are included.
  To select a single time, lon, lat or level, set both bounds to the same value.
  """

    description = 'Convert a CDAT xml catalogue to a single netCDF file and/or crop temporal and spatial dimensions'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("variable", type=str, help="""Variable to extract. If 'None', all variables will be extracted.""")
    parser.add_argument("infile", type=str, help="Name of input netCDF file or cdscan xml catalogue file")
    parser.add_argument("outfile", type=str, help="Name of output netCDF file")

    parser.add_argument("--time_bounds", type=str, nargs=2, metavar=('START_DATE', 'END_DATE'),
                        help="Bounds of the time period to extract from infile [default = all times]. Date format is YYYY-MM-DD.")
    parser.add_argument("--lon_bounds", type=float, nargs=2, metavar=('WEST_LON', 'EAST_LON'),
                        help="Longitude bounds of the region to extract from infile [default = all longitudes].")
    parser.add_argument("--lat_bounds", type=float, nargs=2, metavar=('SOUTH_LAT', 'NORTH_LAT'),
                        help="Latitude bounds of the region to extract from infile [default = all latitudes].")
    parser.add_argument("--level_bounds", type=float, nargs=2, metavar=('BOTTOM_LEVEL', 'TOP_LEVEL'),
                        help="Vertical level bounds of the region to extract from infile [default = all vetical levels].")

    args = parser.parse_args()

    args.time_bounds = ':' if not args.time_bounds else args.time_bounds
    args.lon_bounds = ':' if not args.lon_bounds else tuple(args.lon_bounds)
    args.lat_bounds = ':' if not args.lat_bounds else tuple(args.lat_bounds)
    args.level_bounds = ':' if not args.level_bounds else tuple(args.level_bounds)

    main(args.variable, args.infile, args.outfile,
         time_bounds=args.time_bounds,
         lon_bounds=args.lon_bounds,
         lat_bounds=args.lat_bounds,
         level_bounds=args.level_bounds)

<|endoftext|>


<commit_before>import json
import warnings
from cqlengine import SizeTieredCompactionStrategy, LeveledCompactionStrategy
from cqlengine import ONE
from cqlengine.named import NamedTable

from cqlengine.connection import connection_manager, execute, execute_native, get_session, get_cluster
from cqlengine.exceptions import CQLEngineException

import logging
from collections import namedtuple
Field = namedtuple('Field', ['name', 'type'])

logger = logging.getLogger(__name__)

# system keyspaces
schema_columnfamilies = NamedTable('system', 'schema_columnfamilies')

def create_keyspace(name, strategy_class='SimpleStrategy', replication_factor=3, durable_writes=True, **replication_values):
    """
    creates a keyspace

    :param name: name of keyspace to create
    :param strategy_class: keyspace replication strategy class
    :param replication_factor: keyspace replication factor
    :param durable_writes: 1.2 only, write log is bypassed if set to False
    :param **replication_values: 1.2 only, additional values to ad to the replication data map
    """
    cluster = get_cluster()

    if name not in cluster.metadata.keyspaces:
        #try the 1.2 method
        replication_map = {
            'class': strategy_class,
            'replication_factor':replication_factor
        }
        replication_map.update(replication_values)
        if strategy_class.lower() != 'simplestrategy':
            # Although the Cassandra documentation states for `replication_factor`
            # that it is "Required if class is SimpleStrategy; otherwise,
            # not used." we get an error if it is present.
            replication_map.pop('replication_factor', None)

        query = """
        CREATE KEYSPACE {}
        WITH REPLICATION = {}
        """.format(name, json.dumps(replication_map).replace('"', "'"))

        if strategy_class != 'SimpleStrategy':
            query += " AND DURABLE_WRITES = {}".format('true' if durable_writes else 'false')

        execute_native(query)

def delete_keyspace(name):
    keyspaces = execute_native("""SELECT keyspace_name FROM system.schema_keyspaces""", {}, ONE)
    if name in [r.keyspace_name for r in keyspaces]:
        execute_native("DROP KEYSPACE {}".format(name))

def create_table(model, create_missing_keyspace=True):
    warnings.warn("create_table has been deprecated in favor of sync_table and will be removed in a future release", DeprecationWarning)
    sync_table(model, create_missing_keyspace)

def sync_table(model, create_missing_keyspace=True):
    """
    Inspects the model and creates / updates the corresponding table and columns.

    Note that the attributes removed from the model are not deleted on the database.
    They become effectively ignored by (will not show up on) the model.

    :param create_missing_keyspace: (Defaults to True) Flags to us that we need to create missing keyspace
        mentioned in the model automatically.
    :type create_missing_keyspace: bool
    """

    if model.__abstract__:
        raise CQLEngineException("cannot create table from abstract model")

    #construct query string
    cf_name = model.column_family_name()
    raw_cf_name = model.column_family_name(include_keyspace=False)

    ks_name = model._get_keyspace()
    #create missing keyspace
    if create_missing_keyspace:
        create_keyspace(ks_name)

    tables = execute_native(
        "SELECT columnfamily_name from system.schema_columnfamilies WHERE keyspace_name = ?",
        [ks_name], ONE
    )

    tables = [x.columnfamily_name for x in tables]

    #check for an existing column family
    if raw_cf_name not in tables:
        qs = get_create_table(model)

        try:
            execute_native(qs)
        except CQLEngineException as ex:
            # 1.2 doesn't return cf names, so we have to examine the exception
            # and ignore if it says the column family already exists
            if "Cannot add already existing column family" not in unicode(ex):
                raise
    else:
        # see if we're missing any columns
        fields = get_fields(model)
        field_names = [x.name for x in fields]
        for name, col in model._columns.items():
            if col.primary_key or col.partition_key: continue # we can't mess with the PK
            if col.db_field_name in field_names: continue # skip columns already defined

            # add missing column using the column def
            query = "ALTER TABLE {} add {}".format(cf_name, col.get_column_def())
            logger.debug(query)
            execute_native(query)

        update_compaction(model)

    #get existing index names, skip ones that already exist
    with connection_manager() as con:
        _, idx_names = con.execute(
            "SELECT index_name from system.\"IndexInfo\" WHERE table_name=:table_name",
            {'table_name': raw_cf_name},
            ONE
        )

    idx_names = [i[0] for i in idx_names]
    idx_names = filter(None, idx_names)

    indexes = [c for n,c in model._columns.items() if c.index]
    if indexes:
        for column in indexes:
            if column.db_index_name in idx_names: continue
            qs = ['CREATE INDEX index_{}_{}'.format(raw_cf_name, column.db_field_name)]
            qs += ['ON {}'.format(cf_name)]
            qs += ['("{}")'.format(column.db_field_name)]
            qs = ' '.join(qs)

            try:
                execute(qs)
            except CQLEngineException:
                # index already exists
                pass

def get_create_table(model):
    cf_name = model.column_family_name()
    qs = ['CREATE TABLE {}'.format(cf_name)]

    #add column types
    pkeys = [] # primary keys
    ckeys = [] # clustering keys
    qtypes = [] # field types
    def add_column(col):
        s = col.get_column_def()
        if col.primary_key:
            keys = (pkeys if col.partition_key else ckeys)
            keys.append('"{}"'.format(col.db_field_name))
        qtypes.append(s)
    for name, col in model._columns.items():
        add_column(col)

    qtypes.append('PRIMARY KEY (({}){})'.format(', '.join(pkeys), ckeys and ', ' + ', '.join(ckeys) or ''))

    qs += ['({})'.format(', '.join(qtypes))]

    with_qs = ['read_repair_chance = {}'.format(model.__read_repair_chance__)]

    _order = ['"{}" {}'.format(c.db_field_name, c.clustering_order or 'ASC') for c in model._clustering_keys.values()]

    if _order:
        with_qs.append('clustering order by ({})'.format(', '.join(_order)))

    compaction_options = get_compaction_options(model)

    if compaction_options:
        compaction_options = json.dumps(compaction_options).replace('"', "'")
        with_qs.append("compaction = {}".format(compaction_options))

    # add read_repair_chance
    qs += ['WITH {}'.format(' AND '.join(with_qs))]

    qs = ' '.join(qs)
    return qs

def get_compaction_options(model):
    """
    Generates dictionary (later converted to a string) for creating and altering
    tables with compaction strategy

    :param model:
    :return:
    """
    if not model.__compaction__:
        return {}

    result = {'class':model.__compaction__}

    def setter(key, limited_to_strategy = None):
        """
        sets key in result, checking if the key is limited to either SizeTiered or Leveled
        :param key: one of the compaction options, like "bucket_high"
        :param limited_to_strategy: SizeTieredCompactionStrategy, LeveledCompactionStrategy
        :return:
        """
        mkey = "__compaction_{}__".format(key)
        tmp = getattr(model, mkey)
        if tmp and limited_to_strategy and limited_to_strategy != model.__compaction__:
            raise CQLEngineException("{} is limited to {}".format(key, limited_to_strategy))

        if tmp:
            # Explicitly cast the values to strings to be able to compare the
            # values against introspected values from Cassandra.
            result[key] = str(tmp)

    setter('tombstone_compaction_interval')
    setter('tombstone_threshold')

    setter('bucket_high', SizeTieredCompactionStrategy)
    setter('bucket_low', SizeTieredCompactionStrategy)
    setter('max_threshold', SizeTieredCompactionStrategy)
    setter('min_threshold', SizeTieredCompactionStrategy)
    setter('min_sstable_size', SizeTieredCompactionStrategy)

    setter('sstable_size_in_mb', LeveledCompactionStrategy)

    return result

def get_fields(model):
    # returns all fields that aren't part of the PK
    ks_name = model._get_keyspace()
    col_family = model.column_family_name(include_keyspace=False)

    query = "select * from system.schema_columns where keyspace_name = ? and columnfamily_name = ?"
    tmp = execute_native(query, [ks_name, col_family])

    # Tables containing only primary keys do not appear to create
    # any entries in system.schema_columns, as only non-primary-key attributes
    # appear to be inserted into the schema_columns table
    if not tmp:
        return []

    try:
        return [Field(x.column_name, x.validator) for x in tmp if x.type == 'regular']
    except ValueError:
        return [Field(x.column_name, x.validator) for x in tmp]
    # convert to Field named tuples

def get_table_settings(model):
    return schema_columnfamilies.objects.consistency(ONE).get(
        keyspace_name=model._get_keyspace(),
        columnfamily_name=model.column_family_name(include_keyspace=False))

def update_compaction(model):
    """Updates the compaction options for the given model if necessary.

    :param model: The model to update.

    :return: `True`, if the compaction options were modified in Cassandra,
        `False` otherwise.
    :rtype: bool
    """
    logger.debug("Checking %s for compaction differences", model)
    row = get_table_settings(model)
    # check compaction_strategy_class
    if not model.__compaction__:
        return

    do_update = not row['compaction_strategy_class'].endswith(model.__compaction__)

    existing_options = json.loads(row['compaction_strategy_options'])
    # The min/max thresholds are stored differently in the system data dictionary
    existing_options.update({
        'min_threshold': str(row['min_compaction_threshold']),
        'max_threshold': str(row['max_compaction_threshold']),
        })

    desired_options = get_compaction_options(model)
    desired_options.pop('class', None)

    for k, v in desired_options.items():
        val = existing_options.pop(k, None)
        if val != v:
            do_update = True

    # check compaction_strategy_options
    if do_update:
        options = get_compaction_options(model)
        # jsonify
        options = json.dumps(options).replace('"', "'")
        cf_name = model.column_family_name()
        query = "ALTER TABLE {} with compaction = {}".format(cf_name, options)
        logger.debug(query)
        execute_native(query)
        return True

    return False

def delete_table(model):
    warnings.warn("delete_table has been deprecated in favor of drop_table()", DeprecationWarning)
    return drop_table(model)

def drop_table(model):

    # don't try to delete non existant tables
    meta = get_cluster().metadata

    ks_name = model._get_keyspace()
    raw_cf_name = model.column_family_name(include_keyspace=False)

    try:
        table = meta.keyspaces[ks_name].tables[raw_cf_name]
        execute_native('drop table {};'.format(model.column_family_name(include_keyspace=True)))
    except KeyError:
        pass

<commit_msg>removed a bunch of raw CQL calls in favor of the driver<commit_after>import json
import warnings
from cqlengine import SizeTieredCompactionStrategy, LeveledCompactionStrategy
from cqlengine import ONE
from cqlengine.named import NamedTable

from cqlengine.connection import connection_manager, execute, execute_native, get_session, get_cluster
from cqlengine.exceptions import CQLEngineException

import logging
from collections import namedtuple
Field = namedtuple('Field', ['name', 'type'])

logger = logging.getLogger(__name__)

# system keyspaces
schema_columnfamilies = NamedTable('system', 'schema_columnfamilies')

def create_keyspace(name, strategy_class='SimpleStrategy', replication_factor=3, durable_writes=True, **replication_values):
    """
    creates a keyspace

    :param name: name of keyspace to create
    :param strategy_class: keyspace replication strategy class
    :param replication_factor: keyspace replication factor
    :param durable_writes: 1.2 only, write log is bypassed if set to False
    :param **replication_values: 1.2 only, additional values to ad to the replication data map
    """
    cluster = get_cluster()

    if name not in cluster.metadata.keyspaces:
        #try the 1.2 method
        replication_map = {
            'class': strategy_class,
            'replication_factor':replication_factor
        }
        replication_map.update(replication_values)
        if strategy_class.lower() != 'simplestrategy':
            # Although the Cassandra documentation states for `replication_factor`
            # that it is "Required if class is SimpleStrategy; otherwise,
            # not used." we get an error if it is present.
            replication_map.pop('replication_factor', None)

        query = """
        CREATE KEYSPACE {}
        WITH REPLICATION = {}
        """.format(name, json.dumps(replication_map).replace('"', "'"))

        if strategy_class != 'SimpleStrategy':
            query += " AND DURABLE_WRITES = {}".format('true' if durable_writes else 'false')

        execute_native(query)

def delete_keyspace(name):
    cluster = get_cluster()
    if name in cluster.metadata.keyspaces:
        execute_native("DROP KEYSPACE {}".format(name))

def create_table(model, create_missing_keyspace=True):
    warnings.warn("create_table has been deprecated in favor of sync_table and will be removed in a future release", DeprecationWarning)
    sync_table(model, create_missing_keyspace)

def sync_table(model, create_missing_keyspace=True):
    """
    Inspects the model and creates / updates the corresponding table and columns.

    Note that the attributes removed from the model are not deleted on the database.
    They become effectively ignored by (will not show up on) the model.

    :param create_missing_keyspace: (Defaults to True) Flags to us that we need to create missing keyspace
        mentioned in the model automatically.
    :type create_missing_keyspace: bool
    """

    if model.__abstract__:
        raise CQLEngineException("cannot create table from abstract model")

    #construct query string
    cf_name = model.column_family_name()
    raw_cf_name = model.column_family_name(include_keyspace=False)

    ks_name = model._get_keyspace()
    #create missing keyspace
    if create_missing_keyspace:
        create_keyspace(ks_name)

    cluster = get_cluster()

    tables = cluster.metadata.keyspaces[ks_name].tables

    #check for an existing column family
    if raw_cf_name not in tables:
        qs = get_create_table(model)

        try:
            execute_native(qs)
        except CQLEngineException as ex:
            # 1.2 doesn't return cf names, so we have to examine the exception
            # and ignore if it says the column family already exists
            if "Cannot add already existing column family" not in unicode(ex):
                raise
    else:
        # see if we're missing any columns
        fields = get_fields(model)
        field_names = [x.name for x in fields]
        for name, col in model._columns.items():
            if col.primary_key or col.partition_key: continue # we can't mess with the PK
            if col.db_field_name in field_names: continue # skip columns already defined

            # add missing column using the column def
            query = "ALTER TABLE {} add {}".format(cf_name, col.get_column_def())
            logger.debug(query)
            execute_native(query)

        update_compaction(model)

    #get existing index names, skip ones that already exist
    with connection_manager() as con:
        _, idx_names = con.execute(
            "SELECT index_name from system.\"IndexInfo\" WHERE table_name=:table_name",
            {'table_name': raw_cf_name},
            ONE
        )

    idx_names = [i[0] for i in idx_names]
    idx_names = filter(None, idx_names)

    indexes = [c for n,c in model._columns.items() if c.index]
    if indexes:
        for column in indexes:
            if column.db_index_name in idx_names: continue
            qs = ['CREATE INDEX index_{}_{}'.format(raw_cf_name, column.db_field_name)]
            qs += ['ON {}'.format(cf_name)]
            qs += ['("{}")'.format(column.db_field_name)]
            qs = ' '.join(qs)

            try:
                execute(qs)
            except CQLEngineException:
                # index already exists
                pass

def get_create_table(model):
    cf_name = model.column_family_name()
    qs = ['CREATE TABLE {}'.format(cf_name)]

    #add column types
    pkeys = [] # primary keys
    ckeys = [] # clustering keys
    qtypes = [] # field types
    def add_column(col):
        s = col.get_column_def()
        if col.primary_key:
            keys = (pkeys if col.partition_key else ckeys)
            keys.append('"{}"'.format(col.db_field_name))
        qtypes.append(s)
    for name, col in model._columns.items():
        add_column(col)

    qtypes.append('PRIMARY KEY (({}){})'.format(', '.join(pkeys), ckeys and ', ' + ', '.join(ckeys) or ''))

    qs += ['({})'.format(', '.join(qtypes))]

    with_qs = ['read_repair_chance = {}'.format(model.__read_repair_chance__)]

    _order = ['"{}" {}'.format(c.db_field_name, c.clustering_order or 'ASC') for c in model._clustering_keys.values()]

    if _order:
        with_qs.append('clustering order by ({})'.format(', '.join(_order)))

    compaction_options = get_compaction_options(model)

    if compaction_options:
        compaction_options = json.dumps(compaction_options).replace('"', "'")
        with_qs.append("compaction = {}".format(compaction_options))

    # add read_repair_chance
    qs += ['WITH {}'.format(' AND '.join(with_qs))]

    qs = ' '.join(qs)
    return qs

def get_compaction_options(model):
    """
    Generates dictionary (later converted to a string) for creating and altering
    tables with compaction strategy

    :param model:
    :return:
    """
    if not model.__compaction__:
        return {}

    result = {'class':model.__compaction__}

    def setter(key, limited_to_strategy = None):
        """
        sets key in result, checking if the key is limited to either SizeTiered or Leveled
        :param key: one of the compaction options, like "bucket_high"
        :param limited_to_strategy: SizeTieredCompactionStrategy, LeveledCompactionStrategy
        :return:
        """
        mkey = "__compaction_{}__".format(key)
        tmp = getattr(model, mkey)
        if tmp and limited_to_strategy and limited_to_strategy != model.__compaction__:
            raise CQLEngineException("{} is limited to {}".format(key, limited_to_strategy))

        if tmp:
            # Explicitly cast the values to strings to be able to compare the
            # values against introspected values from Cassandra.
            result[key] = str(tmp)

    setter('tombstone_compaction_interval')
    setter('tombstone_threshold')

    setter('bucket_high', SizeTieredCompactionStrategy)
    setter('bucket_low', SizeTieredCompactionStrategy)
    setter('max_threshold', SizeTieredCompactionStrategy)
    setter('min_threshold', SizeTieredCompactionStrategy)
    setter('min_sstable_size', SizeTieredCompactionStrategy)

    setter('sstable_size_in_mb', LeveledCompactionStrategy)

    return result

def get_fields(model):
    # returns all fields that aren't part of the PK
    ks_name = model._get_keyspace()
    col_family = model.column_family_name(include_keyspace=False)

    query = "select * from system.schema_columns where keyspace_name = ? and columnfamily_name = ?"
    tmp = execute_native(query, [ks_name, col_family])

    # Tables containing only primary keys do not appear to create
    # any entries in system.schema_columns, as only non-primary-key attributes
    # appear to be inserted into the schema_columns table
    if not tmp:
        return []

    try:
        return [Field(x.column_name, x.validator) for x in tmp if x.type == 'regular']
    except ValueError:
        return [Field(x.column_name, x.validator) for x in tmp]
    # convert to Field named tuples

def get_table_settings(model):
    return schema_columnfamilies.objects.consistency(ONE).get(
        keyspace_name=model._get_keyspace(),
        columnfamily_name=model.column_family_name(include_keyspace=False))

def update_compaction(model):
    """Updates the compaction options for the given model if necessary.

    :param model: The model to update.

    :return: `True`, if the compaction options were modified in Cassandra,
        `False` otherwise.
    :rtype: bool
    """
    logger.debug("Checking %s for compaction differences", model)
    row = get_table_settings(model)
    # check compaction_strategy_class
    if not model.__compaction__:
        return

    do_update = not row['compaction_strategy_class'].endswith(model.__compaction__)

    existing_options = json.loads(row['compaction_strategy_options'])
    # The min/max thresholds are stored differently in the system data dictionary
    existing_options.update({
        'min_threshold': str(row['min_compaction_threshold']),
        'max_threshold': str(row['max_compaction_threshold']),
        })

    desired_options = get_compaction_options(model)
    desired_options.pop('class', None)

    for k, v in desired_options.items():
        val = existing_options.pop(k, None)
        if val != v:
            do_update = True

    # check compaction_strategy_options
    if do_update:
        options = get_compaction_options(model)
        # jsonify
        options = json.dumps(options).replace('"', "'")
        cf_name = model.column_family_name()
        query = "ALTER TABLE {} with compaction = {}".format(cf_name, options)
        logger.debug(query)
        execute_native(query)
        return True

    return False

def delete_table(model):
    warnings.warn("delete_table has been deprecated in favor of drop_table()", DeprecationWarning)
    return drop_table(model)

def drop_table(model):

    # don't try to delete non existant tables
    meta = get_cluster().metadata

    ks_name = model._get_keyspace()
    raw_cf_name = model.column_family_name(include_keyspace=False)

    try:
        table = meta.keyspaces[ks_name].tables[raw_cf_name]
        execute_native('drop table {};'.format(model.column_family_name(include_keyspace=True)))
    except KeyError:
        pass

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from puz import Puzzle

from crossword.compat import range
from crossword.core import Crossword

def from_puz(puzzle):
    known_keys = (
        "width",
        "height",
        "author",
        "copyright"
        "title",
        "solution",
    )
    result = Crossword(puzzle.width, puzzle.height)
    result._format_identifier = Crossword.PUZ
    result.meta.creator = puzzle.author
    result.meta.rights = puzzle.copyright
    result.meta.title = puzzle.title

    rows = []
    for i in range(0, len(puzzle.solution), puzzle.width):
        rows.append(puzzle.solution[i:i + puzzle.width])

    def is_across(x, y):
        if result[x, y].solution == '.':
            return False
        start = x == 0 or (0 <= x - 1 and result[x - 1, y].solution == '.')
        end = (x + 1 < puzzle.width and result[x + 1, y].solution != '.')
        return start and end

    def is_down(x, y):
        if result[x, y].solution == '.':
            return False
        start = y == 0 or (y - 1 >= 0 and result[x, y - 1].solution == '.')
        end = (y + 1 < puzzle.height and result[x, y + 1].solution != '.')
        return start and end

    clue_index = 0
    number = 0
    for y, row in enumerate(rows):
        for x, cell in enumerate(row):
            result[x, y].cell = None
            result[x, y].solution = cell

    for y, row in enumerate(rows):
        for x, cell in enumerate(row):
            is_xy_across = is_across(x, y)
            is_xy_down = is_down(x, y)
            if is_xy_across or is_xy_down:
                number += 1
            if is_xy_across:
                result.clues.across[number] = puzzle.clues[clue_index]
                clue_index += 1
            if is_xy_down:
                result.clues.down[number] = puzzle.clues[clue_index]
                clue_index += 1

    result.block = '.'

    for attr in dir(puzzle):
        if attr in known_keys:
            continue
        if attr.startswith('__'):
            continue
        if callable(getattr(puzzle, attr)):
            continue
        result._format[attr] = getattr(puzzle, attr)

    return result

def to_puz(crossword):
    result = Puzzle()
    result.width = crossword.width
    result.height = crossword.height
    result.author = crossword.meta.creator
    result.copyright = crossword.meta.rights
    result.title = crossword.meta.title

    cells = []
    for row in crossword:
        for cell in row:
            value = None
            if cell.solution is None:
                value = crossword.block if crossword.block else '.'
            else:
                value = cell.solution
            cells.append(value)
    result.solution = ''.join(cells)

    if crossword._format_identifier == Crossword.PUZ:
        for key, value in crossword._format.items():
            setattr(result, key, value)

    return result
<commit_msg>Use our own logic to reconstruct list of clues for .puz objects<commit_after># -*- coding: utf-8 -*-
import collections

from puz import Puzzle

from crossword.compat import range
from crossword.core import Crossword

def from_puz(puzzle):
    known_keys = (
        "width",
        "height",
        "author",
        "copyright"
        "title",
        "clues",
        "solution",
    )
    result = Crossword(puzzle.width, puzzle.height)
    result._format_identifier = Crossword.PUZ
    result.meta.creator = puzzle.author
    result.meta.rights = puzzle.copyright
    result.meta.title = puzzle.title

    rows = []
    for i in range(0, len(puzzle.solution), puzzle.width):
        rows.append(puzzle.solution[i:i + puzzle.width])

    def is_across(x, y):
        if result[x, y].solution == '.':
            return False
        start = x == 0 or (0 <= x - 1 and result[x - 1, y].solution == '.')
        end = (x + 1 < puzzle.width and result[x + 1, y].solution != '.')
        return start and end

    def is_down(x, y):
        if result[x, y].solution == '.':
            return False
        start = y == 0 or (y - 1 >= 0 and result[x, y - 1].solution == '.')
        end = (y + 1 < puzzle.height and result[x, y + 1].solution != '.')
        return start and end

    clue_index = 0
    number = 0
    for y, row in enumerate(rows):
        for x, cell in enumerate(row):
            result[x, y].cell = None
            result[x, y].solution = cell

    for y, row in enumerate(rows):
        for x, cell in enumerate(row):
            is_xy_across = is_across(x, y)
            is_xy_down = is_down(x, y)
            if is_xy_across or is_xy_down:
                number += 1
            if is_xy_across:
                result.clues.across[number] = puzzle.clues[clue_index]
                clue_index += 1
            if is_xy_down:
                result.clues.down[number] = puzzle.clues[clue_index]
                clue_index += 1

    result.block = '.'

    for attr in dir(puzzle):
        if attr in known_keys:
            continue
        if attr.startswith('__'):
            continue
        if callable(getattr(puzzle, attr)):
            continue
        result._format[attr] = getattr(puzzle, attr)

    return result

def to_puz(crossword):
    result = Puzzle()
    result.width = crossword.width
    result.height = crossword.height
    result.author = crossword.meta.creator
    result.copyright = crossword.meta.rights
    result.title = crossword.meta.title

    cells = []
    for row in crossword:
        for cell in row:
            value = None
            if cell.solution is None:
                value = crossword.block if crossword.block else '.'
            else:
                value = cell.solution
            cells.append(value)
    result.solution = ''.join(cells)

    result.clues = []
    clues = collections.defaultdict(list)
    for number, clue in crossword.clues.across(sort=None):
        clues[number].append(clue)
    for number, clue in crossword.clues.down(sort=None):
        clues[number].append(clue)
    for number, clues in sorted(clues.items()):
        for clue in clues:
            result.clues.append(clue)

    if crossword._format_identifier == Crossword.PUZ:
        for key, value in crossword._format.items():
            setattr(result, key, value)

    return result
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import collections

from puz import Puzzle

from crossword import CrosswordException
from crossword.compat import range
from crossword.core import Crossword

def from_puz(puzzle):
    known_keys = (
        "width",
        "height",
        "author",
        "copyright"
        "title",
        "clues",
        "solution",
    )
    result = Crossword(puzzle.width, puzzle.height)
    result._format_identifier = Crossword.PUZ
    result.meta.creator = puzzle.author
    result.meta.rights = puzzle.copyright
    result.meta.title = puzzle.title
    result.block = '.'

    rows = []
    for i in range(0, len(puzzle.solution), puzzle.width):
        rows.append(puzzle.solution[i:i + puzzle.width])

    def is_across(x, y):
        if result[x, y].solution == '.':
            return False
        start = x == 0 or (0 <= x - 1 and result[x - 1, y].solution == '.')
        end = (x + 1 < puzzle.width and result[x + 1, y].solution != '.')
        return start and end

    def is_down(x, y):
        if result[x, y].solution == '.':
            return False
        start = y == 0 or (y - 1 >= 0 and result[x, y - 1].solution == '.')
        end = (y + 1 < puzzle.height and result[x, y + 1].solution != '.')
        return start and end

    for y, row in enumerate(rows):
        for x, cell in enumerate(row):
            result[x, y].cell = None
            result[x, y].solution = cell

    clue_index = 0
    number = 0
    for y, row in enumerate(rows):
        for x, cell in enumerate(row):
            is_xy_across = is_across(x, y)
            is_xy_down = is_down(x, y)
            if is_xy_across or is_xy_down:
                number += 1
            if is_xy_across:
                result.clues.across[number] = puzzle.clues[clue_index]
                clue_index += 1
            if is_xy_down:
                result.clues.down[number] = puzzle.clues[clue_index]
                clue_index += 1

    for attr in dir(puzzle):
        if attr in known_keys:
            continue
        if attr.startswith('__'):
            continue
        if callable(getattr(puzzle, attr)):
            continue
        result._format[attr] = getattr(puzzle, attr)

    return result

def to_puz(crossword):
    result = Puzzle()
    result.width = crossword.width
    result.height = crossword.height
    result.author = crossword.meta.creator
    result.copyright = crossword.meta.rights
    result.title = crossword.meta.title

    cells = []
    for row in crossword:
        for cell in row:
            value = None
            if cell.solution is None:
                value = crossword.block if crossword.block else '.'
            else:
                value = cell.solution
            cells.append(value)
    result.solution = ''.join(cells)

    result.clues = []
    clues = collections.defaultdict(list)
    try:
        for number, clue in crossword.clues.across(sort=int):
            clues[number].append(clue)
    except ValueError:
        raise CrosswordException
    try:
        for number, clue in crossword.clues.down(sort=int):
            clues[number].append(clue)
    except ValueError:
        raise CrosswordException
    for number, clues in sorted(clues.items()):
        for clue in clues:
            result.clues.append(clue)

    if crossword._format_identifier == Crossword.PUZ:
        for key, value in crossword._format.items():
            setattr(result, key, value)

    return result
<commit_msg>crossword: treat blank squares as blocks for clue numbering<commit_after># -*- coding: utf-8 -*-
import collections

from puz import Puzzle

from crossword import CrosswordException
from crossword.compat import range
from crossword.core import Crossword

def from_puz(puzzle):
    known_keys = (
        "width",
        "height",
        "author",
        "copyright"
        "title",
        "clues",
        "solution",
    )
    result = Crossword(puzzle.width, puzzle.height)
    result._format_identifier = Crossword.PUZ
    result.meta.creator = puzzle.author
    result.meta.rights = puzzle.copyright
    result.meta.title = puzzle.title
    result.block = '.'

    rows = []
    for i in range(0, len(puzzle.solution), puzzle.width):
        rows.append(puzzle.solution[i:i + puzzle.width])

    def is_across(x, y):
        if result[x, y].solution in ':.':
            return False
        start = x == 0 or (0 <= x - 1 and result[x - 1, y].solution in ':.')
        end = (x + 1 < puzzle.width and result[x + 1, y].solution not in ':.')
        return start and end

    def is_down(x, y):
        if result[x, y].solution in ':.':
            return False
        start = y == 0 or (y - 1 >= 0 and result[x, y - 1].solution in ':.')
        end = (y + 1 < puzzle.height and result[x, y + 1].solution not in ':.')
        return start and end

    for y, row in enumerate(rows):
        for x, cell in enumerate(row):
            result[x, y].cell = None
            result[x, y].solution = cell

    clue_index = 0
    number = 0
    for y, row in enumerate(rows):
        for x, cell in enumerate(row):
            is_xy_across = is_across(x, y)
            is_xy_down = is_down(x, y)
            if is_xy_across or is_xy_down:
                number += 1
            if is_xy_across:
                result.clues.across[number] = puzzle.clues[clue_index]
                clue_index += 1
            if is_xy_down:
                result.clues.down[number] = puzzle.clues[clue_index]
                clue_index += 1

    for attr in dir(puzzle):
        if attr in known_keys:
            continue
        if attr.startswith('__'):
            continue
        if callable(getattr(puzzle, attr)):
            continue
        result._format[attr] = getattr(puzzle, attr)

    return result

def to_puz(crossword):
    result = Puzzle()
    result.width = crossword.width
    result.height = crossword.height
    result.author = crossword.meta.creator
    result.copyright = crossword.meta.rights
    result.title = crossword.meta.title

    cells = []
    for row in crossword:
        for cell in row:
            value = None
            if cell.solution is None:
                value = crossword.block if crossword.block else '.'
            else:
                value = cell.solution
            cells.append(value)
    result.solution = ''.join(cells)

    result.clues = []
    clues = collections.defaultdict(list)
    try:
        for number, clue in crossword.clues.across(sort=int):
            clues[number].append(clue)
    except ValueError:
        raise CrosswordException
    try:
        for number, clue in crossword.clues.down(sort=int):
            clues[number].append(clue)
    except ValueError:
        raise CrosswordException
    for number, clues in sorted(clues.items()):
        for clue in clues:
            result.clues.append(clue)

    if crossword._format_identifier == Crossword.PUZ:
        for key, value in crossword._format.items():
            setattr(result, key, value)

    return result
<|endoftext|>


<commit_before>from django.test import TestCase

from .models import Feed

from .crawler import update_feeds

class FetchTest(TestCase):
    """ Tests relating to fetching and parsing of feeds. """

    # def test_atom_feed(self):
    #   """ Test fetching and parsing an atom feed. """
    #   feed = Feed(url='')

    def test_feed(self):
        """ Test fetching and parsing an Atom feed. """

        feed = Feed(
            url='http://feeds.rijksoverheid.nl/onderwerpen/telecomgegevens-'
                'voor-opsporing/documenten-en-publicaties.rss')

        # Save and fetch
        feed.save()

<commit_msg>Automated test for all BOF fixtures<commit_after>from django.test import TestCase

from .models import Feed

from .crawler import update_feeds

class FetchTest(TestCase):
    """ Tests relating to fetching and parsing of feeds. """

    # def test_atom_feed(self):
    #   """ Test fetching and parsing an atom feed. """
    #   feed = Feed(url='')

    def test_feed(self):
        """ Test fetching and parsing an Atom feed. """

        feed = Feed(
            url='http://feeds.rijksoverheid.nl/onderwerpen/telecomgegevens-'
                'voor-opsporing/documenten-en-publicaties.rss')

        # Save and fetch
        feed.save()

class BofFeedsTest(TestCase):
    """ Test fetching all feeds in the Bits of Freedom fixture. """

    fixtures = ['feeds_bof']

    def test_fetch(self):
        update_feeds()
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2015 Matthew Earl
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
#     The above copyright notice and this permission notice shall be included
#     in all copies or substantial portions of the Software.
#
#     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
#     OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#     MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
#     NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#     DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#     OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
#     USE OR OTHER DEALINGS IN THE SOFTWARE.

"""
Routines for stacking registered images.

"""

__all__ = (
    'StackedImage',
)

import collections

class _BoundingRect(collections.namedtuple('_Rect', ('x', 'y', 'w', 'h'))):
    """
    A rectangle which bounds a set of points.

    """
    @classmethod
    def from_points(cls, points):
        """Create a bounding rectangle from a set of points."""
        top_left = numpy.min(points, axis=1)
        bottom_right = numpy.max(points, axis=1)

        return _BoundingRect(top_left[0, 0],
                             top_left[1, 0],
                             bottom_right[0, 0] - top_left[0, 0],
                             bottom_right[1, 0] - top_left[1, 0])

    def expand(self, points):
        """Expand the bounding rect with a matrix of points."""
        points = numpy.vstack(numpy.matrix(points), self.corners)

        return self.from_points(points)

    @property
    def corners(self):
        return numpy.matrix([[x, y],
                             [x + w, y],
                             [x, y + h],
                             [x + w, y + h]]).T

    @property
    def size(self):
        return self.w, self.h

def _translate_matrix(v):
    """Return a translation matrix."""
    out = numpy.identity(3)
    out[:2, 2:] = v

    return out

def get_bounding_rect(ims_and_transforms):
    """
    Return a bounding rectangle for a set of (image, transformation) pairs.

    Argument:
        ims_and_transforms: Sequence of image and transformation matrices. Each
            element is an `(im, M)` pair, where `M` converts points in a
            reference coordinate frame into the image's coordinate frame.

    """

    def im_corners(im):
        return numpy.matrix([0, 0],
                            [0, im.shape[0]],
                            [im.shape[1], 0],
                            [im.shape[1], im.shape[0]],
                            dtype=numpy.float64).T

    points = numpy.hstack([M.I * im_corners(im)
                           for im, M in ims_and_transforms])

    return _BoundingRect.from_points(points)

class StackedImage(object):
    """
    Represents an image composed of a set of overlaid images.

    Input images are rotated and translated into a reference coordinate system.
    The output image bounds are determined by a rectangle, whose coordinates
    are in the same reference coordinate system.

    `get_bounding_rect` is used to obtain the bounding rectangle. For example,
    to stack a list of (image, transformation) pairs where the output image
    bounds all the input images:

        stacked = StackedImage(get_bounding_rect(ims_and_transforms))
        for im, M in ims_and_transforms:
            stacked.add_image(im, M)
        # stacked.im is now the stacked image.

    To stack of list of (image, transformation) pairs where the output image
    bounds just the first image:

        stacked = StackedImage(get_bounding_rect(ims_and_transforms[:1]))
        for im, M in ims_and_transforms:
            stacked.add_image(im, M)
        # stacked.im is now the stacked image.

    """

    def __init__(self, rect):
        """
        Initialize a new StackedImage.

        Arguments:
            rect: Rectangle defining the bounds of the output image, relative
                to the reference coordinate system. Bounding rectangles are
                obtained by calling `get_bounding_rect`.

        """
        self._rect = _Rect(*rect)
        self._im = numpy.zeros((self._rect.h, self._rect.w))

    @property
    def im(self):
        return self._im

    def add_image(im, M):
        cv2.warpAffine(im,
                       M * _translate_matrix(-self._rect.corners[:, 0]),
                       self._rect.size,
                       dst=self._im,
                       borderMode=cv2.BORDER_TRANSPARENT,
                       flags=cv2.WARP_INVERSE_MAP)

<commit_msg>Fix imports and whitespace issues in stack.py<commit_after>#!/usr/bin/python
# Copyright (c) 2015 Matthew Earl
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
#     The above copyright notice and this permission notice shall be included
#     in all copies or substantial portions of the Software.
#
#     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
#     OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#     MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
#     NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#     DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#     OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
#     USE OR OTHER DEALINGS IN THE SOFTWARE.

"""
Routines for stacking registered images.

"""

__all__ = (
    'get_bounding_rect',
    'StackedImage',
)

import collections

import cv2
import numpy

class _BoundingRect(collections.namedtuple('_Rect', ('x', 'y', 'w', 'h'))):
    """
    A rectangle which bounds a set of points.

    """
    @classmethod
    def from_points(cls, points):
        """Create a bounding rectangle from a set of points."""
        top_left = numpy.min(points, axis=1)
        bottom_right = numpy.max(points, axis=1)

        return _BoundingRect(top_left[0, 0],
                             top_left[1, 0],
                             bottom_right[0, 0] - top_left[0, 0],
                             bottom_right[1, 0] - top_left[1, 0])

    def expand(self, points):
        """Expand the bounding rect with a matrix of points."""
        points = numpy.vstack(numpy.matrix(points), self.corners)

        return self.from_points(points)

    @property
    def corners(self):
        return numpy.matrix([[x, y],
                             [x + w, y],
                             [x, y + h],
                             [x + w, y + h]]).T

    @property
    def size(self):
        return self.w, self.h

def _translate_matrix(v):
    """Return a translation matrix."""
    out = numpy.identity(3)
    out[:2, 2:] = v

    return out

def get_bounding_rect(ims_and_transforms):
    """
    Return a bounding rectangle for a set of (image, transformation) pairs.

    Argument:
        ims_and_transforms: Sequence of image and transformation matrices. Each
            element is an `(im, M)` pair, where `M` converts points in a
            reference coordinate frame into the image's coordinate frame.

    """

    def im_corners(im):
        return numpy.matrix([0, 0],
                            [0, im.shape[0]],
                            [im.shape[1], 0],
                            [im.shape[1], im.shape[0]],
                            dtype=numpy.float64).T

    points = numpy.hstack([M.I * im_corners(im)
                           for im, M in ims_and_transforms])

    return _BoundingRect.from_points(points)

class StackedImage(object):
    """
    Represents an image composed of a set of overlaid images.

    Input images are rotated and translated into a reference coordinate system.
    The output image bounds are determined by a rectangle, whose coordinates
    are in the same reference coordinate system.

    `get_bounding_rect` is used to obtain the bounding rectangle. For example,
    to stack a list of (image, transformation) pairs where the output image
    bounds all the input images:

        stacked = StackedImage(get_bounding_rect(ims_and_transforms))
        for im, M in ims_and_transforms:
            stacked.add_image(im, M)
        # stacked.im is now the stacked image.

    To stack of list of (image, transformation) pairs where the output image
    bounds just the first image:

        stacked = StackedImage(get_bounding_rect(ims_and_transforms[:1]))
        for im, M in ims_and_transforms:
            stacked.add_image(im, M)
        # stacked.im is now the stacked image.

    """

    def __init__(self, rect):
        """
        Initialize a new StackedImage.

        Arguments:
            rect: Rectangle defining the bounds of the output image, relative
                to the reference coordinate system. Bounding rectangles are
                obtained by calling `get_bounding_rect`.

        """
        self._rect = _Rect(*rect)
        self._im = numpy.zeros((self._rect.h, self._rect.w))

    @property
    def im(self):
        return self._im

    def add_image(self, im, M):
        cv2.warpAffine(im,
                       M * _translate_matrix(-self._rect.corners[:, 0]),
                       self._rect.size,
                       dst=self._im,
                       borderMode=cv2.BORDER_TRANSPARENT,
                       flags=cv2.WARP_INVERSE_MAP)

<|endoftext|>


<commit_before>from linked_list import LinkedList, Node

class Stack(object):
    def __init__(self, iterable=[]):
        self.ll = LinkedList(iterable)

    def push(self, val):
        return self.ll.insert(val)

    def pop(self):
        return self.ll.pop()
<commit_msg>added docstrings<commit_after>from linked_list import LinkedList, Node

class Stack(object):
    """Stack data type is composed using linked list data type"""
    def __init__(self, iterable=[]):
        self.ll = LinkedList(iterable)

    def push(self, val):
        """Add val to top of stack"""
        return self.ll.insert(val)

    def pop(self):
        """Remove and return first element of stack"""
        return self.ll.pop()
<|endoftext|>