<commit_before>def to_color(value):
    import re

    if value is None:
        return None

    if re.match('#[a-fA-F0-9]{6,8}', value):
        return value

    if re.match('#[a-fA-F0-9]{3,4}', value):
        r = '#'
        for i in range(1, len(value)):
            r += value[i] + value[i]

        return r

    try:
        from .color_names import color_names
    except ValueError:
        global color_names

    if value in color_names:
        return color_names[value]

    return None

def check_color(value):
    if value is None:
        return None

    ret = to_color(value)

    if ret is None:
        debug("Warning: Unknown color '{}'".format(value))

    return ret
<commit_msg>Colors::to_color: handle '' like None<commit_after>def to_color(value):
    import re

    if value is None:
        return None

    if re.match('#[a-fA-F0-9]{6,8}', value):
        return value

    if re.match('#[a-fA-F0-9]{3,4}', value):
        r = '#'
        for i in range(1, len(value)):
            r += value[i] + value[i]

        return r

    try:
        from .color_names import color_names
    except ValueError:
        global color_names

    if value in color_names:
        return color_names[value]

    return None

def check_color(value):
    if value is None or value == '':
        return None

    ret = to_color(value)

    if ret is None:
        debug("Warning: Unknown color '{}'".format(value))

    return ret
<|endoftext|>


<commit_before>import numpy as npy
import random as rand
import sys
import Queue
import pickle
import os.path
from pymongo import MongoClient

class KMeansTree:
	isLeave = False
	center = []
	# When leave this field is abused for the data
	children = []

	def __init__(self, isLeave, center, children):
		self.isLeave = isLeave
		self.center = center
		self.children = children

	"""
	@param data		[(features, key),...] List of pairs of features and keys
	@param k		Giving the max amount of leaves a node should have
	@param maxiterations	limiting the iterations for the center finding
	"""
	def buildTree(self, data, k, maxiterations):
		# Output to get a feeling how long it takes (a long time)
		# 76 minutes for 1.000.000 entries with 1024 vectors
		if len(data) > 1000:
			print "len(data): ", len(data)
		# If there are less elements in data than a node should have children...
		if len(data) < k:
			# Add all elements and become a leave node
			self.children = data
			self.isLeave = True
			return
		# Get random centers as starting point
		centers = calg(data,k)

		centersFound = False
		iterations = 0

		# Improve the centers
		while not centersFound and iterations < maxiterations:
			iterations += 1
			clusters = []

			for _ in range(len(centers)):
				clusters.append([])

			# cluster the points around the closest centers
			for position,value in data:
				index = 0
				mindist = sys.maxint
				for i,center in enumerate(centers):
					distance = dist(position, center)
					if distance < mindist:
						mindist = distance
						index = i
				clusters[index].append((position, value))

			centersNew = []
			# calculate new centers
			for cluster in clusters:
				# only if at least one point was added to the cluster it can be kept
				# TODO: this is a problem as certain data could cause all points to be in one cluster
				# like a data set of many equal entries
				# it would cause an not stopped recursion
				# Kind of fixed. See the Todo below
				if not cluster == []:
					center = npy.mean(npy.transpose([i[0] for i in cluster]), axis=1, dtype=npy.int)
					centersNew.append(center)

			# TODO Quick fix for now. If all points fall into one center they just become a child node
			# Might be problematic if there are many of them as that could slow down searches.
			# So they should be splitted in this case.
			if len(centersNew) == 1:
				print "Waring building a child node with", len(data), "entries"
				self.children = data
				self.isLeave = True
				return

			# Check if the centers changed
			if npy.array_equal(centers, centersNew):
				centersFound = True

			centers = centersNew
		for center,cluster in zip(centers,clusters):
			# Create a child for each cluster
			child = KMeansTree(False, center, [])
			# Fill it with values
			child.buildTree(cluster, k, maxiterations)
			# Add chlid to children
			self.children.append(child)

	"""
	@param query		Feature array for the request. Find NNs to this one
	@param wantedNNs	Amount of NNs to be found
	@param maxPointsToTouch	Limit of leaves that get touched. Higher value -> better results but slower calculation
	"""
	def search(self, query, wantedNNs = 1, maxPointsToTouch = 42):
		# for the nodes that get checked later
		nextNodes = Queue.PriorityQueue()
		# for the results
		results = Queue.PriorityQueue()

		# search from root
		self.traverse(nextNodes, results, query)
		# while there are nextNodes and the max amount of points or the number of NNs is not reached
		while (not nextNodes.empty()) and (results.qsize() < maxPointsToTouch or results.qsize() < wantedNNs):
			# get the next clostest node
			_,nextNode = nextNodes.get()
			# and continue searching there
			nextNode.traverse(nextNodes, results, query)
		return results

	"""
	@param nextNodes	PrioQueue for 'checkout-later'-nodes
	@param results		PrioQueue for the results
	@param query		Feature array for the request. Find NNs to this one.
	"""
	def traverse(self, nextNodes, results, query):
		if self.isLeave:
			# put the leave to the results
			for center,value in self.children:
				results.put((dist(query, center),value))
		else:
			# find the closest child
			closestChild = None
			mindist = sys.maxint
			for child in self.children:
				distance = dist(child.center, query)
				if distance < mindist:
					# add the previously clostest child to the later to check nodes
					# doing that all but the actual closest child should get added
					if not closestChild == None:
						nextNodes.put((mindist,closestChild))
					mindist = distance
					# set new closest child
					closestChild = child
				else:
					# add the child to the later to check nodes
					nextNodes.put((distance,child))
			# go on searching in the closest child
			closestChild.traverse(nextNodes, results, query)

	def __str__(self):
		return self.str2("")

	def str2(self,i):
		retval = i
		if self.isLeave:
			retval += i + "     [" + str(self.children) + " - " + str(self.center) + "]"
		else:
			retval += "Node:[" + str(self.center) + "\n"
			for c in self.children:
				retval += c.str2(i + "    ") + "\n"
			retval += i + "]"
		return retval

# Calculate the euclidian distance between two arrays
def dist(array1, array2):
	return npy.linalg.norm(array1 - array2)

# TODO: Crappy center selection algorithm. Should be replaced with something cooler.
def calg(arr,k):
	result = []
	N = 0

	for x,_ in arr:
		N += 1
		if len(result) < k:
			result.append(x)
		else:
			s = int(rand.random() * N)
			if s < k:
				result[s] = x
	return result

# TODO For now set the type of feature here!
usedFeature = 'tinyimg'

"""
Build a tree from a given database containing videodata

@param db	The database

@return		kmeans tree to search on
"""
def buildTreeFromDatabase(db):
	print "Reading data from database"
	videos = db['features']

	vids = videos.find()

	data = []
	for vid in vids:
		if vid['searchable']:
			#print vid['filename']
			scenes = vid['scenes']
			vidHash = vid['_id']
			for scene in scenes:
				# TODO better replaced with enumerate
				sceneId = scene['_id']
				feature = npy.array(scene[usedFeature])
				data.append((feature,(vidHash,sceneId)))

	print "Building Tree"
	tree = KMeansTree(False, [], [])
	tree.buildTree(data, 32, 15)
	return tree

"""
Loads a tree from a file if the file exists, else it
builds the tree from a given database containing videodata

@param db	The database
@param filename	filename of the tree

@return		kmeans tree to search on
"""
def loadOrBuildAndSaveTree(db, filename):
	if os.path.isfile(filename):
		print "Loading Tree from file"
		return pickle.load(open(filename, "rb"))
	tree = buildTreeFromDatabase(db)
	print "Saving Tree"
	pickle.dump(tree, open(filename, "wb"))
	return tree

"""
Search for a scene from a database

@param db		Database containing the query
@param tree		kmeans tree to search on
@param vidHash		id of the video of the query scene
@param sceneId		id of the query scene
@param wantedNNs	amount of NNs you want
@param maxTouches	how many leaves should be touched at max. currently not different to wantedNNs

@return			PrioriyQueue containing the results (>= wantedNNs if the tree is big enough)
"""
def searchForScene(db, tree, vidHash, sceneId, wantedNNs, maxTouches):
	videos = db['features']
	vid = videos.find_one({'_id':vidHash})
	scene = vid['scenes'][sceneId]
	feature = npy.array(scene[usedFeature])
	results = tree.search(feature, wantedNNs, maxTouches)
	# TODO Make this nice
	# Add the newlyUploaded scenes to the results
	searchRest(feature, results)
	return results

newlyUploadedScenes = []

"""
Add a video after the tree is build

@param db	the database the video is in
@param vidHash	hash of the video
"""
def addVideoDynamic(db, vidHash):
	videos = db['features']
	vid = videos.find_one({'_id':vidHash})
	if vid['searchable']:
		scenes = vid['scenes']
		for scene in scenes:
			# TODO better replaced with enumerate
			sceneId = scene['_id']
			feature = npy.array(scene[usedFeature])
			newlyUploadedScenes.append((feature,(vidHash,sceneId)))

"""
Searching the linear part

@param query	query feature
@param results	PrioiryQueue for the results
"""
def searchRest(query, results):
	for feature,scene in newlyUploadedScenes:
		results.put((dist(query,feature),scene))

# Example code
"""
client = MongoClient()
db = client["findvid"]
videos = db["videos"]

vid = videos.find_one({'filename':{'$regex':'.*hardcuts\.mp4.*'}})

usedFeature = 'edges'
tree = loadOrBuildAndSaveTree(db, "treeEdges.p")

usedFeature = 'colorhist'
tree = loadOrBuildAndSaveTree(db, "treeColorhist.p")

usedFeature = 'tinyimg'
tree = loadOrBuildAndSaveTree(db, "treeTinyimg.p")

addVideoDynamic(db, vid["_id"])

results = searchForScene(db, tree, vid['_id'], 0, 100, 1000)

print results.get()
print results.get()
print results.get()
print results.get()
print results.get()
#"""

"""#
print "Building test data..."
data = []
for i in range(0, 10000):
	data.append((npy.random.randint(0, 1000000, 1024), "Data-" + str(i)))

print "Building tree..."
tree = KMeansTree(False, [], [])
tree.buildTree(data, 8, 10)

#" ""
print "Saving data..."
pickle.dump(data, open("data.p", "wb"))

print "Saving tree..."
pickle.dump(tree, open("tree.p", "wb"))
#" ""

print "Loading tree..."
data = pickle.load(open("data.p", "rb"))
print "Loading data..."
tree = pickle.load(open("tree.p", "rb"))
#" ""
query = npy.random.randint(0, 1000000, 1024)

#print "Tree: ", str(tree)
nns = tree.search(query, 5, 1000)

print "Q: ", query
for i in range(0, 5):
	nn = nns.get()
	print "NN #", i, ": ", nn

raw_input("\nPress key to start sanity check\n")

print "1 Tree search (5 NNs, 1000 Touches):"
nns = tree.search(query, 5, 1000)
print nns.get()
print nns.get()
print nns.get()
print nns.get()
print nns.get()

print "\n1 Linear search:"

closest = None
mindist = sys.maxint
for point,name in data:
	distance = dist(point, query)
	if distance < mindist:
		mindist = distance
		closest = (distance, name)

print closest
"""
<commit_msg>Changed all calls to get a collection not a database<commit_after>import numpy as npy
import random as rand
import sys
import Queue
import pickle
import os.path
from pymongo import MongoClient

newlyUploadedScenes = []
usedFeature = 'tinyimg'

"""
Loads a tree from a file if the file exists, else it
builds the tree from a given a collection containing videodata

@param videos	The collection
@param filename	filename of the tree

@return		kmeans tree to search on
"""
def loadOrBuildAndSaveTree(videos, filename):
	if os.path.isfile(filename):
		print "Loading Tree from file"
		return pickle.load(open(filename, "rb"))
	tree = buildTreeFromCollection(videos)
	print "Saving Tree"
	pickle.dump(tree, open(filename, "wb"))
	return tree

"""
Build a tree from a given collection containing videodata

@param videos	The collection containing the videos

@return		kmeans tree to search on
"""
def buildTreeFromCollection(videos):
	print "Reading data from database"

	# Get all searchable videos. This also gets rid of the config entry
	vids = videos.find({'searchable' : True})

	data = []
	for vid in vids:
		scenes = vid['scenes']
		vidHash = vid['_id']
		for scene in scenes:
			sceneId = scene['_id']
			# TODO Add call to some smart flatening here
			feature = npy.array(scene[usedFeature])
			data.append((feature,(vidHash,sceneId)))

	print "Building Tree"
	tree = KMeansTree(False, [], [])
	tree.buildTree(data, 32, 15)
	return tree

"""
Search for a scene from a collection

@param videos		Collection containing the query
@param tree		kmeans tree to search on
@param vidHash		id of the video of the query scene
@param sceneId		id of the query scene
@param wantedNNs	amount of NNs you want
@param maxTouches	how many leaves should be touched at max. currently not different to wantedNNs

@return			PrioriyQueue containing the results (>= wantedNNs if the tree is big enough)
"""
def searchForScene(videos, tree, vidHash, sceneId, wantedNNs, maxTouches):
	vid = videos.find_one({'_id':vidHash})
	scene = vid['scenes'][sceneId]
	feature = npy.array(scene[usedFeature])
	results = tree.search(feature, wantedNNs, maxTouches)
	# Add the newlyUploaded scenes to the results
	searchRest(feature, results)
	return results

"""
Add a video after the tree is build

@param videos	the collection the video is in
@param vidHash	hash of the video
"""
def addVideoDynamic(videos, vidHash):
	# TODO set searchable flag
	vid = videos.find_one({'_id':vidHash})
	if vid['searchable']:
		scenes = vid['scenes']
		for scene in scenes:
			sceneId = scene['_id']
			feature = npy.array(scene[usedFeature])
			newlyUploadedScenes.append((feature,(vidHash,sceneId)))

"""
Searching the linear part

@param query	query feature
@param results	PrioiryQueue for the results
"""
def searchRest(query, results):
	for feature,scene in newlyUploadedScenes:
		results.put((dist(query,feature),scene))

class KMeansTree:
	isLeave = False
	center = []
	# When leave this field is abused for the data
	children = []

	def __init__(self, isLeave, center, children):
		self.isLeave = isLeave
		self.center = center
		self.children = children

	"""
	@param data		[(features, key),...] List of pairs of features and keys
	@param k		Giving the max amount of leaves a node should have
	@param maxiterations	limiting the iterations for the center finding
	"""
	def buildTree(self, data, k, maxiterations):
		# Output to get a feeling how long it takes (a long time)
		# 76 minutes for 1.000.000 entries with 1024 vectors
		if len(data) > 1000:
			print "len(data): ", len(data)
		# If there are less elements in data than a node should have children...
		if len(data) < k:
			# Add all elements and become a leave node
			self.children = data
			self.isLeave = True
			return
		# Get random centers as starting point
		centers = calg(data,k)

		centersFound = False
		iterations = 0

		# Improve the centers
		while not centersFound and iterations < maxiterations:
			iterations += 1
			clusters = []

			for _ in range(len(centers)):
				clusters.append([])

			# cluster the points around the closest centers
			for position,value in data:
				index = 0
				mindist = sys.maxint
				for i,center in enumerate(centers):
					distance = dist(position, center)
					if distance < mindist:
						mindist = distance
						index = i
				clusters[index].append((position, value))

			centersNew = []
			# calculate new centers
			for cluster in clusters:
				# only if at least one point was added to the cluster it can be kept
				# TODO: this is a problem as certain data could cause all points to be in one cluster
				# like a data set of many equal entries
				# it would cause an not stopped recursion
				# Kind of fixed. See the Todo below
				if not cluster == []:
					center = npy.mean(npy.transpose([i[0] for i in cluster]), axis=1, dtype=npy.int)
					centersNew.append(center)

			# TODO Quick fix for now. If all points fall into one center they just become a child node
			# Might be problematic if there are many of them as that could slow down searches.
			# So they should be splitted in this case.
			if len(centersNew) == 1:
				print "Waring building a child node with", len(data), "entries"
				self.children = data
				self.isLeave = True
				return

			# Check if the centers changed
			if npy.array_equal(centers, centersNew):
				centersFound = True

			centers = centersNew
		for center,cluster in zip(centers,clusters):
			# Create a child for each cluster
			child = KMeansTree(False, center, [])
			# Fill it with values
			child.buildTree(cluster, k, maxiterations)
			# Add chlid to children
			self.children.append(child)

	"""
	@param query		Feature array for the request. Find NNs to this one
	@param wantedNNs	Amount of NNs to be found
	@param maxPointsToTouch	Limit of leaves that get touched. Higher value -> better results but slower calculation
	"""
	def search(self, query, wantedNNs = 1, maxPointsToTouch = 42):
		# for the nodes that get checked later
		nextNodes = Queue.PriorityQueue()
		# for the results
		results = Queue.PriorityQueue()

		# search from root
		self.traverse(nextNodes, results, query)
		# while there are nextNodes and the max amount of points or the number of NNs is not reached
		while (not nextNodes.empty()) and (results.qsize() < maxPointsToTouch or results.qsize() < wantedNNs):
			# get the next clostest node
			_,nextNode = nextNodes.get()
			# and continue searching there
			nextNode.traverse(nextNodes, results, query)
		return results

	"""
	@param nextNodes	PrioQueue for 'checkout-later'-nodes
	@param results		PrioQueue for the results
	@param query		Feature array for the request. Find NNs to this one.
	"""
	def traverse(self, nextNodes, results, query):
		if self.isLeave:
			# put the leave to the results
			for center,value in self.children:
				results.put((dist(query, center),value))
		else:
			# find the closest child
			closestChild = None
			mindist = sys.maxint
			for child in self.children:
				distance = dist(child.center, query)
				if distance < mindist:
					# add the previously clostest child to the later to check nodes
					# doing that all but the actual closest child should get added
					if not closestChild == None:
						nextNodes.put((mindist,closestChild))
					mindist = distance
					# set new closest child
					closestChild = child
				else:
					# add the child to the later to check nodes
					nextNodes.put((distance,child))
			# go on searching in the closest child
			closestChild.traverse(nextNodes, results, query)

	def __str__(self):
		return self.str2("")

	def str2(self,i):
		retval = i
		if self.isLeave:
			retval += i + "     [" + str(self.children) + " - " + str(self.center) + "]"
		else:
			retval += "Node:[" + str(self.center) + "\n"
			for c in self.children:
				retval += c.str2(i + "    ") + "\n"
			retval += i + "]"
		return retval

# Calculate the euclidian distance between two arrays
def dist(array1, array2):
	return npy.linalg.norm(array1 - array2)

# TODO: Crappy center selection algorithm. Should be replaced with something cooler.
def calg(arr,k):
	result = []
	N = 0

	for x,_ in arr:
		N += 1
		if len(result) < k:
			result.append(x)
		else:
			s = int(rand.random() * N)
			if s < k:
				result[s] = x
	return result

# Example code
"""
client = MongoClient()
db = client["findvid"]
videos = db["videos"]

vid = videos.find_one({'filename':{'$regex':'.*hardcuts\.mp4.*'}})

usedFeature = 'edges'
tree = loadOrBuildAndSaveTree(videos, "treeEdges.p")

usedFeature = 'colorhist'
tree = loadOrBuildAndSaveTree(videos, "treeColorhist.p")

usedFeature = 'tinyimg'
tree = loadOrBuildAndSaveTree(videos, "treeTinyimg.p")

addVideoDynamic(videos, vid["_id"])

results = searchForScene(videos, tree, vid['_id'], 0, 100, 1000)

print results.get()
print results.get()
print results.get()
print results.get()
print results.get()
#"""
<|endoftext|>


<commit_before>from docutils.nodes import literal, Text
from docutils.parsers.rst import roles

from sphinx.roles import XRefRole
from sphinx import version_info as sphinx_version_info

class SymbolRole(XRefRole):
    def __call__(self, *args, **kwargs):
        nodes, messages = XRefRole.__call__(self, *args, **kwargs)
        for node in nodes:
            attrs = node.attributes
            target = attrs['reftarget']
            parens = ''
            if target.endswith('()'):
                # Function call, :symbol:`mongoc_init()`
                target = target[:-2]
                parens = '()'

            if ':' in target:
                # E.g., 'bson:bson_t' has domain 'bson', target 'bson_t'
                attrs['domain'], name = target.split(':', 1)
                attrs['reftarget'] = name

                assert isinstance(node.children[0].children[0], Text)
                node.children[0].children[0] = Text(name + parens,
                                                    name + parens)

            else:
                attrs['reftarget'] = target

            attrs['reftype'] = 'doc'
            attrs['classes'].append('symbol')

            if sphinx_version_info >= (1, 6):
                # https://github.com/sphinx-doc/sphinx/issues/3698
                attrs['refdomain'] = 'std'

        return nodes, messages

def setup(app):
    roles.register_local_role(
        'symbol', SymbolRole(warn_dangling=True, innernodeclass=literal))

    return {
        'version': '1.0',
        'parallel_read_safe': True,
        'parallel_write_safe': True,
    }
<commit_msg>CDRIVER-2161 sphinx 1.6 compatibility<commit_after>from docutils.nodes import literal, Text
from docutils.parsers.rst import roles

from sphinx.roles import XRefRole
from sphinx import version_info as sphinx_version_info

class SymbolRole(XRefRole):
    def __call__(self, *args, **kwargs):
        nodes, messages = XRefRole.__call__(self, *args, **kwargs)
        for node in nodes:
            attrs = node.attributes
            target = attrs['reftarget']
            parens = ''
            if target.endswith('()'):
                # Function call, :symbol:`mongoc_init()`
                target = target[:-2]
                parens = '()'

            if ':' in target:
                # E.g., 'bson:bson_t' has domain 'bson', target 'bson_t'
                attrs['domain'], name = target.split(':', 1)
                attrs['reftarget'] = name

                old = node.children[0].children[0]
                assert isinstance(old, Text)
                new = Text(name + parens, name + parens)
                # Ensure setup_child is called.
                node.children[0].replace(old, new)

            else:
                attrs['reftarget'] = target

            attrs['reftype'] = 'doc'
            attrs['classes'].append('symbol')

            if sphinx_version_info >= (1, 6):
                # https://github.com/sphinx-doc/sphinx/issues/3698
                attrs['refdomain'] = 'std'

        return nodes, messages

def setup(app):
    roles.register_local_role(
        'symbol', SymbolRole(warn_dangling=True, innernodeclass=literal))

    return {
        'version': '1.0',
        'parallel_read_safe': True,
        'parallel_write_safe': True,
    }
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals

import os

from django.db import models

from apps.core.fields import HashField

class Directory(models.Model):
    """
    Unix-style directory path.
    """
    path = models.CharField(max_length=255, unique=True)

    class Meta:
        db_table = 'directories'
        verbose_name_plural = 'directories'
        ordering = ('path',)

    def __unicode__(self):
        return self.path

    def list_repr(self):
        """
        String representation in lists
        """
        return self.path

class File(models.Model):
    """
    A file in a directory.
    """
    name = models.CharField(max_length=255, db_index=True)
    directory = models.ForeignKey(Directory, db_column='dir')

    class Meta:
        db_table = 'files'
        ordering = ('name',)

    def __unicode__(self):
        return '%s/%s' % (self.directory.path, self.name)

    def list_repr(self):
        """
        String representation in lists
        """
        return '%s/%s' % (self.directory.path, self.name)

    @classmethod
    def filter(cls, search_term, order_by=Meta.ordering):
        path_part, file_part = os.path.split(search_term)

        files = dirs = None

        # collecting the data from two tables
        if file_part and not path_part:
            files = cls.objects.filter(name__icontains=file_part)
            dirs = Directory.objects.filter(path__icontains=file_part)

        if path_part and not file_part:
            dirs = Directory.objects.filter(path__icontains=path_part)

        if path_part and file_part:
            files = cls.objects.filter(name__icontains=file_part)
            dirs = Directory.objects.filter(path__icontains=search_term)

        resulting_files = []

        # prepare results from collected data
        if files and not dirs:
            resulting_files = files

        if dirs and not files:
            resulting_files = cls.objects.filter(directory__in=dirs)

        if dirs and files:
            resulting_files = files | cls.objects.filter(directory__in=dirs)

        try:
            return resulting_files.order_by(*order_by)
        except AttributeError:  # It's a plain list
            return resulting_files

class Algorithm(models.Model):
    """
    A hashing algorithm.
    """
    name = models.CharField(max_length=20)

    class Meta:
        db_table = 'algorithms'
        ordering = ('name',)

    def __unicode__(self):
        return self.name

    def list_repr(self):
        """
        String representation in lists
        """
        return self.name

class FileHash(models.Model):
    """
    A file hash.
    """
    file = models.ForeignKey(File, on_delete=models.CASCADE, db_column='file')
    product = models.ForeignKey('devices.Product', db_column='product')
    device = models.IntegerField(null=False, default=0)  # TODO convert to nullable(?) FK
    algorithm = models.ForeignKey(Algorithm, db_column='algo', on_delete=models.PROTECT)
    hash = HashField(db_column='hash')

    class Meta:
        db_table = 'file_hashes'
        verbose_name_plural = 'file hashes'

    def __unicode__(self):
        return '%s (%s)' % (self.hash, self.algorithm)

    def list_repr(self):
        """
        String representation in lists
        """
        return '%s (%s)' % (self.hash, self.algorithm)
<commit_msg>File hashes currently don't have a device assigned<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals

import os

from django.db import models

from apps.core.fields import HashField

class Directory(models.Model):
    """
    Unix-style directory path.
    """
    path = models.CharField(max_length=255, unique=True)

    class Meta:
        db_table = 'directories'
        verbose_name_plural = 'directories'
        ordering = ('path',)

    def __unicode__(self):
        return self.path

    def list_repr(self):
        """
        String representation in lists
        """
        return self.path

class File(models.Model):
    """
    A file in a directory.
    """
    name = models.CharField(max_length=255, db_index=True)
    directory = models.ForeignKey(Directory, db_column='dir')

    class Meta:
        db_table = 'files'
        ordering = ('name',)

    def __unicode__(self):
        return '%s/%s' % (self.directory.path, self.name)

    def list_repr(self):
        """
        String representation in lists
        """
        return '%s/%s' % (self.directory.path, self.name)

    @classmethod
    def filter(cls, search_term, order_by=Meta.ordering):
        path_part, file_part = os.path.split(search_term)

        files = dirs = None

        # collecting the data from two tables
        if file_part and not path_part:
            files = cls.objects.filter(name__icontains=file_part)
            dirs = Directory.objects.filter(path__icontains=file_part)

        if path_part and not file_part:
            dirs = Directory.objects.filter(path__icontains=path_part)

        if path_part and file_part:
            files = cls.objects.filter(name__icontains=file_part)
            dirs = Directory.objects.filter(path__icontains=search_term)

        resulting_files = []

        # prepare results from collected data
        if files and not dirs:
            resulting_files = files

        if dirs and not files:
            resulting_files = cls.objects.filter(directory__in=dirs)

        if dirs and files:
            resulting_files = files | cls.objects.filter(directory__in=dirs)

        try:
            return resulting_files.order_by(*order_by)
        except AttributeError:  # It's a plain list
            return resulting_files

class Algorithm(models.Model):
    """
    A hashing algorithm.
    """
    name = models.CharField(max_length=20)

    class Meta:
        db_table = 'algorithms'
        ordering = ('name',)

    def __unicode__(self):
        return self.name

    def list_repr(self):
        """
        String representation in lists
        """
        return self.name

class FileHash(models.Model):
    """
    A file hash.
    """
    file = models.ForeignKey(File, on_delete=models.CASCADE, db_column='file')
    product = models.ForeignKey('devices.Product', db_column='product')
    device = models.ForeignKey('devices.Device', null=True, blank=True)
    algorithm = models.ForeignKey(Algorithm, db_column='algo', on_delete=models.PROTECT)
    hash = HashField(db_column='hash')

    class Meta:
        db_table = 'file_hashes'
        verbose_name_plural = 'file hashes'

    def __unicode__(self):
        return '%s (%s)' % (self.hash, self.algorithm)

    def list_repr(self):
        """
        String representation in lists
        """
        return '%s (%s)' % (self.hash, self.algorithm)
<|endoftext|>


<commit_before>from cqlengine.operators import BaseWhereOperator, BaseAssignmentOperator

class StatementException(Exception): pass

class BaseClause(object):

    def __init__(self, field, operator, value):
        self.field = field
        self.operator = operator
        self.value = value

    def __unicode__(self):
        return u'"{}" {} {}'.format(self.field, self.operator, self.value)

    def __str__(self):
        return str(unicode(self))

class WhereClause(BaseClause):
    """ a single where statement used in queries """

    def __init__(self, field, operator, value):
        if not isinstance(operator, BaseWhereOperator):
            raise StatementException(
                "operator must be of type {}, got {}".format(BaseWhereOperator, type(operator))
            )
        super(WhereClause, self).__init__(field, operator, value)

class AssignmentClause(BaseClause):
    """ a single variable st statement """

    def __init__(self, field, operator, value):
        if not isinstance(operator, BaseAssignmentOperator):
            raise StatementException(
                "operator must be of type {}, got {}".format(BaseAssignmentOperator, type(operator))
            )
        super(AssignmentClause, self).__init__(field, operator, value)

class BaseCQLStatement(object):
    """ The base cql statement class """

    def __init__(self, table, consistency=None, where=None):
        super(BaseCQLStatement, self).__init__()
        self.table = table
        self.consistency = consistency
        self.where_clauses = where or []

    def add_where_clause(self, clause):
        if not isinstance(clause, WhereClause):
            raise StatementException("only instances of WhereClause can be added to statements")
        self.where_clauses.append(clause)

    def __str__(self):
        return str(unicode(self))

    @property
    def _where(self):
        return 'WHERE {}'.format(' AND '.join([unicode(c) for c in self.where_clauses]))

class SelectStatement(BaseCQLStatement):
    """ a cql select statement """

    def __init__(self,
                 table,
                 fields,
                 consistency=None,
                 where=None,
                 order_by=None,
                 limit=None,
                 allow_filtering=False):

        super(SelectStatement, self).__init__(
            table,
            consistency=consistency,
            where=where
        )

        self.fields = [fields] if isinstance(fields, basestring) else fields
        self.order_by = [order_by] if isinstance(order_by, basestring) else order_by
        self.limit = limit
        self.allow_filtering = allow_filtering

    def __unicode__(self):
        qs = ['SELECT']
        qs += [', '.join(['"{}"'.format(f) for f in self.fields]) if self.fields else '*']
        qs += ['FROM', self.table]

        if self.where_clauses:
            qs += [self._where]

        if self.order_by:
            qs += ['ORDER BY {}'.format(', '.join(unicode(o) for o in self.order_by))]

        if self.limit:
            qs += ['LIMIT {}'.format(self.limit)]

        if self.allow_filtering:
            qs += ['ALLOW FILTERING']

        return ' '.join(qs)

class DMLStatement(BaseCQLStatement):
    """ mutation statements with ttls """

    def __init__(self, table, consistency=None, where=None, ttl=None):
        super(DMLStatement, self).__init__(
            table,
            consistency=consistency,
            where=where)
        self.ttl = ttl

class AssignmentStatement(DMLStatement):
    """ value assignment statements """

    def __init__(self,
                 table,
                 assignments,
                 consistency=None,
                 where=None,
                 ttl=None):
        super(AssignmentStatement, self).__init__(
            table,
            consistency=consistency,
            where=where,
            ttl=ttl
        )
        self.assignments = assignments or []

    def add_assignment_clause(self, clause):
        if not isinstance(clause, AssignmentClause):
            raise StatementException("only instances of AssignmentClause can be added to statements")
        self.assignments.append(clause)

class InsertStatement(AssignmentStatement):
    """ an cql insert select statement """

    def __init__(self, table, values, consistency=None):
        super(InsertStatement, self).__init__(
            table,
            consistency=consistency,
            where=None
        )
        self.values = values

    def add_where_clause(self, clause):
        raise StatementException("Cannot add where clauses to insert statements")

class UpdateStatement(AssignmentStatement):
    """ an cql update select statement """

    def __init__(self, table, consistency=None, where=None):
        super(UpdateStatement, self).__init__(table, consistency, where)

class DeleteStatement(DMLStatement):
    """ a cql delete statement """
<commit_msg>setting up constructors for other statements<commit_after>from cqlengine.operators import BaseWhereOperator, BaseAssignmentOperator

class StatementException(Exception): pass

class BaseClause(object):

    def __init__(self, field, operator, value):
        self.field = field
        self.operator = operator
        self.value = value

    def __unicode__(self):
        return u'"{}" {} {}'.format(self.field, self.operator, self.value)

    def __str__(self):
        return str(unicode(self))

class WhereClause(BaseClause):
    """ a single where statement used in queries """

    def __init__(self, field, operator, value):
        if not isinstance(operator, BaseWhereOperator):
            raise StatementException(
                "operator must be of type {}, got {}".format(BaseWhereOperator, type(operator))
            )
        super(WhereClause, self).__init__(field, operator, value)

class AssignmentClause(BaseClause):
    """ a single variable st statement """

    def __init__(self, field, operator, value):
        if not isinstance(operator, BaseAssignmentOperator):
            raise StatementException(
                "operator must be of type {}, got {}".format(BaseAssignmentOperator, type(operator))
            )
        super(AssignmentClause, self).__init__(field, operator, value)

class BaseCQLStatement(object):
    """ The base cql statement class """

    def __init__(self, table, consistency=None, where=None):
        super(BaseCQLStatement, self).__init__()
        self.table = table
        self.consistency = consistency
        self.where_clauses = where or []

    def add_where_clause(self, clause):
        if not isinstance(clause, WhereClause):
            raise StatementException("only instances of WhereClause can be added to statements")
        self.where_clauses.append(clause)

    def __str__(self):
        return str(unicode(self))

    @property
    def _where(self):
        return 'WHERE {}'.format(' AND '.join([unicode(c) for c in self.where_clauses]))

class SelectStatement(BaseCQLStatement):
    """ a cql select statement """

    def __init__(self,
                 table,
                 fields,
                 consistency=None,
                 where=None,
                 order_by=None,
                 limit=None,
                 allow_filtering=False):

        super(SelectStatement, self).__init__(
            table,
            consistency=consistency,
            where=where
        )

        self.fields = [fields] if isinstance(fields, basestring) else (fields or [])
        self.order_by = [order_by] if isinstance(order_by, basestring) else order_by
        self.limit = limit
        self.allow_filtering = allow_filtering

    def __unicode__(self):
        qs = ['SELECT']
        qs += [', '.join(['"{}"'.format(f) for f in self.fields]) if self.fields else '*']
        qs += ['FROM', self.table]

        if self.where_clauses:
            qs += [self._where]

        if self.order_by:
            qs += ['ORDER BY {}'.format(', '.join(unicode(o) for o in self.order_by))]

        if self.limit:
            qs += ['LIMIT {}'.format(self.limit)]

        if self.allow_filtering:
            qs += ['ALLOW FILTERING']

        return ' '.join(qs)

class DMLStatement(BaseCQLStatement):
    """ mutation statements with ttls """

    def __init__(self, table, consistency=None, where=None, ttl=None):
        super(DMLStatement, self).__init__(
            table,
            consistency=consistency,
            where=where)
        self.ttl = ttl

class AssignmentStatement(DMLStatement):
    """ value assignment statements """

    def __init__(self,
                 table,
                 assignments,
                 consistency=None,
                 where=None,
                 ttl=None):
        super(AssignmentStatement, self).__init__(
            table,
            consistency=consistency,
            where=where,
            ttl=ttl
        )
        self.assignments = assignments or []

    def add_assignment_clause(self, clause):
        if not isinstance(clause, AssignmentClause):
            raise StatementException("only instances of AssignmentClause can be added to statements")
        self.assignments.append(clause)

class InsertStatement(AssignmentStatement):
    """ an cql insert select statement """

    def __init__(self, table, values, consistency=None):
        super(InsertStatement, self).__init__(
            table,
            consistency=consistency,
            where=None
        )
        self.values = values

    def add_where_clause(self, clause):
        raise StatementException("Cannot add where clauses to insert statements")

class UpdateStatement(AssignmentStatement):
    """ an cql update select statement """

    def __init__(self, table, assignments, consistency=None, where=None, ttl=None):
        super(UpdateStatement, self).__init__(
            table,
            assignments,
            consistency=consistency,
            where=where,
            ttl=ttl
        )

class DeleteStatement(DMLStatement):
    """ a cql delete statement """

    def __init__(self, table, fields, consistency=None, where=None, ttl=None):
        super(DeleteStatement, self).__init__(
            table,
            consistency=consistency,
            where=where,
            ttl=ttl
        )
        self.fields = [fields] if isinstance(fields, basestring) else (fields or [])

<|endoftext|>


<commit_before># This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

from django.conf import settings

import sys
sys.path.append("/usr/lib/archivematica/archivematicaCommon/externals")

from django.db import connection, transaction
from django.shortcuts import render
from django.http import HttpResponse
from main import models
from components import helpers

from django import forms
from django.forms.widgets import *

INPUT_ATTRS = {'class': 'span11'}

def getTools():
    query = 'SELECT description FROM FileIDTypes'

    cursor = connection.cursor()
    cursor.execute(query)

    ret = []
    for tool in cursor.fetchall():
        ret.append( (tool[0], tool[0]) )

    return ret

def getCommandTypes():
    query = 'SELECT type from CommandTypes'

    cursor = connection.cursor()
    cursor.execute(query)

    ret = []
    for commandType in cursor.fetchall():
        ret.append( (commandType[0], commandType[0]) )

    return ret

def getCommands(usage = 'command'):

    query = 'Select pk, description from Commands where commandUsage = "{}"'.format(usage)

    cursor = connection.cursor()
    cursor.execute(query)

    ret = []
    for vCommand in cursor.fetchall():
        ret.append( (vCommand[0], vCommand[1]) )

    return ret

def getPurposes():
    query = 'SELECT classification FROM CommandClassifications'

    cursor = connection.cursor()
    cursor.execute(query)

    ret = []
    for purpose in cursor.fetchall():
        ret.append( (purpose[0], purpose[0]) )

    return ret

def getFormatIDs(purpose = 'all'):
    query = 'Select pk, description from FileIDs';
    if purpose == 'access':
        query = 'Select pk, description from FileIDs where validAccessFormat = 1'
    elif purpose == 'preservation':
        query = 'Select pk, description from FileIDs where validPreservationFormat = 1'

    cursor = connection.cursor()
    cursor.execute(query)

    ret = []
    for formatID in cursor.fetchall():
        ret.append( (formatID[0], formatID[1]))

    return ret

class FPRSearchForm(forms.Form):
    query = forms.CharField(label='', required=False, widget=TextInput(attrs=settings.INPUT_ATTRS))

class FPREditFormatID(forms.Form):
    formatID = forms.HiddenInput()
    tool = forms.ChoiceField(choices = getTools(), label= "File Identification Tool")
    formatDescription = forms.CharField(label = 'Format ID Description', required = False, max_length = 100,
        widget = TextInput(attrs = {'class':'Description'}))
    validPreservation = forms.CheckboxInput(label = 'Valid for preservation')
    validAccess = forms.CheckboxInput(label = 'Valid for access')
    enabled = forms.CheckboxInput(label = 'Enabled', check_test=True)

class FPREditCommand(forms.Form):
    COMMAND_USAGE_CHOICES = (('command','command'), ('verification','verification'), ('eventDetail','eventDetail'))

    commandUsage = forms.ChoiceField(choices = COMMAND_USAGE_CHOICES, label='Usage')
    commandType = forms.ChoiceField(choices = getCommandTypes())
    command = forms.CharField(label = 'Command', required = False, max_length = 100,
        widget = TextInput(attrs = {'class':'Description'}))
    outputLocation = forms.CharField(label= 'Output location', required= False, max_length = 255,
        widget =  TextInput(attrs = {'class':'Description'}))
    outputFileFormat = forms.CharField(label= 'Output File Format', required= False, max_length = 15,
        widget =  TextInput(attrs = {'class':'Description'}))
    commandDescription = forms.CharField(label = 'Description', required = False, max_length = 100,
        widget = TextInput(attrs = {'class':'Description'}))
    verificationCommand = forms.ChoiceField(choices = getCommands('verification'), label = 'Verification command', required = False)
    eventDetailCommand = forms.ChoiceField(choices = getCommands('eventDetail'), label = 'Event detail command', required = False)

class FPREditRule(forms.Form):
    purpose = forms.ChoiceField(choices = getPurposes())
    formatID = forms.ChoiceField(choices = getFormatIDs(), label = 'Format ID', required = True)
    command = forms.ChoiceField(choices = getCommands(), label = 'Command', required = True)

<commit_msg>refs #4454<commit_after># This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

from django.conf import settings

import sys
sys.path.append("/usr/lib/archivematica/archivematicaCommon/externals")

from django.db import connection, transaction
from django.shortcuts import render
from django.http import HttpResponse
from main import models
from components import helpers

from django import forms
from django.forms.widgets import *

INPUT_ATTRS = {'class': 'span11'}

def getTools():
    query = 'SELECT description FROM FileIDTypes'

    cursor = connection.cursor()
    cursor.execute(query)

    ret = []
    for tool in cursor.fetchall():
        ret.append( (tool[0], tool[0]) )

    return ret

def getCommandTypes():
    query = 'SELECT type from CommandTypes'

    cursor = connection.cursor()
    cursor.execute(query)

    ret = []
    for commandType in cursor.fetchall():
        ret.append( (commandType[0], commandType[0]) )

    return ret

def getCommands(usage = 'command'):

    query = 'Select pk, description from Commands where commandUsage = "{}"'.format(usage)

    cursor = connection.cursor()
    cursor.execute(query)

    ret = []
    for vCommand in cursor.fetchall():
        ret.append( (vCommand[0], vCommand[1]) )

    return ret

def getPurposes():
    query = 'SELECT classification FROM CommandClassifications'

    cursor = connection.cursor()
    cursor.execute(query)

    ret = []
    for purpose in cursor.fetchall():
        ret.append( (purpose[0], purpose[0]) )

    return ret

def getFormatIDs(purpose = 'all'):
    query = 'Select pk, description from FileIDs';
    if purpose == 'access':
        query = 'Select pk, description from FileIDs where validAccessFormat = 1'
    elif purpose == 'preservation':
        query = 'Select pk, description from FileIDs where validPreservationFormat = 1'

    cursor = connection.cursor()
    cursor.execute(query)

    ret = []
    for formatID in cursor.fetchall():
        ret.append( (formatID[0], formatID[1]))

    return ret

class FPRSearchForm(forms.Form):
    query = forms.CharField(label='', required=False, widget=TextInput(attrs=settings.INPUT_ATTRS))

class FPREditFormatID(forms.Form):
    formatID = forms.HiddenInput()
    tool = forms.ChoiceField(choices = getTools(), label= "File Identification Tool")
    formatDescription = forms.CharField(label = 'Format ID Description', required = False, max_length = 100,
        widget = TextInput(attrs = {'class':'Description'}))
    validPreservation = forms.CheckboxInput()
    validAccess = forms.CheckboxInput()
    enabled = forms.CheckboxInput(check_test=True)

class FPREditCommand(forms.Form):
    COMMAND_USAGE_CHOICES = (('command','command'), ('verification','verification'), ('eventDetail','eventDetail'))

    commandUsage = forms.ChoiceField(choices = COMMAND_USAGE_CHOICES, label='Usage')
    commandType = forms.ChoiceField(choices = getCommandTypes())
    command = forms.CharField(label = 'Command', required = False, max_length = 100,
        widget = TextInput(attrs = {'class':'Description'}))
    outputLocation = forms.CharField(label= 'Output location', required= False, max_length = 255,
        widget =  TextInput(attrs = {'class':'Description'}))
    outputFileFormat = forms.CharField(label= 'Output File Format', required= False, max_length = 15,
        widget =  TextInput(attrs = {'class':'Description'}))
    commandDescription = forms.CharField(label = 'Description', required = False, max_length = 100,
        widget = TextInput(attrs = {'class':'Description'}))
    verificationCommand = forms.ChoiceField(choices = getCommands('verification'), label = 'Verification command', required = False)
    eventDetailCommand = forms.ChoiceField(choices = getCommands('eventDetail'), label = 'Event detail command', required = False)

class FPREditRule(forms.Form):
    purpose = forms.ChoiceField(choices = getPurposes())
    formatID = forms.ChoiceField(choices = getFormatIDs(), label = 'Format ID', required = True)
    command = forms.ChoiceField(choices = getCommands(), label = 'Command', required = True)

<|endoftext|>


<commit_before># $Filename$
# $Authors$
# Last Changed: $Date$ $Committer$ $Revision-Id$
#
# Copyright (c) 2003-2011, German Aerospace Center (DLR)
#
# All rights reserved.
#Redistribution and use in source and binary forms, with or without
#modification, are permitted provided that the following conditions are
#
#met:
#
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the
#   distribution.
#
# * Neither the name of the German Aerospace Center nor the names of
#   its contributors may be used to endorse or promote products derived
#   from this software without specific prior written permission.
#
#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""
This module implements how the meta data is persisted on the SVN server.
"""

import json
import logging
import mimetypes

from datafinder.persistence.adapters.svn.constants import XPS_JSON_PROPERTY, SVN_MIME_TYPE
from datafinder.persistence.adapters.svn.error import SubversionError
from datafinder.persistence.adapters.svn.util import util
from datafinder.persistence.error import PersistenceError
from datafinder.persistence.metadata import constants, value_mapping
from datafinder.persistence.metadata.metadatastorer import NullMetadataStorer

__version__ = "$Revision-Id$"

_log = logging.getLogger()

class MetadataSubversionAdapter(NullMetadataStorer):
    """ Implements meta data storer interface for subversion. """

    def __init__(self, identifier, connectionPool):
        """
        Constructor.

        @param identifier: Logical identifier of the resource.
        @type identifier: C{unicode}
        @param connectionPool: Connection pool.
        @type connectionPool: L{Connection<datafinder.persistence.svn.connection_pool.SVNConnectionPool>}
        @param modelIdentifier: Logical identifier of the model.
        @type modelIdentifier: C{unicode}
        """

        NullMetadataStorer.__init__(self, identifier)
        self.__connectionPool = connectionPool
        self.__persistenceId = util.mapIdentifier(identifier)

    def retrieve(self, propertyIds=None):
        """ @see: L{NullMetadataStorer<datafinder.persistence.metadata.metadatastorer.NullMetadataStorer>}"""

        connection = self.__connectionPool.acquire()
        try:
            rawResult = dict()
            try:
                persistenceJsonProperties = self._retrieveProperties(connection)
                rawResult = json.loads(persistenceJsonProperties)
            except SubversionError:
                _log.debug("No subversion property is set!")
            mappedResult = self._mapRawResult(connection, rawResult)
            return self._filterResult(propertyIds, mappedResult)
        finally:
            self.__connectionPool.release(connection)

    def _retrieveProperties(self, connection):
        """ Retrieves all properties. """

        return connection.getProperty(self.__persistenceId, XPS_JSON_PROPERTY)

    def _mapRawResult(self, connection, rawResult):
        """ Maps the SVN specific result to interface format. """

        try:
            infoDict = connection.info(self.__persistenceId)
        except SubversionError, error:
            errorMessage = "Problem during meta data retrieval. " \
                           + "Reason: '%s'" % error
            raise PersistenceError(errorMessage)
        mappedResult = dict()
        mappedResult[constants.CREATION_DATETIME] = value_mapping.MetadataValue("")
        mappedResult[constants.MODIFICATION_DATETIME] = value_mapping.MetadataValue(infoDict["lastChangedDate"])
        mappedResult[constants.SIZE] = value_mapping.MetadataValue("")
        mappedResult[constants.OWNER] = value_mapping.MetadataValue(infoDict["lastChangedAuthor"])

        try:
            mimeType = connection.getProperty(self.__persistenceId, SVN_MIME_TYPE)
            mappedResult[constants.MIME_TYPE] = value_mapping.MetadataValue(mimeType)
        except SubversionError:
            _log.debug("No subversion property for mimetype is set! Trying to determine mimetype by persitenceId.")
            mimeType = mimetypes.guess_type(self.__persistenceId, False)
            if mimeType[0] is None:
                mappedResult[constants.MIME_TYPE] = value_mapping.MetadataValue("")
            else:
                mappedResult[constants.MIME_TYPE] = value_mapping.MetadataValue(mimeType[0])

        for key, value in rawResult.iteritems():
            mappedResult[key] = value_mapping.MetadataValue(value)

        return mappedResult

    @staticmethod
    def _filterResult(selectedPropertyIds, mappedResult):
        """ Filters the result so it contains only the specified properties. """

        if not selectedPropertyIds is None and len(selectedPropertyIds) >= 0:
            result = dict()
            for propertyId in selectedPropertyIds:
                if propertyId in mappedResult:
                    result[propertyId] = mappedResult[propertyId]
            return result
        else:
            return mappedResult

    def update(self, properties):
        """ @see: L{NullMetadataStorer<datafinder.persistence.metadata.metadatastorer.NullMetadataStorer>}"""

        connection = self.__connectionPool.acquire()
        try:
            try:
                jsonProperties = value_mapping.getPersistenceRepresentation(properties)
                connection.setProperty(self.__persistenceId, XPS_JSON_PROPERTY, jsonProperties)
            except SubversionError, error:
                errorMessage = "Cannot update properties of item '%s'. " % self.identifier \
                               + "Reason: '%s'" % error
                raise PersistenceError(errorMessage)
        finally:
            self.__connectionPool.release(connection)

    def delete(self, propertyIds):
        """ @see: L{NullMetadataStorer<datafinder.persistence.metadata.metadatastorer.NullMetadataStorer>}"""

        connection = self.__connectionPool.acquire()
        try:
            try:
                persistenceJsonProperties = self._retrieveProperties(connection)
                persistenceProperties = json.loads(persistenceJsonProperties)
                for propertyId in propertyIds:
                    if isinstance(persistenceProperties[propertyId], type(list())):
                        persistenceProperties[propertyId] = []
                    else:
                        persistenceProperties[propertyId] = u""
                persistenceJsonProperties = json.dumps(persistenceProperties)
                connection.setProperty(self.__persistenceId, XPS_JSON_PROPERTY, persistenceJsonProperties)
            except SubversionError, error:
                errorMessage = "Cannot delete properties of item '%s'. " % self.identifier \
                               + "Reason: '%s'" % error
                raise PersistenceError(errorMessage)
            except KeyError, error:
                errorMessage = "Cannot delete properties of item '%s'. " % self.identifier \
                               + "Reason: '%s'" % error
                raise PersistenceError(errorMessage)
        finally:
            self.__connectionPool.release(connection)

    def search(self, restrictions):
        """
        Unsupported delegating to default implementation.
        @see: L{NullMetadataStorer<datafinder.persistence.metadata.metadatastorer.NullMetadataStorer>}
        """

        return NullMetadataStorer.search(self, restrictions)
    <commit_msg>Fixed update method of the MetadataSubversionAdapter.<commit_after># $Filename$
# $Authors$
# Last Changed: $Date$ $Committer$ $Revision-Id$
#
# Copyright (c) 2003-2011, German Aerospace Center (DLR)
#
# All rights reserved.
#Redistribution and use in source and binary forms, with or without
#modification, are permitted provided that the following conditions are
#
#met:
#
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the
#   distribution.
#
# * Neither the name of the German Aerospace Center nor the names of
#   its contributors may be used to endorse or promote products derived
#   from this software without specific prior written permission.
#
#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""
This module implements how the meta data is persisted on the SVN server.
"""

import json
import logging
import mimetypes

from datafinder.persistence.adapters.svn.constants import XPS_JSON_PROPERTY, SVN_MIME_TYPE
from datafinder.persistence.adapters.svn.error import SubversionError
from datafinder.persistence.adapters.svn.util import util
from datafinder.persistence.error import PersistenceError
from datafinder.persistence.metadata import constants, value_mapping
from datafinder.persistence.metadata.metadatastorer import NullMetadataStorer

__version__ = "$Revision-Id$"

_log = logging.getLogger()

class MetadataSubversionAdapter(NullMetadataStorer):
    """ Implements meta data storer interface for subversion. """

    def __init__(self, identifier, connectionPool):
        """
        Constructor.

        @param identifier: Logical identifier of the resource.
        @type identifier: C{unicode}
        @param connectionPool: Connection pool.
        @type connectionPool: L{Connection<datafinder.persistence.svn.connection_pool.SVNConnectionPool>}
        @param modelIdentifier: Logical identifier of the model.
        @type modelIdentifier: C{unicode}
        """

        NullMetadataStorer.__init__(self, identifier)
        self.__connectionPool = connectionPool
        self.__persistenceId = util.mapIdentifier(identifier)

    def retrieve(self, propertyIds=None):
        """ @see: L{NullMetadataStorer<datafinder.persistence.metadata.metadatastorer.NullMetadataStorer>}"""

        connection = self.__connectionPool.acquire()
        try:
            rawResult = dict()
            try:
                persistenceJsonProperties = self._retrieveProperties(connection)
                rawResult = json.loads(persistenceJsonProperties)
            except SubversionError:
                _log.debug("No subversion property is set!")
            mappedResult = self._mapRawResult(connection, rawResult)
            return self._filterResult(propertyIds, mappedResult)
        finally:
            self.__connectionPool.release(connection)

    def _retrieveProperties(self, connection):
        """ Retrieves all properties. """

        return connection.getProperty(self.__persistenceId, XPS_JSON_PROPERTY)

    def _mapRawResult(self, connection, rawResult):
        """ Maps the SVN specific result to interface format. """

        try:
            infoDict = connection.info(self.__persistenceId)
        except SubversionError, error:
            errorMessage = "Problem during meta data retrieval. " \
                           + "Reason: '%s'" % error
            raise PersistenceError(errorMessage)
        mappedResult = dict()
        mappedResult[constants.CREATION_DATETIME] = value_mapping.MetadataValue("")
        mappedResult[constants.MODIFICATION_DATETIME] = value_mapping.MetadataValue(infoDict["lastChangedDate"])
        mappedResult[constants.SIZE] = value_mapping.MetadataValue("")
        mappedResult[constants.OWNER] = value_mapping.MetadataValue(infoDict["lastChangedAuthor"])

        try:
            mimeType = connection.getProperty(self.__persistenceId, SVN_MIME_TYPE)
            mappedResult[constants.MIME_TYPE] = value_mapping.MetadataValue(mimeType)
        except SubversionError:
            _log.debug("No subversion property for mimetype is set! Trying to determine mimetype by persitenceId.")
            mimeType = mimetypes.guess_type(self.__persistenceId, False)
            if mimeType[0] is None:
                mappedResult[constants.MIME_TYPE] = value_mapping.MetadataValue("")
            else:
                mappedResult[constants.MIME_TYPE] = value_mapping.MetadataValue(mimeType[0])

        for key, value in rawResult.iteritems():
            mappedResult[key] = value_mapping.MetadataValue(value)

        return mappedResult

    @staticmethod
    def _filterResult(selectedPropertyIds, mappedResult):
        """ Filters the result so it contains only the specified properties. """

        if not selectedPropertyIds is None and len(selectedPropertyIds) >= 0:
            result = dict()
            for propertyId in selectedPropertyIds:
                if propertyId in mappedResult:
                    result[propertyId] = mappedResult[propertyId]
            return result
        else:
            return mappedResult

    def update(self, properties):
        """ @see: L{NullMetadataStorer<datafinder.persistence.metadata.metadatastorer.NullMetadataStorer>}"""

        connection = self.__connectionPool.acquire()
        try:
            try:
                persistenceJsonProperties = self._retrieveProperties(connection)
                persistenceProperties = json.loads(persistenceJsonProperties)
                persistenceProperties.update(properties)
                jsonProperties = value_mapping.getPersistenceRepresentation(persistenceProperties)
                connection.setProperty(self.__persistenceId, XPS_JSON_PROPERTY, jsonProperties)
            except SubversionError, error:
                errorMessage = "Cannot update properties of item '%s'. " % self.identifier \
                               + "Reason: '%s'" % error
                raise PersistenceError(errorMessage)
        finally:
            self.__connectionPool.release(connection)

    def delete(self, propertyIds):
        """ @see: L{NullMetadataStorer<datafinder.persistence.metadata.metadatastorer.NullMetadataStorer>}"""

        connection = self.__connectionPool.acquire()
        try:
            try:
                persistenceJsonProperties = self._retrieveProperties(connection)
                persistenceProperties = json.loads(persistenceJsonProperties)
                for propertyId in propertyIds:
                    if isinstance(persistenceProperties[propertyId], type(list())):
                        persistenceProperties[propertyId] = []
                    else:
                        persistenceProperties[propertyId] = u""
                persistenceJsonProperties = json.dumps(persistenceProperties)
                connection.setProperty(self.__persistenceId, XPS_JSON_PROPERTY, persistenceJsonProperties)
            except SubversionError, error:
                errorMessage = "Cannot delete properties of item '%s'. " % self.identifier \
                               + "Reason: '%s'" % error
                raise PersistenceError(errorMessage)
            except KeyError, error:
                errorMessage = "Cannot delete properties of item '%s'. " % self.identifier \
                               + "Reason: '%s'" % error
                raise PersistenceError(errorMessage)
        finally:
            self.__connectionPool.release(connection)

    def search(self, restrictions):
        """
        Unsupported delegating to default implementation.
        @see: L{NullMetadataStorer<datafinder.persistence.metadata.metadatastorer.NullMetadataStorer>}
        """

        return NullMetadataStorer.search(self, restrictions)
    <|endoftext|>


<commit_before>"""Core models used by the CMS."""

import datetime

from django.conf import settings
from django.db import models
from django.db.models import Q

from cms.apps.pages import content
from cms.apps.utils.optimizations import cached_getter, cached_setter
from cms.apps.pages.models.base import PublishedModel, PageBase
from cms.apps.pages.models.managers import PublishedModelManager, PageBaseManager, PageManager, cache, publication_manager
from cms.apps.pages.models.fields import PageField, HtmlField, NullBooleanField, EnumField

class Page(PageBase):

    """A page within the site."""

    objects = PageManager()

    @classmethod
    def select_published(cls, queryset):
        """Selects only published pages."""
        queryset = super(Page, cls).select_published(queryset)
        now = datetime.datetime.now()
        queryset = queryset.filter(Q(publication_date=None) | Q(publication_date__lte=now))
        queryset = queryset.filter(Q(expiry_date=None) | Q(expiry_date__gt=now))
        return queryset

    # Base fields.

    url_title = models.SlugField("URL title",
                                 db_index=False)

    def __init__(self, *args, **kwargs):
        """"Initializes the Page."""
        super(Page, self).__init__(*args, **kwargs)
        if self.id:
            cache.add(self)

    # Hierarchy fields.

    parent = PageField(blank=True,
                       null=True)

    def get_all_parents(self):
        """Returns a list of all parents of this page."""
        if self.parent:
            return [self.parent] + self.parent.all_parents
        return []

    all_parents = property(get_all_parents,
                           doc="A list of all parents of this page.")

    order = models.PositiveIntegerField(unique=True,
                                        editable=False,
                                        blank=True,
                                        null=True)

    @cached_getter
    def get_children(self):
        """
        Returns all the children of this page, regardless of their publication
        state.
        """
        return Page.objects.filter(parent=self)

    children = property(get_children,
                        doc="All children of this page.")

    def get_all_children(self):
        """
        Returns all the children of this page, cascading down to their children
        too.
        """
        children = []
        for child in self.children:
            children.append(child)
            children.extend(child.all_children)
        return children

    all_children = property(get_all_children,
                            doc="All the children of this page, cascading down to their children too.")

    # Publication fields.

    publication_date = models.DateTimeField(blank=True,
                                            null=True,
                                            help_text="The date that this page will appear on the website.  Leave this blank to immediately publish this page.")

    expiry_date = models.DateTimeField(blank=True,
                                       null=True,
                                       help_text="The date that this page will be removed from the website.  Leave this blank to never expire this page.")

    # Navigation fields.

    in_navigation = models.BooleanField("add to navigation",
                                        default=True,
                                        help_text="Uncheck this box to remove this content from the site navigation.")

    permalink = models.SlugField(blank=True,
                                 help_text="A unique identifier for this page.  This will be set by your design team in order to link to this page from any custom templates they write.")

    # Content fields.

    content_type = models.CharField(max_length=100,
                                    editable=False,
                                    db_index=True,
                                    help_text="The type of page content.")

    content_data = models.TextField(editable=False,
                                    help_text="The encoded data of this page.")

    @cached_getter
    def get_content(self):
        """Returns the content object associated with this page."""
        if not self.content_type:
            return None
        content_cls = content.lookup(self.content_type)
        content_instance = content_cls(self)
        return content_instance

    @cached_setter(get_content)
    def set_content(self, content):
        """Sets the content object for this page."""
        self.content_data = content.serialized_data

    content = property(get_content,
                       set_content,
                       doc="The content object associated with this page.")

    # Standard model methods.

    def get_absolute_url(self):
        """Generates the absolute url of the page."""
        if self.parent:
            return self.parent.url + self.url_title + "/"
        return "/"

    def save(self, *args, **kwargs):
        """Saves the page."""
        super(Page, self).save(*args, **kwargs)
        cache.add(self)

    def delete(self, *args, **kwargs):
        """Deletes the page."""
        super(Page, self).delete(*args, **kwargs)
        cache.remove(self)

    class Meta:
        unique_together = (("parent", "url_title",),)
        ordering = ("order",)

<commit_msg>Using defered content data loading for pages.<commit_after>"""Core models used by the CMS."""

import datetime

from django.conf import settings
from django.db import models
from django.db.models import Q

from cms.apps.pages import content
from cms.apps.utils.optimizations import cached_getter, cached_setter
from cms.apps.pages.models.base import PublishedModel, PageBase
from cms.apps.pages.models.managers import PublishedModelManager, PageBaseManager, PageManager, cache, publication_manager
from cms.apps.pages.models.fields import PageField, HtmlField, NullBooleanField, EnumField

class Page(PageBase):

    """A page within the site."""

    objects = PageManager()

    @classmethod
    def select_published(cls, queryset):
        """Selects only published pages."""
        queryset = super(Page, cls).select_published(queryset)
        now = datetime.datetime.now()
        queryset = queryset.filter(Q(publication_date=None) | Q(publication_date__lte=now))
        queryset = queryset.filter(Q(expiry_date=None) | Q(expiry_date__gt=now))
        return queryset

    # Base fields.

    url_title = models.SlugField("URL title",
                                 db_index=False)

    def __init__(self, *args, **kwargs):
        """"Initializes the Page."""
        super(Page, self).__init__(*args, **kwargs)
        if self.id:
            cache.add(self)

    # Hierarchy fields.

    parent = PageField(blank=True,
                       null=True)

    def get_all_parents(self):
        """Returns a list of all parents of this page."""
        if self.parent:
            return [self.parent] + self.parent.all_parents
        return []

    all_parents = property(get_all_parents,
                           doc="A list of all parents of this page.")

    order = models.PositiveIntegerField(unique=True,
                                        editable=False,
                                        blank=True,
                                        null=True)

    @cached_getter
    def get_children(self):
        """
        Returns all the children of this page, regardless of their publication
        state.
        """
        return Page.objects.filter(parent=self).defer("content_data")

    children = property(get_children,
                        doc="All children of this page.")

    def get_all_children(self):
        """
        Returns all the children of this page, cascading down to their children
        too.
        """
        children = []
        for child in self.children:
            children.append(child)
            children.extend(child.all_children)
        return children

    all_children = property(get_all_children,
                            doc="All the children of this page, cascading down to their children too.")

    # Publication fields.

    publication_date = models.DateTimeField(blank=True,
                                            null=True,
                                            help_text="The date that this page will appear on the website.  Leave this blank to immediately publish this page.")

    expiry_date = models.DateTimeField(blank=True,
                                       null=True,
                                       help_text="The date that this page will be removed from the website.  Leave this blank to never expire this page.")

    # Navigation fields.

    in_navigation = models.BooleanField("add to navigation",
                                        default=True,
                                        help_text="Uncheck this box to remove this content from the site navigation.")

    permalink = models.SlugField(blank=True,
                                 help_text="A unique identifier for this page.  This will be set by your design team in order to link to this page from any custom templates they write.")

    # Content fields.

    content_type = models.CharField(max_length=100,
                                    editable=False,
                                    db_index=True,
                                    help_text="The type of page content.")

    content_data = models.TextField(editable=False,
                                    help_text="The encoded data of this page.")

    @cached_getter
    def get_content(self):
        """Returns the content object associated with this page."""
        if not self.content_type:
            return None
        content_cls = content.lookup(self.content_type)
        content_instance = content_cls(self)
        return content_instance

    @cached_setter(get_content)
    def set_content(self, content):
        """Sets the content object for this page."""
        self.content_data = content.serialized_data

    content = property(get_content,
                       set_content,
                       doc="The content object associated with this page.")

    # Standard model methods.

    def get_absolute_url(self):
        """Generates the absolute url of the page."""
        if self.parent:
            return self.parent.url + self.url_title + "/"
        return "/"

    def save(self, *args, **kwargs):
        """Saves the page."""
        super(Page, self).save(*args, **kwargs)
        cache.add(self)

    def delete(self, *args, **kwargs):
        """Deletes the page."""
        super(Page, self).delete(*args, **kwargs)
        cache.remove(self)

    class Meta:
        unique_together = (("parent", "url_title",),)
        ordering = ("order",)

<|endoftext|>


<commit_before>from django.db import models
from mangaki.models import Anime
import locale

class Location(models.Model):
    title = models.CharField(max_length=64, verbose_name='Titre')
    address = models.TextField(verbose_name='Adresse')
    postal_code = models.CharField(max_length=5, verbose_name='Code postal')
    city = models.CharField(max_length=64, verbose_name='Ville')

    def __str__(self):
        return '%s, %s' % (self.title, self.city)

class Event(models.Model):
    anime = models.ForeignKey(Anime)
    location = models.ForeignKey('Location', blank=True, null=True)
    event_type = models.CharField(max_length=9, choices=(
        ('premiere', 'avant-première'),
        ('screening', 'projection'),
        ('release', 'sortie'),
        ('tv', 'diffusion')
    ))
    language = models.CharField(max_length=6, choices=(
        ('vf', 'VF'),
        ('vostfr', 'VOSTFR'),
        ('vosta', 'VOSTA'),
    ))
    channel = models.CharField(max_length=16, blank=True, default='')
    date = models.DateTimeField()
    link = models.URLField(blank=True, default='')

    def __str__(self):
        return '%s %s' % (self.event_type, self.anime.title)

    def get_date(self):
        locale.setlocale(locale.LC_ALL, 'fr_FR.UTF-8')
        return self.date.strftime('%A %-d %B %Y à %H h %M').lower()

    def to_html(self):
        date = self.get_date()
        if self.event_type == 'tv':
            return '%s %s le %s sur %s' % (self.get_event_type_display(), self.anime.title, date, self.channel)
        else:
            location = self.location
            if self.link:
                location = '%s' % (self.link, self.location)
            return '%s %s le %s, %s' % (self.get_event_type_display(), self.anime.title, date, location)

    class Meta:
        ordering = ['date']

class Partner(models.Model):
    name = models.CharField(max_length=32)
    url = models.CharField(max_length=512)
    image = models.CharField(max_length=32, verbose_name="Fichier logo")<commit_msg>Adding partner ordering<commit_after>from django.db import models
from mangaki.models import Anime
import locale

class Location(models.Model):
    title = models.CharField(max_length=64, verbose_name='Titre')
    address = models.TextField(verbose_name='Adresse')
    postal_code = models.CharField(max_length=5, verbose_name='Code postal')
    city = models.CharField(max_length=64, verbose_name='Ville')

    def __str__(self):
        return '%s, %s' % (self.title, self.city)

class Event(models.Model):
    anime = models.ForeignKey(Anime)
    location = models.ForeignKey('Location', blank=True, null=True)
    event_type = models.CharField(max_length=9, choices=(
        ('premiere', 'avant-première'),
        ('screening', 'projection'),
        ('release', 'sortie'),
        ('tv', 'diffusion')
    ))
    language = models.CharField(max_length=6, choices=(
        ('vf', 'VF'),
        ('vostfr', 'VOSTFR'),
        ('vosta', 'VOSTA'),
    ))
    channel = models.CharField(max_length=16, blank=True, default='')
    date = models.DateTimeField()
    link = models.URLField(blank=True, default='')

    def __str__(self):
        return '%s %s' % (self.event_type, self.anime.title)

    def get_date(self):
        locale.setlocale(locale.LC_ALL, 'fr_FR.UTF-8')
        return self.date.strftime('%A %-d %B %Y à %H h %M').lower()

    def to_html(self):
        date = self.get_date()
        if self.event_type == 'tv':
            return '%s %s le %s sur %s' % (self.get_event_type_display(), self.anime.title, date, self.channel)
        else:
            location = self.location
            if self.link:
                location = '%s' % (self.link, self.location)
            return '%s %s le %s, %s' % (self.get_event_type_display(), self.anime.title, date, location)

    class Meta:
        ordering = ['date']

class Partner(models.Model):
    name = models.CharField(max_length=32)
    url = models.CharField(max_length=512)
    image = models.CharField(max_length=32, verbose_name="Fichier logo")

    class Meta:
        ordering = ['name']
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    Mango tasks system logic functions.
'''

# This file is part of mango.

# Distributed under the terms of the last AGPL License.
# The full license is in the file LICENCE, distributed as part of this software.

__author__ = 'Jean Chassoul'

import logging

import arrow
import motor

import uuid

# import numpy as np
import pandas as pd

from tornado import gen

from mango.messages import tasks
from mango.messages import reports

from mango.tools import clean_structure
#from mango.tools import clean_results
from mango.tools import check_times

class Tasks(object):
    '''
        Tasks resources
    '''

    @gen.coroutine
    def get_task(self, account, task_uuid):
        '''
            Get a detail task
        '''
        if not account:
            task = yield self.db.tasks.find_one({'uuid':task_uuid},{'_id':0})
        else:

            # change accountcode to account, because the accountcode is a uuid
            # and we're expecting an account name.

            task = yield self.db.tasks.find_one({'uuid':task_uuid,
                                                     'account':account},
                                                    {'_id':0})
        try:
            if task:
                task = tasks.Task(task)
                task.validate()
        except Exception, e:
            logging.exception(e) # catch some daemon here!
            raise e
        finally:
            raise gen.Return(task)

    @gen.coroutine
    def get_task_list(self, account, start, end, lapse, status, page_num):
        '''
            Get detail tasks
        '''

        # Note how the status come after the lapse meaning the complete time values.

        # time arguments: [start, end, lapse]

        # status argument, first rage against the finite-state machine.

        # Notes on pagination: every get list function returns the resource count, current page and results.

        page_num = int(page_num)
        von_count = 0
        page_size = self.settings['page_size']
        task_list = []
        message = None
        query = {'public':False}

        if status != 'all':
            query['status'] = status

        if not account:
            query = self.db.tasks.find(query,
                                       {'_id':0, 'comments':0})
        elif type(account) is list:
            accounts = [{'accountcode':a, 'assigned': True} for a in account]
            query = self.db.tasks.find({'$or':accounts},
                                       {'_id':0, 'comments':0})
        else:
            query = self.db.tasks.find({'accountcode':account,
                                        'assigned':True},
                                       {'_id':0, 'comments':0})

        try:

            von_count = yield query.count()

        except Exception, e:
            logging.exception(e)
            raise e

        query = query.sort([('uuid', -1)]).skip(page_num * page_size).limit(page_size)

        try:

            while (yield query.fetch_next):
                result = query.next_object()
                task_list.append(tasks.Task(result))

        except Exception, e:
            logging.exception(e)
            raise e

        try:
            struct = {'results': task_list, 'page': page_num, 'count': von_count}

            message = reports.BaseResult(struct)

            # this is coming from our datastorage, we can gain those milliseconds and avoid validation.
            # from shit that is in good theory already validated.
            # message.validate()

            message = message.to_primitive()

        except Exception, e:
            logging.exception(e)
            raise e
        finally:
            raise gen.Return(message)

    @gen.coroutine
    def new_task(self, struct):
        '''
            Create a new task entry
        '''
        try:
            task = tasks.Task(struct)
            task.validate()
        except Exception, e:
            logging.exception(e)
            raise e

        task = clean_structure(task)

        result = yield self.db.tasks.insert(task)

        raise gen.Return(task.get('uuid'))

    @gen.coroutine
    def set_assigned_flag(self, account, task_uuid):
        '''
            Set the task assigned flag
        '''
        logging.info('set_assigned_flag account: %s, task: %s' % (account, task_uuid))

        result = yield self.db.tasks.update(
                                {'uuid':task_uuid,
                                 'accountcode':account},
                                {'$set': {'assigned': True}})

        raise gen.Return(result)

    @gen.coroutine
    def remove_task(self, task_uuid):
        '''
            Remove a task entry
        '''
        result = yield self.db.tasks.remove({'uuid':task_uuid})
        raise gen.Return(result)

    @gen.coroutine
    def modify_task(self, account, task_uuid, struct):
        '''
            Modify task
        '''
        try:
            logging.info(struct)
            task = tasks.ModifyTask(struct)
            task.validate()
            task = clean_structure(task)
        except Exception, e:
            logging.error(e)
            raise e

        try:
            result = yield self.db.tasks.update(
                {'account':account,
                 'uuid':task_uuid},
                {'$set':task}
            )
            logging.info(result)
        except Exception, e:
            logging.error(e)
            message = str(e)

        raise gen.Return(bool(result.get('n')))

    @gen.coroutine
    def replace_task(self, struct):
        '''
            Replace a existent task entry
        '''
        # put implementation
        pass

    @gen.coroutine
    def resource_options(self):
        '''
            Return resource options
        '''
        # options implementation
        pass<commit_msg>ok go<commit_after># -*- coding: utf-8 -*-
'''
    Mango tasks system logic functions.
'''

# This file is part of mango.

# Distributed under the terms of the last AGPL License.
# The full license is in the file LICENCE, distributed as part of this software.

__author__ = 'Jean Chassoul'

import logging

import arrow
import motor

import uuid

# import numpy as np
import pandas as pd

from tornado import gen

from mango.messages import tasks
from mango.messages import reports

from mango.tools import clean_structure
#from mango.tools import clean_results
from mango.tools import check_times

class Tasks(object):
    '''
        Tasks resources
    '''

    @gen.coroutine
    def get_task(self, account, task_uuid):
        '''
            Get a detail task
        '''
        if not account:
            task = yield self.db.tasks.find_one({'uuid':task_uuid},{'_id':0})
        else:

            # change accountcode to account, because the accountcode is a uuid
            # and we're expecting an account name.

            task = yield self.db.tasks.find_one({'uuid':task_uuid,
                                                     'account':account},
                                                    {'_id':0})
        try:
            if task:
                task = tasks.Task(task)
                task.validate()
        except Exception, e:
            logging.exception(e) # catch some daemon here!
            raise e
        finally:
            raise gen.Return(task)

    @gen.coroutine
    def get_task_list(self, account, start, end, lapse, status, page_num):
        '''
            Get detail tasks
        '''

        # Note how the status come after the lapse meaning the complete time values.

        # time arguments: [start, end, lapse]

        # status argument, first rage against the finite-state machine.

        # Notes on pagination: every get list function returns the resource count, current page and results.

        page_num = int(page_num)
        von_count = 0
        page_size = self.settings['page_size']
        task_list = []
        message = None
        query = {'public':False}

        if status != 'all':
            query['status'] = status

        if not account:
            query = self.db.tasks.find(query,
                                       {'_id':0, 'comments':0})
        elif type(account) is list:
            accounts = [{'accountcode':a, 'assigned': True} for a in account]
            query = self.db.tasks.find({'$or':accounts},
                                       {'_id':0, 'comments':0})
        else:
            query = self.db.tasks.find({'accountcode':account,
                                        'assigned':True},
                                       {'_id':0, 'comments':0})

        try:

            von_count = yield query.count()

        except Exception, e:
            logging.exception(e)
            raise e

        query = query.sort([('uuid', -1)]).skip(page_num * page_size).limit(page_size)

        try:

            while (yield query.fetch_next):
                result = query.next_object()
                task_list.append(tasks.Task(result))

        except Exception, e:
            logging.exception(e)
            raise e

        try:
            struct = {'results': task_list, 'page': page_num, 'count': von_count}

            logging.warning(struct)

            message = reports.BaseResult(struct)

            # this is coming from our datastorage, we can gain those milliseconds and avoid validation.
            # from shit that is in good theory already validated.
            # message.validate()

            message = message.to_primitive()

        except Exception, e:
            logging.exception(e)
            raise e
        finally:
            raise gen.Return(message)

    @gen.coroutine
    def new_task(self, struct):
        '''
            Create a new task entry
        '''
        try:
            task = tasks.Task(struct)
            task.validate()
        except Exception, e:
            logging.exception(e)
            raise e

        task = clean_structure(task)

        result = yield self.db.tasks.insert(task)

        raise gen.Return(task.get('uuid'))

    @gen.coroutine
    def set_assigned_flag(self, account, task_uuid):
        '''
            Set the task assigned flag
        '''
        logging.info('set_assigned_flag account: %s, task: %s' % (account, task_uuid))

        result = yield self.db.tasks.update(
                                {'uuid':task_uuid,
                                 'accountcode':account},
                                {'$set': {'assigned': True}})

        raise gen.Return(result)

    @gen.coroutine
    def remove_task(self, task_uuid):
        '''
            Remove a task entry
        '''
        result = yield self.db.tasks.remove({'uuid':task_uuid})
        raise gen.Return(result)

    @gen.coroutine
    def modify_task(self, account, task_uuid, struct):
        '''
            Modify task
        '''
        try:
            logging.info(struct)
            task = tasks.ModifyTask(struct)
            task.validate()
            task = clean_structure(task)
        except Exception, e:
            logging.error(e)
            raise e

        try:
            result = yield self.db.tasks.update(
                {'account':account,
                 'uuid':task_uuid},
                {'$set':task}
            )
            logging.info(result)
        except Exception, e:
            logging.error(e)
            message = str(e)

        raise gen.Return(bool(result.get('n')))

    @gen.coroutine
    def replace_task(self, struct):
        '''
            Replace a existent task entry
        '''
        # put implementation
        pass

    @gen.coroutine
    def resource_options(self):
        '''
            Return resource options
        '''
        # options implementation
        pass<|endoftext|>