<commit_before>import clustering
import matplotlib.pyplot as plt
from shapely.geometry import Polygon,MultiPolygon
import itertools
import math
from shapely.validation import explain_validity
import matplotlib
from shapely.ops import cascaded_union
import numpy

def findsubsets(S,m):
    return set(itertools.combinations(S, m))

def nCr(n,r):
    f = math.factorial
    return f(n) / f(r) / f(n-r)

class QuadTree:
    def __init__(self,((lb_x,lb_y),(ub_x,ub_y)),parent=None):
        self.lb_x = lb_x
        self.ub_x = ub_x
        self.lb_y = lb_y
        self.ub_y = ub_y

        self.children = None
        self.parent = parent
        self.polygons = {}

        self.bounding_box = Polygon([(lb_x,lb_y),(ub_x,lb_y),(ub_x,ub_y),(lb_x,ub_y)])

        self.user_ids = []

    def __get_splits__(self):
        if (self.bounding_box.area < 25) or (len(self.polygons) <= 2):
            return []

        complete_agreement = 0

        for user in self.polygons:
            # extract just the polygons - don't worry about type
            u = cascaded_union(zip(*self.polygons[user])[0])
            if math.fabs(self.bounding_box.intersection(u).area - self.bounding_box.area) < 1:
                complete_agreement += 1

        if complete_agreement >= 3:
            return []

        # calculate the height and width of the new children nodes
        new_width = (self.lb_x+self.ub_x)/2. - self.lb_x
        new_height = (self.ub_y+self.lb_y)/2. - self.lb_y

        lower_left = (self.lb_x,self.lb_y),(self.lb_x+new_width,self.lb_y+new_height)
        lower_right = (self.lb_x+new_width,self.lb_y),(self.ub_x,self.lb_y+new_height)
        upper_left = (self.lb_x,self.lb_y+new_height),(self.lb_x+new_width,self.ub_y)
        upper_right = (self.lb_x+new_width,self.lb_y+new_height),(self.ub_x,self.ub_y)

        self.children = [QuadTree(lower_left,self) ,QuadTree(lower_right,self),QuadTree(upper_left,self),QuadTree(upper_right,self)]
        for c in self.children:
            assert isinstance(c,QuadTree)
            assert c.bounding_box.area == self.bounding_box.area/4.

        return self.children

    def __plot__(self,ax):
        if self.children is None:
            if len(self.polygons) >= 3:
                plt.plot((self.lb_x,self.ub_x,self.ub_x,self.lb_x,self.lb_x),(self.lb_y,self.lb_y,self.ub_y,self.ub_y,self.lb_y),color="red")
                # rect = plt.Rectangle((self.lb_x,self.lb_y),(self.ub_x-self.lb_x),(self.ub_y-self.lb_y),color="red")
                # print (self.lb_x,self.lb_y),(self.ub_x-self.lb_x),(self.ub_y-self.lb_y)
                # ax.add_artist(rect)
        else:
            for c in self.children:
                assert isinstance(c,QuadTree)
                c.__plot__(ax)

    def __aggregate__(self):
        if self.children is None:
            all_colours = {1: "blue",2: "green",3:"yellow",4:"black"}

            if len(self.polygons) >= 3:
                # what is the majority vote for what type of "kind" this box outlines
                # for example, some people might say broad leave tree while others say it is a needle leaf tree
                # technically speaking, people could outline this region with different polygons
                # if so, we'll ignore such users, under the assumption that we don't really know
                vote_counts = {}
                for u in self.polygons:
                    polys,types = zip(*self.polygons[u])
                    if min(types) == max(types):
                        vote = min(types)
                        if vote not in vote_counts:
                            vote_counts[vote] = 1
                        else:
                            vote_counts[vote] += 1

                # extract the most likely type according to pluraity voting
                # ties will get resolved in an arbitrary fashion
                most_likely,num_votes = sorted(vote_counts.items(),key = lambda x:x[1],reverse=True)[0]
                percentage = num_votes/float(sum(vote_counts.values()))

                # return two sets of values - one with the bounding box
                # the other with the voting results

                return {most_likely:self.bounding_box},{most_likely:(percentage,self.bounding_box.area)},0
            else:
                if len(self.polygons) == 0:
                    return {},{},0
                else:
                    return {},{},self.bounding_box.area
        else:
            return_polygons = {}
            return_stats = {}
            new_area = {}
            new_percentage = {}

            total_incorrect_area = 0

            for c in self.children:
                # get all the polygons that are in c's bounding boxes plus stats about which tools made
                # which reasons
                c_polygons,c_stats,incorrect_area = c.__aggregate__()
                total_incorrect_area += incorrect_area

                # go through the
                for tool_id in c_polygons:
                    # first time we've seen this polygon
                    if tool_id not in return_polygons:
                        return_polygons[tool_id] = c_polygons[tool_id]
                        # return_stats[tool_id] = c_stats[tool_id]

                        new_area[tool_id] = [c_stats[tool_id][1],]
                        new_percentage[tool_id] = [c_stats[tool_id][0],]
                    else:
                        # we need to merge (yay!!)
                        # note this will probably result in more than one polygon
                        return_polygons[tool_id] = return_polygons[tool_id].union(c_polygons[tool_id])
                        # I think I could fold these values in as I go, but just to be careful (don't want to mess
                        # with the stats) save the values until end
                        new_area[tool_id].append(c_stats[tool_id][1])
                        new_percentage[tool_id].append(c_stats[tool_id][0])

            # now that we've gone through all of the children
            # calculate the weighted values
            for tool_id in new_area:
                # print new_percentage[tool_id]
                return_stats[tool_id] = numpy.average(new_percentage[tool_id],weights=new_area[tool_id]),sum(new_area[tool_id])

                assert not isinstance(return_polygons[tool_id],list)
            return return_polygons,return_stats,total_incorrect_area

    def __add_polygon__(self,user,polygon,poly_type):
        assert isinstance(polygon,Polygon)
        # don't add it if there is no intersection
        if self.bounding_box.intersection(polygon).is_empty:
            return

        if user not in self.polygons:
            self.polygons[user] = [(polygon,poly_type)]
            self.user_ids.append(user)
        else:
            self.polygons[user].append((polygon,poly_type))

    def __poly_iteration__(self):
        class Iterator:
            def __init__(self,node):
                assert isinstance(node,QuadTree)
                self.node = node
                self.user_index = None
                self.polygon_index = None

            def __iter__(self):
                return self

            def next(self):
                if self.user_index is None:
                    self.user_index = 0
                    self.polygon_index = 0
                else:
                    self.polygon_index += 1
                    if self.polygon_index == len(self.node.polygons[self.node.user_ids[self.user_index]]):
                        self.polygon_index = 0
                        self.user_index += 1

                    if self.user_index == len(self.node.user_ids):
                        raise StopIteration

                user_id = self.node.user_ids[self.user_index]
                return user_id,self.node.polygons[user_id][self.polygon_index]

        return Iterator(self)

class BlobClustering(clustering.Cluster):
    def __init__(self,shape,dim_reduction_alg):
        assert shape != "point"
        clustering.Cluster.__init__(self,shape,dim_reduction_alg)
        self.rectangle = (shape == "rectangle")

    def __fix_polygon__(self,points):
        fixed_polygons = None

        points = list(points)

        validity = explain_validity(Polygon(points))

        # x,y = zip(*Polygon(points).exterior.coords)
        # x = list(x)
        # y = list(y)
        # x.append(x[0])
        # y.append(y[0])
        # plt.plot(x,y)
        # plt.show()

        assert isinstance(validity,str)
        s,t = validity.split("[")
        x_0,y_0 = t.split(" ")
        x_0 = float(x_0)
        y_0 = float(y_0[:-1])

        # search for all of the line segments which touch the intersection point
        # we need to wrap around to the beginning to get all of the line segments
        splits = []
        for line_index in range(len(points)):
            (x_1,y_1) = points[line_index]
            (x_2,y_2) = points[(line_index+1)%len(points)]

            # the equation from a point to the nearest place on a line is from
            # https://en.wikipedia.org/wiki/Distance_from_a_point_to_a_line
            dist = math.fabs((y_2-y_1)*x_0-(x_2-x_1)*y_0+x_2*y_1-y_2*x_1)/math.sqrt((y_2-y_1)**2+(x_2-x_1)**2)
            if dist < 0.01:
                splits.append(line_index)

        # seems to be the easiest way to dealing with needing to extract
        # sublists which wrap around the end/beginning of the list

        points.extend(points)
        for intersection_index,line_index in enumerate(splits):
            # find the index for the next line segment with intersect (x_0,y_0)
            # if we have reached the end of the list then we need to wrap around
            if (intersection_index+1) < len(splits):
                line_index2 = splits[intersection_index+1]
            else:
                # keep in mind that we've doubled the length of points
                line_index2 = splits[0] + len(points)/2

            # always create the new polygon starting at the intersection point
            new_polygon_points = [(x_0,y_0)]
            new_polygon_points.extend(points[line_index+1:line_index2+1])

            if explain_validity(Polygon(new_polygon_points)) != "Valid Geometry":
                # if this is the first "sub"polygon - just accept it
                if fixed_polygons is None:
                    fixed_polygons = self.__fix_polygon__(new_polygon_points)
                # else try to merge the results in
                else:
                    fixed_polygons.extend(self.__fix_polygon__(new_polygon_points))
            else:
                if fixed_polygons is None:
                    fixed_polygons = [Polygon(new_polygon_points)]
                else:
                    fixed_polygons.append(Polygon(new_polygon_points))

        return fixed_polygons

    def __inner_fit__(self,markings,user_ids,tools,reduced_markings,dimensions):
        poly_dictionary = {}
        for polygon_pts,u,t in zip(markings,user_ids,tools):
            # we need at least 3 points to made a valid polygon
            if len(polygon_pts) < 3:
                continue

            poly = Polygon(polygon_pts)
            validity = explain_validity(poly)

            if validity != "Valid Geometry":
                corrected_polygon = self.__fix_polygon__(polygon_pts)
                if isinstance(corrected_polygon,Polygon):
                    if u not in poly_dictionary:
                        poly_dictionary[u] = [(corrected_polygon,t),]
                    else:
                        for p in corrected_polygon:
                            poly_dictionary[u].append((p,t))
            else:
                if u not in poly_dictionary:
                    poly_dictionary[u] = [(poly,t)]
                else:
                    poly_dictionary[u].append((poly,t))

        box = [[0,0],[800,0],[800,500],[0,500]]

        quad_root = QuadTree((box[0],box[2]))

        for user,polygon_list in poly_dictionary.items():
            for polygon,poly_type in polygon_list:
                quad_root.__add_polygon__(user,polygon,poly_type)

        to_process = [quad_root]

        while to_process != []:
            node = to_process.pop(-1)
            assert isinstance(node,QuadTree)

            # if we have parent != the root => need to read in
            if node.parent is not None:
                for user,(poly,poly_type) in node.parent.__poly_iteration__():
                    node.__add_polygon__(user,poly,poly_type)

            new_children = node.__get_splits__()

            to_process.extend(new_children)

        # results.append({"users":merged_users,"cluster members":merged_points,"tools":merged_tools,"num users":num_users})
        # total incorrect area is the total area which at least one person marked/outlined by not enough people
        # so typically just 1 or 2 (unless we change the threshold)
        aggregate_polygons,aggregate_stats,total_incorrect_area = quad_root.__aggregate__()

        incorrect_area_as_percent = None

        # if we have  been provided the dimensions we can express blob area as a % of total area
        if dimensions is not None:
            image_area = dimensions[0]*dimensions[1]
            for tool_id in aggregate_stats:
                vote_percentage,tool_area = aggregate_stats[tool_id]
                aggregate_stats[tool_id] = vote_percentage,tool_area/float(image_area)

            incorrect_area_as_percent = total_incorrect_area/image_area

        results = []
        # a lot of this stuff is done in the classification code for other tool types but it makes more
        # sense for polygons to do it here
        for tool_id in aggregate_polygons:
            # have one set of results per tool type
            # the center will be a list of all polygons
            next_result = dict()

            # we will either have a multi-polygon as our aggregation result - or if we are really
            # lucky, a single polygon
            if isinstance(aggregate_polygons[tool_id],Polygon):
                poly = aggregate_polygons[tool_id]
                next_result["center"] = [zip(poly.exterior.xy[0],poly.exterior.xy[0])]
            elif isinstance(aggregate_polygons[tool_id], MultiPolygon):
                next_result["center"] = []
                for poly in aggregate_polygons[tool_id]:

                    # go through each of the individual polygons making up this multipolygon
                    if isinstance(poly,Polygon):
                        next_result["center"].append(zip(poly.exterior.xy[0],poly.exterior.xy[0]))
            else:
                # unknown type
                print type(aggregate_polygons[tool_id])
                assert False

            # a value of None -> not really relevant to polygon aggregation
            # or really hard to keep track of
            next_result["users"] = None
            next_result["num users"] = None
            next_result["cluster_members"] = None
            next_result["tool classification"] = tool_id

            # these are global values which are not really specific to any one polygon
            # but this seems to be the best place to store the values
            next_result["incorrect area"] = incorrect_area_as_percent
            # todo - don't hard code this
            next_result["minimum users"] = 3
            next_result["area"] = aggregate_stats[tool_id][1]
            next_result["certainty"] = aggregate_stats[tool_id][0]

            results.append(next_result)

        return results,0
<commit_msg>corrected how "incorrect" area is calculated<commit_after>import clustering
import matplotlib.pyplot as plt
from shapely.geometry import Polygon,MultiPolygon
import itertools
import math
from shapely.validation import explain_validity
import matplotlib
from shapely.ops import cascaded_union
import numpy

def findsubsets(S,m):
    return set(itertools.combinations(S, m))

def nCr(n,r):
    f = math.factorial
    return f(n) / f(r) / f(n-r)

class QuadTree:
    def __init__(self,((lb_x,lb_y),(ub_x,ub_y)),parent=None):
        self.lb_x = lb_x
        self.ub_x = ub_x
        self.lb_y = lb_y
        self.ub_y = ub_y

        self.children = None
        self.parent = parent
        self.polygons = {}

        self.bounding_box = Polygon([(lb_x,lb_y),(ub_x,lb_y),(ub_x,ub_y),(lb_x,ub_y)])

        self.user_ids = []

    def __get_splits__(self):
        if (self.bounding_box.area < 25) or (len(self.polygons) <= 2):
            return []

        complete_agreement = 0

        for user in self.polygons:
            # extract just the polygons - don't worry about type
            u = cascaded_union(zip(*self.polygons[user])[0])
            if math.fabs(self.bounding_box.intersection(u).area - self.bounding_box.area) < 1:
                complete_agreement += 1

        if complete_agreement >= 3:
            return []

        # calculate the height and width of the new children nodes
        new_width = (self.lb_x+self.ub_x)/2. - self.lb_x
        new_height = (self.ub_y+self.lb_y)/2. - self.lb_y

        lower_left = (self.lb_x,self.lb_y),(self.lb_x+new_width,self.lb_y+new_height)
        lower_right = (self.lb_x+new_width,self.lb_y),(self.ub_x,self.lb_y+new_height)
        upper_left = (self.lb_x,self.lb_y+new_height),(self.lb_x+new_width,self.ub_y)
        upper_right = (self.lb_x+new_width,self.lb_y+new_height),(self.ub_x,self.ub_y)

        self.children = [QuadTree(lower_left,self) ,QuadTree(lower_right,self),QuadTree(upper_left,self),QuadTree(upper_right,self)]
        for c in self.children:
            assert isinstance(c,QuadTree)
            assert c.bounding_box.area == self.bounding_box.area/4.

        return self.children

    def __plot__(self,ax):
        if self.children is None:
            if len(self.polygons) >= 3:
                plt.plot((self.lb_x,self.ub_x,self.ub_x,self.lb_x,self.lb_x),(self.lb_y,self.lb_y,self.ub_y,self.ub_y,self.lb_y),color="red")
                # rect = plt.Rectangle((self.lb_x,self.lb_y),(self.ub_x-self.lb_x),(self.ub_y-self.lb_y),color="red")
                # print (self.lb_x,self.lb_y),(self.ub_x-self.lb_x),(self.ub_y-self.lb_y)
                # ax.add_artist(rect)
        else:
            for c in self.children:
                assert isinstance(c,QuadTree)
                c.__plot__(ax)

    def __aggregate__(self,total_area):
        if self.children is None:
            if len(self.polygons) >= 3:
                # what is the majority vote for what type of "kind" this box outlines
                # for example, some people might say broad leave tree while others say it is a needle leaf tree
                # technically speaking, people could outline this region with different polygons
                # if so, we'll ignore such users, under the assumption that we don't really know
                vote_counts = {}
                for u in self.polygons:
                    polys,types = zip(*self.polygons[u])
                    if min(types) == max(types):
                        vote = min(types)
                        if vote not in vote_counts:
                            vote_counts[vote] = 1
                        else:
                            vote_counts[vote] += 1

                # extract the most likely type according to pluraity voting
                # ties will get resolved in an arbitrary fashion
                most_likely,num_votes = sorted(vote_counts.items(),key = lambda x:x[1],reverse=True)[0]
                percentage = num_votes/float(sum(vote_counts.values()))

                # return two sets of values - one with the bounding box
                # the other with the voting results

                return {most_likely:self.bounding_box},{most_likely:(percentage,self.bounding_box.area)},0
            else:
                if len(self.polygons) == 0:
                    return {},{},0
                else:
                    # we have come polygons inside this box which amount to noise
                    # find out the area of these polygons - with respect to inside this box
                    # for smaller boxes, we could just use the area of the box as an approximation
                    # but for larger boxes, especially if we are the root, we need to do better
                    if (self.bounding_box.area/float(total_area)) > 0.02:
                        noise_polygons = []
                        for poly_list in self.polygons.values():
                            noise_polygons.extend(zip(*poly_list)[0])
                        combined_polygon = cascaded_union(noise_polygons).intersection(self.bounding_box)

                        return {},{},combined_polygon.area
                    else:
                        return {},{},self.bounding_box.area
        else:
            return_polygons = {}
            return_stats = {}
            new_area = {}
            new_percentage = {}

            total_incorrect_area = 0

            for c in self.children:
                # get all the polygons that are in c's bounding boxes plus stats about which tools made
                # which reasons
                c_polygons,c_stats,incorrect_area = c.__aggregate__(total_area)
                total_incorrect_area += incorrect_area

                # go through the
                for tool_id in c_polygons:
                    # first time we've seen this polygon
                    if tool_id not in return_polygons:
                        return_polygons[tool_id] = c_polygons[tool_id]
                        # return_stats[tool_id] = c_stats[tool_id]

                        new_area[tool_id] = [c_stats[tool_id][1],]
                        new_percentage[tool_id] = [c_stats[tool_id][0],]
                    else:
                        # we need to merge (yay!!)
                        # note this will probably result in more than one polygon
                        return_polygons[tool_id] = return_polygons[tool_id].union(c_polygons[tool_id])
                        # I think I could fold these values in as I go, but just to be careful (don't want to mess
                        # with the stats) save the values until end
                        new_area[tool_id].append(c_stats[tool_id][1])
                        new_percentage[tool_id].append(c_stats[tool_id][0])

            # now that we've gone through all of the children
            # calculate the weighted values
            for tool_id in new_area:
                # print new_percentage[tool_id]
                return_stats[tool_id] = numpy.average(new_percentage[tool_id],weights=new_area[tool_id]),sum(new_area[tool_id])

                assert not isinstance(return_polygons[tool_id],list)
            return return_polygons,return_stats,total_incorrect_area

    def __add_polygon__(self,user,polygon,poly_type):
        assert isinstance(polygon,Polygon)
        # don't add it if there is no intersection
        if self.bounding_box.intersection(polygon).is_empty:
            return

        if user not in self.polygons:
            self.polygons[user] = [(polygon,poly_type)]
            self.user_ids.append(user)
        else:
            self.polygons[user].append((polygon,poly_type))

    def __poly_iteration__(self):
        class Iterator:
            def __init__(self,node):
                assert isinstance(node,QuadTree)
                self.node = node
                self.user_index = None
                self.polygon_index = None

            def __iter__(self):
                return self

            def next(self):
                if self.user_index is None:
                    self.user_index = 0
                    self.polygon_index = 0
                else:
                    self.polygon_index += 1
                    if self.polygon_index == len(self.node.polygons[self.node.user_ids[self.user_index]]):
                        self.polygon_index = 0
                        self.user_index += 1

                    if self.user_index == len(self.node.user_ids):
                        raise StopIteration

                user_id = self.node.user_ids[self.user_index]
                return user_id,self.node.polygons[user_id][self.polygon_index]

        return Iterator(self)

class BlobClustering(clustering.Cluster):
    def __init__(self,shape,dim_reduction_alg):
        assert shape != "point"
        clustering.Cluster.__init__(self,shape,dim_reduction_alg)
        self.rectangle = (shape == "rectangle")

    def __fix_polygon__(self,points):
        fixed_polygons = None

        points = list(points)

        validity = explain_validity(Polygon(points))

        # x,y = zip(*Polygon(points).exterior.coords)
        # x = list(x)
        # y = list(y)
        # x.append(x[0])
        # y.append(y[0])
        # plt.plot(x,y)
        # plt.show()

        assert isinstance(validity,str)
        s,t = validity.split("[")
        x_0,y_0 = t.split(" ")
        x_0 = float(x_0)
        y_0 = float(y_0[:-1])

        # search for all of the line segments which touch the intersection point
        # we need to wrap around to the beginning to get all of the line segments
        splits = []
        for line_index in range(len(points)):
            (x_1,y_1) = points[line_index]
            (x_2,y_2) = points[(line_index+1)%len(points)]

            # the equation from a point to the nearest place on a line is from
            # https://en.wikipedia.org/wiki/Distance_from_a_point_to_a_line
            dist = math.fabs((y_2-y_1)*x_0-(x_2-x_1)*y_0+x_2*y_1-y_2*x_1)/math.sqrt((y_2-y_1)**2+(x_2-x_1)**2)
            if dist < 0.01:
                splits.append(line_index)

        # seems to be the easiest way to dealing with needing to extract
        # sublists which wrap around the end/beginning of the list

        points.extend(points)
        for intersection_index,line_index in enumerate(splits):
            # find the index for the next line segment with intersect (x_0,y_0)
            # if we have reached the end of the list then we need to wrap around
            if (intersection_index+1) < len(splits):
                line_index2 = splits[intersection_index+1]
            else:
                # keep in mind that we've doubled the length of points
                line_index2 = splits[0] + len(points)/2

            # always create the new polygon starting at the intersection point
            new_polygon_points = [(x_0,y_0)]
            new_polygon_points.extend(points[line_index+1:line_index2+1])

            if explain_validity(Polygon(new_polygon_points)) != "Valid Geometry":
                # if this is the first "sub"polygon - just accept it
                if fixed_polygons is None:
                    fixed_polygons = self.__fix_polygon__(new_polygon_points)
                # else try to merge the results in
                else:
                    fixed_polygons.extend(self.__fix_polygon__(new_polygon_points))
            else:
                if fixed_polygons is None:
                    fixed_polygons = [Polygon(new_polygon_points)]
                else:
                    fixed_polygons.append(Polygon(new_polygon_points))

        return fixed_polygons

    def __inner_fit__(self,markings,user_ids,tools,reduced_markings,dimensions):
        poly_dictionary = {}
        for polygon_pts,u,t in zip(markings,user_ids,tools):
            # we need at least 3 points to made a valid polygon
            if len(polygon_pts) < 3:
                continue

            poly = Polygon(polygon_pts)
            validity = explain_validity(poly)

            if validity != "Valid Geometry":
                corrected_polygon = self.__fix_polygon__(polygon_pts)
                if isinstance(corrected_polygon,Polygon):
                    if u not in poly_dictionary:
                        poly_dictionary[u] = [(corrected_polygon,t),]
                    else:
                        for p in corrected_polygon:
                            poly_dictionary[u].append((p,t))
            else:
                if u not in poly_dictionary:
                    poly_dictionary[u] = [(poly,t)]
                else:
                    poly_dictionary[u].append((poly,t))

        # if dimensions have been provided, use those as our initial bounding box
        # otherwise, use some default values - I have NO idea how well those will work in practice
        if dimensions is None:
            box = [[0,0],[800,0],[800,500],[0,500]]
        else:
            box = [[0,0],[dimensions[1],0],[dimensions[1],dimensions[0]],[0,dimensions[1]]]

        quad_root = QuadTree((box[0],box[2]))

        for user,polygon_list in poly_dictionary.items():
            for polygon,poly_type in polygon_list:
                quad_root.__add_polygon__(user,polygon,poly_type)

        to_process = [quad_root]

        while to_process != []:
            node = to_process.pop(-1)
            assert isinstance(node,QuadTree)

            # if (we have parent =>  !the root) => need to read in parent's polygons
            # some of which will become ours
            if node.parent is not None:
                for user,(poly,poly_type) in node.parent.__poly_iteration__():
                    node.__add_polygon__(user,poly,poly_type)

            new_children = node.__get_splits__()

            to_process.extend(new_children)

        # total incorrect area is the total area which at least one person marked/outlined by not enough people
        # so typically just 1 or 2 (unless we change the threshold)
        image_area = dimensions[0]*dimensions[1]

        aggregate_polygons,aggregate_stats,total_incorrect_area = quad_root.__aggregate__(image_area)

        for tool_id in aggregate_stats:
            vote_percentage,tool_area = aggregate_stats[tool_id]
            aggregate_stats[tool_id] = vote_percentage,tool_area/float(image_area)

        incorrect_area_as_percent = total_incorrect_area/image_area

        results = []
        # a lot of this stuff is done in the classification code for other tool types but it makes more
        # sense for polygons to do it here
        for tool_id in aggregate_polygons:
            # have one set of results per tool type
            # the center will be a list of all polygons
            next_result = dict()

            # we will either have a multi-polygon as our aggregation result - or if we are really
            # lucky, a single polygon
            if isinstance(aggregate_polygons[tool_id],Polygon):
                poly = aggregate_polygons[tool_id]
                next_result["center"] = [zip(poly.exterior.xy[0],poly.exterior.xy[0])]
            elif isinstance(aggregate_polygons[tool_id], MultiPolygon):
                next_result["center"] = []
                for poly in aggregate_polygons[tool_id]:

                    # go through each of the individual polygons making up this multipolygon
                    if isinstance(poly,Polygon):
                        next_result["center"].append(zip(poly.exterior.xy[0],poly.exterior.xy[0]))
            else:
                # unknown type
                print type(aggregate_polygons[tool_id])
                assert False

            # a value of None -> not really relevant to polygon aggregation
            # or really hard to keep track of
            next_result["users"] = None
            next_result["num users"] = None
            next_result["cluster_members"] = None
            next_result["tool classification"] = tool_id

            # these are global values which are not really specific to any one polygon
            # but this seems to be the best place to store the values
            next_result["incorrect area"] = incorrect_area_as_percent
            # todo - don't hard code this
            next_result["minimum users"] = 3
            next_result["area"] = aggregate_stats[tool_id][1]
            next_result["certainty"] = aggregate_stats[tool_id][0]

            results.append(next_result)

        if results == []:
            # add in a dummy polygon so that we can report back the size of
            # the incorrect area
            results = [{"area":0,"incorrect area":incorrect_area_as_percent,"certain": -1},]
            print "  " + str(incorrect_area_as_percent)

        return results,0
<|endoftext|>


<commit_before># Copyright 2015 The go-python Authors.  All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

### py2/py3 compat
from __future__ import print_function

__doc__ = """gopy is a convenience module to wrap and bind a Go package"""
__author__ = "The go-python authors"

__all__ = [
        "load",
        ]

### stdlib imports ---
import imp
import os
import sys

def load(pkg, output=""):
    """
    `load` takes a fully qualified Go package name and runs `gopy bind` on it.
    @returns the C-extension module object
    """

    from subprocess import check_call, check_output
    if output == "":
        output = os.getcwd()
        pass

    print("gopy> inferring package name...")
    pkg = check_output(["go", "list", pkg]).strip()
    print("gopy> loading '%s'..." % pkg)

    check_call(["gopy","bind", "-output=%s" % output, pkg])

    n = os.path.basename(pkg)
    print("gopy> importing '%s'" % (pkg,))

    ok = imp.find_module(n, [output])
    if not ok:
        raise RuntimeError("could not find module '%s'" % pkg)
    fname, path, descr = ok
    mod = imp.load_module('__gopy__.'+n, fname, path, descr)
    mod.__name__ = pkg
    sys.modules[pkg] = mod
    del sys.modules['__gopy__.'+n]
    return mod

<commit_msg>gopy: dont reload C-extension module<commit_after># Copyright 2015 The go-python Authors.  All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

### py2/py3 compat
from __future__ import print_function

__doc__ = """gopy is a convenience module to wrap and bind a Go package"""
__author__ = "The go-python authors"

__all__ = [
        "load",
        ]

### stdlib imports ---
import imp
import os
import sys

def load(pkg, output=""):
    """
    `load` takes a fully qualified Go package name and runs `gopy bind` on it.
    @returns the C-extension module object
    """

    from subprocess import check_call, check_output
    if output == "":
        output = os.getcwd()
        pass

    print("gopy> inferring package name...")
    pkg = check_output(["go", "list", pkg]).strip()
    if pkg in sys.modules:
        print("gopy> package '%s' already wrapped and loaded!" % (pkg,))
        print("gopy> NOT recompiling it again (see issue #27)")
        return sys.modules[pkg]
    print("gopy> loading '%s'..." % pkg)

    check_call(["gopy","bind", "-output=%s" % output, pkg])

    n = os.path.basename(pkg)
    print("gopy> importing '%s'" % (pkg,))

    ok = imp.find_module(n, [output])
    if not ok:
        raise RuntimeError("could not find module '%s'" % pkg)
    fname, path, descr = ok
    mod = imp.load_module('__gopy__.'+n, fname, path, descr)
    mod.__name__ = pkg
    sys.modules[pkg] = mod
    del sys.modules['__gopy__.'+n]
    return mod

<|endoftext|>


<commit_before>

__author__ = 'scribblemaniac'

from renderchan.module import RenderChanModule
from renderchan.utils import which
from distutils.spawn import find_executable
import subprocess
import os
import re
import random

class RenderChanGimpModule(RenderChanModule):
    def __init__(self):
        RenderChanModule.__init__(self)
        if os.name == 'nt':
            self.conf['binary']=os.path.join(os.path.dirname(__file__),"..\\..\\..\\gimp\\gimp.exe")
        else:
            self.conf['binary']="gimp"
        self.conf["packetSize"]=0

    def getInputFormats(self):
        return ["xcf", "fli", "flc", "dds", "dcm", "dicom", "eps", "fit", "fits", "g3", "xjt", "cel", "wmf", "ico", "pnm", "ppm", "pgm", "pbm", "psp", "psd", "pdf", "ps", "tiff", "bmp", "xbm", "xwd", "pcx", "pcc"]

    def getOutputFormats(self):
        return ["png", "jpg", "jpeg", "pdf", "psd", "tif", "tiff", "bmp", "ico", "txt", "html", "gif", "mng"]

    def render(self, filename, outputPath, startFrame, endFrame, format, updateCompletion, extraParams={}):

        comp=0.0
        updateCompletion(comp)

        # Determines drawable (depending on layer support)
        if format in ("gif", "ico", "mng", "psd"):
            drawable="gimp-image-get-active-drawable image"
        else:
            drawable="gimp-image-merge-visible-layers image CLIP-TO-IMAGE"

        # Preprocessing procedures
        if format == "gif":
            preprocedure="(gimp-image-convert-indexed image 0 0 255 0 0 \"\")"
        else:
            preprocedure=""

        # Determine procedure name
        if format in ("png", "tiff"):
            saveProcedure="file-%s-save2" % format
        elif format in ("txt", "html"):
            saveProcedure="file-aa-save"
        elif format == "jpg":
            saveProcedure="file-jpeg-save"
        elif format == "tif":
            saveProcedure="file-tiff-save2"
        elif format in ("jpeg", "pdf", "psd", "gif", "mng"):
            saveProcedure="file-%s-save" % format
        else:
            saveProcedure="gimp-file-save"

        # Determine paramters
        if format == "png":
            # PNG get special treatment because default parameters can be fetched from GIMP
            #saveParameters=" ".join(subprocess.check_output([self.conf['binary'], "-i", "-b", "(let ((str \"\") (defaults (file-png-get-defaults))) (gimp-message (do ((i 0 (+ i 1))) ((= i 9) str) (set! str (string-append str (number->string (nth i defaults)))))))", "-b", "(gimp-quit 0)"], stderr=subprocess.STDOUT)[19:28])
            #print saveParameters
            saveParameters="0 9 0 0 0 1 0 0 0"
        elif format in ("jpg", "jpeg"):
            saveParameters=".9 0 0 0 \" \" 0 1 0 1"
        elif format == "pdf":
            saveParameters="0 1 1"
        elif format == "psd":
            saveParameters="1 1"
        elif format in ("tif", "tiff"):
            saveParameters="1 0"
        elif format == "txt":
            saveParameters="\"Text file\""
        elif format == "html":
            saveParameters="\"Pure html\""
        elif format == "gif":
            saveParameters="0 1 100 2"
        elif format == "mng":
            saveParameters="0 9 .9 0 1 100 2 1 0 0 1 0"
        else:
            saveParameters=""

        # See docs for readable script-fu code
        commandline=[self.conf['binary'], "-i", "-b", "(let*  ((filename \"%s\") (outpath \"%s\") (image (car (gimp-file-load RUN-NONINTERACTIVE filename filename))) (drawable (car (%s)))) %s (gimp-image-scale image %s %s) (%s RUN-NONINTERACTIVE image drawable outpath outpath %s) (gimp-image-delete image))" % (filename, outputPath, drawable, preprocedure, extraParams['width'], extraParams['height'], saveProcedure, saveParameters), "-b", "(gimp-quit 0)"]

        subprocess.check_call(commandline)

        updateCompletion(1.0)
<commit_msg>Remove unused import<commit_after>

__author__ = 'scribblemaniac'

from renderchan.module import RenderChanModule
from renderchan.utils import which
import subprocess
import os
import re
import random

class RenderChanGimpModule(RenderChanModule):
    def __init__(self):
        RenderChanModule.__init__(self)
        if os.name == 'nt':
            self.conf['binary']=os.path.join(os.path.dirname(__file__),"..\\..\\..\\gimp\\gimp.exe")
        else:
            self.conf['binary']="gimp"
        self.conf["packetSize"]=0

    def getInputFormats(self):
        return ["xcf", "fli", "flc", "dds", "dcm", "dicom", "eps", "fit", "fits", "g3", "xjt", "cel", "wmf", "ico", "pnm", "ppm", "pgm", "pbm", "psp", "psd", "pdf", "ps", "tiff", "bmp", "xbm", "xwd", "pcx", "pcc"]

    def getOutputFormats(self):
        return ["png", "jpg", "jpeg", "pdf", "psd", "tif", "tiff", "bmp", "ico", "txt", "html", "gif", "mng"]

    def render(self, filename, outputPath, startFrame, endFrame, format, updateCompletion, extraParams={}):

        comp=0.0
        updateCompletion(comp)

        # Determines drawable (depending on layer support)
        if format in ("gif", "ico", "mng", "psd"):
            drawable="gimp-image-get-active-drawable image"
        else:
            drawable="gimp-image-merge-visible-layers image CLIP-TO-IMAGE"

        # Preprocessing procedures
        if format == "gif":
            preprocedure="(gimp-image-convert-indexed image 0 0 255 0 0 \"\")"
        else:
            preprocedure=""

        # Determine procedure name
        if format in ("png", "tiff"):
            saveProcedure="file-%s-save2" % format
        elif format in ("txt", "html"):
            saveProcedure="file-aa-save"
        elif format == "jpg":
            saveProcedure="file-jpeg-save"
        elif format == "tif":
            saveProcedure="file-tiff-save2"
        elif format in ("jpeg", "pdf", "psd", "gif", "mng"):
            saveProcedure="file-%s-save" % format
        else:
            saveProcedure="gimp-file-save"

        # Determine paramters
        if format == "png":
            # PNG get special treatment because default parameters can be fetched from GIMP
            #saveParameters=" ".join(subprocess.check_output([self.conf['binary'], "-i", "-b", "(let ((str \"\") (defaults (file-png-get-defaults))) (gimp-message (do ((i 0 (+ i 1))) ((= i 9) str) (set! str (string-append str (number->string (nth i defaults)))))))", "-b", "(gimp-quit 0)"], stderr=subprocess.STDOUT)[19:28])
            #print saveParameters
            saveParameters="0 9 0 0 0 1 0 0 0"
        elif format in ("jpg", "jpeg"):
            saveParameters=".9 0 0 0 \" \" 0 1 0 1"
        elif format == "pdf":
            saveParameters="0 1 1"
        elif format == "psd":
            saveParameters="1 1"
        elif format in ("tif", "tiff"):
            saveParameters="1 0"
        elif format == "txt":
            saveParameters="\"Text file\""
        elif format == "html":
            saveParameters="\"Pure html\""
        elif format == "gif":
            saveParameters="0 1 100 2"
        elif format == "mng":
            saveParameters="0 9 .9 0 1 100 2 1 0 0 1 0"
        else:
            saveParameters=""

        # See docs for readable script-fu code
        commandline=[self.conf['binary'], "-i", "-b", "(let*  ((filename \"%s\") (outpath \"%s\") (image (car (gimp-file-load RUN-NONINTERACTIVE filename filename))) (drawable (car (%s)))) %s (gimp-image-scale image %s %s) (%s RUN-NONINTERACTIVE image drawable outpath outpath %s) (gimp-image-delete image))" % (filename, outputPath, drawable, preprocedure, extraParams['width'], extraParams['height'], saveProcedure, saveParameters), "-b", "(gimp-quit 0)"]

        subprocess.check_call(commandline)

        updateCompletion(1.0)
<|endoftext|>


<commit_before>from flask import Flask, render_template, g
from flask.ext.cache import Cache
from ede_test.config import CACHE_CONFIG
from ede_test.database import engine, db_session
from ede_test.api.utils import ListConverter, IntListConverter

app = Flask(__name__)
app.config.from_object('ede_test.config')
app.url_map.converters['list'] = ListConverter
app.url_map.converters['intlist'] = IntListConverter
app.url_map.strict_slashes = False

cache = Cache(app, config=CACHE_CONFIG)

@app.errorhandler(404)
def not_found(error):
    return render_template('errors/404.html'), 404

@app.errorhandler(403)
def not_found(error):
    return render_template('errors/403.html'), 403

@app.errorhandler(500)
def not_found(error):
    return render_template('errors/500.html'), 500

from ede_test.api.views import api as api_module
app.register_blueprint(api_module)

@app.before_request
def before_request():
    g.db = engine.dispose()

@app.teardown_request
@app.teardown_appcontext
def shutdown_session(exception=None):
    try:
        g.db.close()
        db_session.remove()
    except:
        try:
            db_session.rollback()
            db_session.close()
            g.db.close()
        except:
            pass
<commit_msg>Remove template references<commit_after>try:
    import simplejson as json
except ImportError:
    import json
from datetime import date
from flask import Flask, render_template, g, make_response
from flask.ext.cache import Cache
from ede.config import CACHE_CONFIG
from ede.database import engine, db_session
from ede.api.utils import ListConverter, IntListConverter

app = Flask(__name__)
app.config.from_object('ede_test.config')
app.url_map.converters['list'] = ListConverter
app.url_map.converters['intlist'] = IntListConverter
app.url_map.strict_slashes = False

cache = Cache(app, config=CACHE_CONFIG)

dthandler = lambda obj: obj.isoformat() if isinstance(obj, date) else None

@app.errorhandler(404)
def not_found(error):
    resp = {'meta': {'status': '', 'message': '', },
            'objects': [] }
    resp = make_response(json.dumps(resp, default=dthandler), 404)
    return resp

@app.errorhandler(403)
def not_found(error):
    resp = {'meta': {'status': '', 'message': '', },
            'objects': [] }
    resp = make_response(json.dumps(resp, default=dthandler), 403)
    return resp

@app.errorhandler(500)
def not_found(error):
    resp = {'meta': {'status': '', 'message': '', },
            'objects': [] }
    resp = make_response(json.dumps(resp, default=dthandler), 500)
    return resp

from ede.api.views import api as api_module
app.register_blueprint(api_module)

@app.before_request
def before_request():
    g.db = engine.dispose()

@app.teardown_request
@app.teardown_appcontext
def shutdown_session(exception=None):
    try:
        g.db.close()
        db_session.remove()
    except:
        try:
            db_session.rollback()
            db_session.close()
            g.db.close()
        except:
            pass
<|endoftext|>


<commit_before><commit_msg>Exclude unused options for call of clear_index and update_index<commit_after><|endoftext|>


<commit_before>''' -- imports from python libraries -- '''
import json

''' -- imports from installed packages -- '''
try:
    from bson import ObjectId
except ImportError:  # old pymongo
    from pymongo.objectid import ObjectId

from django.http import HttpResponseRedirect, HttpResponse
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.contrib.auth.decorators import login_required
from django.core.urlresolvers import reverse

''' -- imports from application folders/files -- '''
from gnowsys_ndf.ndf.models import GSystemType, Group, Node  # GSystem, Triple
from gnowsys_ndf.ndf.models import node_collection

from gnowsys_ndf.ndf.views.group import CreateGroup
from gnowsys_ndf.ndf.views.methods import get_execution_time, staff_required, create_gattribute
from gnowsys_ndf.ndf.views.ajax_views import get_collection

gst_base_unit_name, gst_base_unit_id = GSystemType.get_gst_name_id('base_unit')
gst_lesson_name, gst_lesson_id = GSystemType.get_gst_name_id('lesson')
gst_activity_name, gst_activity_id = GSystemType.get_gst_name_id('activity')

@login_required
@get_execution_time
def unit_create_edit(request, group_id, unit_group_id=None):
    '''
    creation as well as eit of units
    '''

    parent_group_name, parent_group_id = Group.get_group_name_id(group_id)
    unit_node = None
    if request.method == "GET":
        unit_node = node_collection.one({'_id': ObjectId(unit_group_id)})
        template = "ndf/create_unit.html"
        all_groups = node_collection.find({'_type': "Group"},{"name":1})
        all_groups_names = [str(each_group.name) for each_group in all_groups]
        context_variables = {'group_id': parent_group_id,'groupid': parent_group_id, 'all_groups_names': all_groups_names}
        if unit_node:
            context_variables.update({'unit_node': unit_node})
        req_context = RequestContext(request, context_variables)
        return render_to_response(template, req_context)

    elif request.method == "POST":
        group_name = request.POST.get('name', '')
        group_altnames = request.POST.get('altnames', '')
        unit_id_post = request.POST.get('node_id', '')
        unit_altnames = request.POST.get('altnames', '')
        content = request.POST.get('content', '')

        educationallevel_val = request.POST.get('educationallevel', '')
        educationalsubject_val = request.POST.get('educationalsubject', '')
        # unit_group_id = unit_id_post if unit_id_post else unit_group_id
        # unit_group_name, unit_group_id = Group.get_group_name_id(unit_group_id)
        if unit_id_post:
            unit_node = node_collection.one({'_id': ObjectId(unit_id_post)})
        success_flag = False
        if unit_node:
            if unit_node.altnames is not unit_altnames:
                unit_node.altnames = unit_altnames
                success_flag = True
        else:
            unit_group = CreateGroup(request)
            result = unit_group.create_group(group_name,
                                            group_id=parent_group_id,
                                            member_of=gst_base_unit_id,
                                            node_id=unit_group_id)
            success_flag = result[0]
            unit_node = result[1]
        if content:
            unit_node.content = content
            unit_node.save()

        if educationallevel_val and "choose" not in educationallevel_val.lower():
            educationallevel_at = node_collection.one({'_type': 'AttributeType', 'name': "educationallevel"})
            create_gattribute(unit_node._id, educationallevel_at, educationallevel_val)
        if educationalsubject_val and "choose" not in educationalsubject_val.lower():
            educationalsubject_at = node_collection.one({'_type': 'AttributeType', 'name': "educationalsubject"})
            create_gattribute(unit_node._id, educationalsubject_at, educationalsubject_val)

        if not success_flag:
            return HttpResponseRedirect(reverse('list_units', kwargs={'group_id': parent_group_id, 'groupid': parent_group_id,}))
        return HttpResponseRedirect(reverse('course_about',
            kwargs={'group_id': unit_node._id}))

@get_execution_time
def unit_detail(request, group_id):
    '''
    detail of of selected units
    '''
    # parent_group_name, parent_group_id = Group.get_group_name_id(group_id)
    unit_group_obj = Group.get_group_name_id(group_id, get_obj=True)

    unit_structure = _get_unit_hierarchy(unit_group_obj)
    # template = "ndf/unit_structure.html"
    template = 'ndf/gevent_base.html'

    # print unit_structure
    req_context = RequestContext(request, {
                                'title': 'unit_authoring',
                                'group_id': group_id,
                                'groupid': group_id,
                                'unit_obj': unit_group_obj,
                                'unit_structure': json.dumps(unit_structure)
                            })
    return render_to_response(template, req_context)

@get_execution_time
def list_units(request, group_id):
    '''
    listing of units
    '''
    parent_group_name, parent_group_id = Group.get_group_name_id(group_id)
    all_base_units = node_collection.find({
                                    '_type': 'Group',
                                    'group_set': {'$in': [parent_group_id]},
                                    'member_of': {'$in': [gst_base_unit_id]},
                                    '$or':[
                                        {'status': u'PUBLIC'},
                                        {
                                            '$and': [
                                                {'access_policy': u"PRIVATE"},
                                                {'created_by': request.user.id}
                                            ]
                                        }
                                    ]
                                }).sort('last_update', -1)

    template = "ndf/explore_2017.html"
    req_context = RequestContext(request, {
                                'group_id': parent_group_id,
                                'all_unit_objs': all_base_units
                            })
    return render_to_response(template, req_context)

########### LESSON methods ##########

@login_required
@get_execution_time
def lesson_create_edit(request, group_id, unit_group_id=None):
    '''
    creation as well as edit of lessons
    returns following list:
    {'success': <BOOL: 0 or 1>, 'unit_hierarchy': <unit hierarchy json>, 'msg': <error msg or objectid of newly created obj>}
    '''
    # parent_group_name, parent_group_id = Group.get_group_name_id(group_id)

    # parent unit id
    unit_id_post = request.POST.get('unit_id', '')
    unit_group_id = unit_id_post if unit_id_post else unit_group_id
    # getting parent unit object
    unit_group_obj = Group.get_group_name_id(unit_group_id, get_obj=True)
    result_dict = {'success': 0, 'unit_hierarchy': [], 'msg': ''}
    if request.method == "POST":
        # lesson name
        lesson_name = request.POST.get('name', '').strip()
        if not lesson_name:
            msg = 'Name can not be empty.'
            result_dict = {'success': 0, 'unit_hierarchy': [], 'msg': msg}
            # return HttpResponse(0)

        # unit_cs: unit collection_set
        unit_cs_list = unit_group_obj.collection_set
        unit_cs_objs_cur = Node.get_nodes_by_ids_list(unit_cs_list)
        unit_cs_names_list = [u.name for u in unit_cs_objs_cur]

        if lesson_name in unit_cs_names_list:
            msg = u'Activity with same name exists in lesson: ' + unit_group_obj.name
            result_dict = {'success': 0, 'unit_hierarchy': [], 'msg': msg}
            # return HttpResponse(0)
        else:
            user_id = request.user.id
            new_lesson_obj = node_collection.collection.GSystem()
            new_lesson_obj.fill_gstystem_values(name=lesson_name,
                                            member_of=gst_lesson_id,
                                            group_set=unit_group_obj._id,
                                            created_by=user_id,
                                            status='PUBLISHED')
            # print new_lesson_obj
            new_lesson_obj.save(groupid=group_id)
            unit_group_obj.collection_set.append(new_lesson_obj._id)
            unit_group_obj.save(groupid=group_id)

            unit_structure = _get_unit_hierarchy(unit_group_obj)

            msg = u'Added lesson under lesson: ' + unit_group_obj.name
            result_dict = {'success': 1, 'unit_hierarchy': unit_structure, 'msg': str(new_lesson_obj._id)}
            # return HttpResponse(json.dumps(unit_structure))

    # return HttpResponse(1)
    return HttpResponse(json.dumps(result_dict))

# @login_required
@get_execution_time
def activity_create_edit(request, group_id, lesson_id=None):
    '''
    creation as well as edit of activities
    returns following list:
    {'success': <BOOL: 0 or 1>, 'unit_hierarchy': <unit hierarchy json>, 'msg': <error msg or objectid of newly created obj>}
    '''
    # parent_group_name, parent_group_id = Group.get_group_name_id(group_id)
    lesson_id = request.POST.get('lesson_id')
    lesson_obj = Node.get_node_by_id(lesson_id)

    # parent unit id
    unit_id_post = request.POST.get('unit_id', '')
    unit_group_id = unit_id_post if unit_id_post else unit_group_id

    # getting parent unit object
    unit_group_obj = Group.get_group_name_id(unit_group_id, get_obj=True)

    result_dict = {'success': 0, 'unit_hierarchy': [], 'msg': ''}

    if request.method == "POST":
        # activity name
        activity_name = request.POST.get('name', '').strip()
        if not activity_name:
            msg = 'Name can not be empty.'
            result_dict = {'success': 0, 'unit_hierarchy': [], 'msg': msg}
            # return HttpResponse(result_dict)

        # unit_cs: unit collection_set
        lesson_cs_list = lesson_obj.collection_set
        lesson_cs_objs_cur = Node.get_nodes_by_ids_list(lesson_cs_list)
        lesson_cs_names_list = [u.name for u in lesson_cs_objs_cur]

        if activity_name in lesson_cs_names_list:
            msg = u'Activity with same name exists in lesson: ' + lesson_obj.name
            result_dict = {'success': 0, 'unit_hierarchy': [], 'msg': msg}
            # return HttpResponse(0)
        else:
            user_id = request.user.id
            new_activity_obj = node_collection.collection.GSystem()
            new_activity_obj.fill_gstystem_values(name=activity_name,
                                            member_of=gst_activity_id,
                                            group_set=unit_group_obj._id,
                                            created_by=user_id,
                                            status='PUBLISHED')
            new_activity_obj.save(groupid=group_id)

            lesson_obj.collection_set.append(new_activity_obj._id)
            lesson_obj.save(groupid=group_id)
            unit_structure = _get_unit_hierarchy(unit_group_obj)

            msg = u'Added activity under lesson: ' + lesson_obj.name
            result_dict = {'success': 1, 'unit_hierarchy': unit_structure, 'msg': str(new_activity_obj._id)}
            # return HttpResponse(json.dumps(unit_structure))

    return HttpResponse(json.dumps(result_dict))

def _get_unit_hierarchy(unit_group_obj):
    '''
    ARGS: unit_group_obj
    Result will be of following form:
    {
        name: 'Lesson1',
        type: 'lesson',
        id: 'l1',
        activities: [
            {
                name: 'Activity 1',
                type: 'activity',
                id: 'a1'
            },
            {
                name: 'Activity 1',
                type: 'activity',
                id: 'a2'
            }
        ]
    }, {
        name: 'Lesson2',
        type: 'lesson',
        id: 'l2',
        activities: [
            {
                name: 'Activity 1',
                type: 'activity',
                id: 'a1'
            }
        ]
    }
    '''
    unit_structure = []
    for each in unit_group_obj.collection_set:
        lesson_dict ={}
        lesson = Node.get_node_by_id(each)
        if lesson:
            lesson_dict['name'] = lesson.name
            lesson_dict['type'] = 'lesson'
            lesson_dict['id'] = str(lesson._id)
            lesson_dict['activities'] = []
            if lesson.collection_set:
                for each_act in lesson.collection_set:
                    activity_dict ={}
                    activity = Node.get_node_by_id(each_act)
                    if activity:
                        activity_dict['name'] = activity.name
                        activity_dict['type'] = 'activity'
                        activity_dict['id'] = str(activity._id)
                        lesson_dict['activities'].append(activity_dict)
            unit_structure.append(lesson_dict)

    return unit_structure
<commit_msg>unit content save-update modified.<commit_after>''' -- imports from python libraries -- '''
import json

''' -- imports from installed packages -- '''
try:
    from bson import ObjectId
except ImportError:  # old pymongo
    from pymongo.objectid import ObjectId

from django.http import HttpResponseRedirect, HttpResponse
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.contrib.auth.decorators import login_required
from django.core.urlresolvers import reverse

''' -- imports from application folders/files -- '''
from gnowsys_ndf.ndf.models import GSystemType, Group, Node  # GSystem, Triple
from gnowsys_ndf.ndf.models import node_collection

from gnowsys_ndf.ndf.views.group import CreateGroup
from gnowsys_ndf.ndf.views.methods import get_execution_time, staff_required, create_gattribute
from gnowsys_ndf.ndf.views.ajax_views import get_collection

gst_base_unit_name, gst_base_unit_id = GSystemType.get_gst_name_id('base_unit')
gst_lesson_name, gst_lesson_id = GSystemType.get_gst_name_id('lesson')
gst_activity_name, gst_activity_id = GSystemType.get_gst_name_id('activity')

@login_required
@get_execution_time
def unit_create_edit(request, group_id, unit_group_id=None):
    '''
    creation as well as eit of units
    '''

    parent_group_name, parent_group_id = Group.get_group_name_id(group_id)
    unit_node = None
    if request.method == "GET":
        unit_node = node_collection.one({'_id': ObjectId(unit_group_id)})
        template = "ndf/create_unit.html"
        all_groups = node_collection.find({'_type': "Group"},{"name":1})
        all_groups_names = [str(each_group.name) for each_group in all_groups]
        context_variables = {'group_id': parent_group_id,'groupid': parent_group_id, 'all_groups_names': all_groups_names}
        if unit_node:
            context_variables.update({'unit_node': unit_node})
        req_context = RequestContext(request, context_variables)
        return render_to_response(template, req_context)

    elif request.method == "POST":
        group_name = request.POST.get('name', '')
        group_altnames = request.POST.get('altnames', '')
        unit_id_post = request.POST.get('node_id', '')
        unit_altnames = request.POST.get('altnames', '')
        content = request.POST.get('content', '')

        educationallevel_val = request.POST.get('educationallevel', '')
        educationalsubject_val = request.POST.get('educationalsubject', '')
        # unit_group_id = unit_id_post if unit_id_post else unit_group_id
        # unit_group_name, unit_group_id = Group.get_group_name_id(unit_group_id)
        if unit_id_post:
            unit_node = node_collection.one({'_id': ObjectId(unit_id_post)})
        success_flag = False
        if unit_node:
            if unit_node.altnames is not unit_altnames:
                unit_node.altnames = unit_altnames
                success_flag = True
        else:
            unit_group = CreateGroup(request)
            result = unit_group.create_group(group_name,
                                            group_id=parent_group_id,
                                            member_of=gst_base_unit_id,
                                            node_id=unit_group_id)
            success_flag = result[0]
            unit_node = result[1]

        if educationallevel_val and "choose" not in educationallevel_val.lower():
            educationallevel_at = node_collection.one({'_type': 'AttributeType', 'name': "educationallevel"})
            create_gattribute(unit_node._id, educationallevel_at, educationallevel_val)
        if educationalsubject_val and "choose" not in educationalsubject_val.lower():
            educationalsubject_at = node_collection.one({'_type': 'AttributeType', 'name': "educationalsubject"})
            create_gattribute(unit_node._id, educationalsubject_at, educationalsubject_val)

        if not success_flag:
            return HttpResponseRedirect(reverse('list_units', kwargs={'group_id': parent_group_id, 'groupid': parent_group_id,}))

        unit_node.content = content
        unit_node.save()
        return HttpResponseRedirect(reverse('course_about',
            kwargs={'group_id': unit_node._id}))

@get_execution_time
def unit_detail(request, group_id):
    '''
    detail of of selected units
    '''
    # parent_group_name, parent_group_id = Group.get_group_name_id(group_id)
    unit_group_obj = Group.get_group_name_id(group_id, get_obj=True)

    unit_structure = _get_unit_hierarchy(unit_group_obj)
    # template = "ndf/unit_structure.html"
    template = 'ndf/gevent_base.html'

    # print unit_structure
    req_context = RequestContext(request, {
                                'title': 'unit_authoring',
                                'group_id': group_id,
                                'groupid': group_id,
                                'unit_obj': unit_group_obj,
                                'unit_structure': json.dumps(unit_structure)
                            })
    return render_to_response(template, req_context)

@get_execution_time
def list_units(request, group_id):
    '''
    listing of units
    '''
    parent_group_name, parent_group_id = Group.get_group_name_id(group_id)
    all_base_units = node_collection.find({
                                    '_type': 'Group',
                                    'group_set': {'$in': [parent_group_id]},
                                    'member_of': {'$in': [gst_base_unit_id]},
                                    '$or':[
                                        {'status': u'PUBLIC'},
                                        {
                                            '$and': [
                                                {'access_policy': u"PRIVATE"},
                                                {'created_by': request.user.id}
                                            ]
                                        }
                                    ]
                                }).sort('last_update', -1)

    template = "ndf/explore_2017.html"
    req_context = RequestContext(request, {
                                'group_id': parent_group_id,
                                'all_unit_objs': all_base_units
                            })
    return render_to_response(template, req_context)

########### LESSON methods ##########

@login_required
@get_execution_time
def lesson_create_edit(request, group_id, unit_group_id=None):
    '''
    creation as well as edit of lessons
    returns following list:
    {'success': <BOOL: 0 or 1>, 'unit_hierarchy': <unit hierarchy json>, 'msg': <error msg or objectid of newly created obj>}
    '''
    # parent_group_name, parent_group_id = Group.get_group_name_id(group_id)

    # parent unit id
    unit_id_post = request.POST.get('unit_id', '')
    unit_group_id = unit_id_post if unit_id_post else unit_group_id
    # getting parent unit object
    unit_group_obj = Group.get_group_name_id(unit_group_id, get_obj=True)
    result_dict = {'success': 0, 'unit_hierarchy': [], 'msg': ''}
    if request.method == "POST":
        # lesson name
        lesson_name = request.POST.get('name', '').strip()
        if not lesson_name:
            msg = 'Name can not be empty.'
            result_dict = {'success': 0, 'unit_hierarchy': [], 'msg': msg}
            # return HttpResponse(0)

        # unit_cs: unit collection_set
        unit_cs_list = unit_group_obj.collection_set
        unit_cs_objs_cur = Node.get_nodes_by_ids_list(unit_cs_list)
        unit_cs_names_list = [u.name for u in unit_cs_objs_cur]

        if lesson_name in unit_cs_names_list:
            msg = u'Activity with same name exists in lesson: ' + unit_group_obj.name
            result_dict = {'success': 0, 'unit_hierarchy': [], 'msg': msg}
            # return HttpResponse(0)
        else:
            user_id = request.user.id
            new_lesson_obj = node_collection.collection.GSystem()
            new_lesson_obj.fill_gstystem_values(name=lesson_name,
                                            member_of=gst_lesson_id,
                                            group_set=unit_group_obj._id,
                                            created_by=user_id,
                                            status='PUBLISHED')
            # print new_lesson_obj
            new_lesson_obj.save(groupid=group_id)
            unit_group_obj.collection_set.append(new_lesson_obj._id)
            unit_group_obj.save(groupid=group_id)

            unit_structure = _get_unit_hierarchy(unit_group_obj)

            msg = u'Added lesson under lesson: ' + unit_group_obj.name
            result_dict = {'success': 1, 'unit_hierarchy': unit_structure, 'msg': str(new_lesson_obj._id)}
            # return HttpResponse(json.dumps(unit_structure))

    # return HttpResponse(1)
    return HttpResponse(json.dumps(result_dict))

# @login_required
@get_execution_time
def activity_create_edit(request, group_id, lesson_id=None):
    '''
    creation as well as edit of activities
    returns following list:
    {'success': <BOOL: 0 or 1>, 'unit_hierarchy': <unit hierarchy json>, 'msg': <error msg or objectid of newly created obj>}
    '''
    # parent_group_name, parent_group_id = Group.get_group_name_id(group_id)
    lesson_id = request.POST.get('lesson_id')
    lesson_obj = Node.get_node_by_id(lesson_id)

    # parent unit id
    unit_id_post = request.POST.get('unit_id', '')
    unit_group_id = unit_id_post if unit_id_post else unit_group_id

    # getting parent unit object
    unit_group_obj = Group.get_group_name_id(unit_group_id, get_obj=True)

    result_dict = {'success': 0, 'unit_hierarchy': [], 'msg': ''}

    if request.method == "POST":
        # activity name
        activity_name = request.POST.get('name', '').strip()
        if not activity_name:
            msg = 'Name can not be empty.'
            result_dict = {'success': 0, 'unit_hierarchy': [], 'msg': msg}
            # return HttpResponse(result_dict)

        # unit_cs: unit collection_set
        lesson_cs_list = lesson_obj.collection_set
        lesson_cs_objs_cur = Node.get_nodes_by_ids_list(lesson_cs_list)
        lesson_cs_names_list = [u.name for u in lesson_cs_objs_cur]

        if activity_name in lesson_cs_names_list:
            msg = u'Activity with same name exists in lesson: ' + lesson_obj.name
            result_dict = {'success': 0, 'unit_hierarchy': [], 'msg': msg}
            # return HttpResponse(0)
        else:
            user_id = request.user.id
            new_activity_obj = node_collection.collection.GSystem()
            new_activity_obj.fill_gstystem_values(name=activity_name,
                                            member_of=gst_activity_id,
                                            group_set=unit_group_obj._id,
                                            created_by=user_id,
                                            status='PUBLISHED')
            new_activity_obj.save(groupid=group_id)

            lesson_obj.collection_set.append(new_activity_obj._id)
            lesson_obj.save(groupid=group_id)
            unit_structure = _get_unit_hierarchy(unit_group_obj)

            msg = u'Added activity under lesson: ' + lesson_obj.name
            result_dict = {'success': 1, 'unit_hierarchy': unit_structure, 'msg': str(new_activity_obj._id)}
            # return HttpResponse(json.dumps(unit_structure))

    return HttpResponse(json.dumps(result_dict))

def _get_unit_hierarchy(unit_group_obj):
    '''
    ARGS: unit_group_obj
    Result will be of following form:
    {
        name: 'Lesson1',
        type: 'lesson',
        id: 'l1',
        activities: [
            {
                name: 'Activity 1',
                type: 'activity',
                id: 'a1'
            },
            {
                name: 'Activity 1',
                type: 'activity',
                id: 'a2'
            }
        ]
    }, {
        name: 'Lesson2',
        type: 'lesson',
        id: 'l2',
        activities: [
            {
                name: 'Activity 1',
                type: 'activity',
                id: 'a1'
            }
        ]
    }
    '''
    unit_structure = []
    for each in unit_group_obj.collection_set:
        lesson_dict ={}
        lesson = Node.get_node_by_id(each)
        if lesson:
            lesson_dict['name'] = lesson.name
            lesson_dict['type'] = 'lesson'
            lesson_dict['id'] = str(lesson._id)
            lesson_dict['activities'] = []
            if lesson.collection_set:
                for each_act in lesson.collection_set:
                    activity_dict ={}
                    activity = Node.get_node_by_id(each_act)
                    if activity:
                        activity_dict['name'] = activity.name
                        activity_dict['type'] = 'activity'
                        activity_dict['id'] = str(activity._id)
                        lesson_dict['activities'].append(activity_dict)
            unit_structure.append(lesson_dict)

    return unit_structure
<|endoftext|>


<commit_before>"""Tests for go.vumitools.metrics_worker."""

import copy
import re

from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet.task import Clock, LoopingCall

from go.vumitools.tests.utils import GoWorkerTestCase
from go.vumitools import metrics_worker

from vumi.tests.utils import LogCatcher

class GoMetricsWorkerTestCase(GoWorkerTestCase):
    worker_class = metrics_worker.GoMetricsWorker

    def setUp(self):
        super(GoMetricsWorkerTestCase, self).setUp()
        self.clock = Clock()
        self.patch(metrics_worker, 'LoopingCall', self.looping_call)

    def get_metrics_worker(self, config=None, start=True,
                           needs_looping=False, needs_hash=False):
        if not needs_looping:
            self.patch(metrics_worker, 'LoopingCall', self.no_looping)
        if not needs_hash:
            self.patch(metrics_worker.GoMetricsWorker,
                       'bucket_for_conversation',
                       self.dummy_bucket_for_conversation)
        if config is None:
            config = {}
        return self.get_worker(self.mk_config(config), start)

    def rkey(self, name):
        return name

    def looping_call(self, *args, **kwargs):
        looping_call = LoopingCall(*args, **kwargs)
        looping_call.clock = self.clock
        return looping_call

    def no_looping(self, *args, **kw):
        return self.looping_call(lambda: None)

    def dummy_bucket_for_conversation(self, conv):
        digits = re.sub('[^0-9]', '', conv.name) or '0'
        return int(digits) % 60

    def make_account(self, worker, username):
        return self.mk_user(worker.vumi_api, username)

    def make_conv(self, user_api, conv_name, conv_type=u'my_conv'):
        return user_api.new_conversation(conv_type, conv_name, u'', {})

    def start_conv(self, conv):
        conv.set_status_started()
        return conv.save()

    def archive_conv(self, conv):
        conv.set_status_stopped()
        conv.set_status_finished()
        return conv.save()

    @inlineCallbacks
    def make_accounts_and_conversations(self, worker, **account_definitions):
        accounts, conversations = {}, {}
        for username, convs in account_definitions.iteritems():
            account = yield self.make_account(worker, unicode(username))
            accounts[username] = account
            user_api = worker.vumi_api.get_user_api(account.key)
            for conv_name in convs:
                conv = yield self.make_conv(user_api, unicode(conv_name))
                conversations[conv_name] = conv

        returnValue((accounts, conversations))

    @inlineCallbacks
    def test_bucket_for_conversation(self):
        worker = yield self.get_metrics_worker(needs_hash=True)
        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=["conv1"])
        bucket = worker.bucket_for_conversation(conversations["conv1"])
        self.assertEqual(bucket, hash(conversations["conv1"].key) % 60)

    def assert_conversations_bucketed(self, worker, expected):
        expected = copy.deepcopy(expected)
        buckets = copy.deepcopy(worker._buckets)
        for key in range(60):
            buckets[key] = sorted(c.key for c in buckets[key])
            expected[key] = sorted(c.key for c in expected.get(key, []))
            if buckets[key] == expected[key]:
                del buckets[key]
                del expected[key]
        self.assertEqual(buckets, expected)

    @inlineCallbacks
    def test_populate_conversation_buckets(self):
        worker = yield self.get_metrics_worker()

        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=["conv1", "conv2a", "conv2b", "conv4"])
        for conv in conversations.values():
            self.start_conv(conv)

        self.assert_conversations_bucketed(worker, {})
        with LogCatcher(message='Scheduled') as lc:
            yield worker.populate_conversation_buckets()
            [log_msg] = lc.messages()
        self.assert_conversations_bucketed(worker, {
            1: [conversations["conv1"]],
            2: [conversations["conv2a"], conversations["conv2b"]],
            4: [conversations["conv4"]],
        })
        self.assertEqual(log_msg, "Scheduled metrics commands for"
                         " 4 conversations in 2 accounts.")

    @inlineCallbacks
    def test_process_bucket(self):
        worker = yield self.get_metrics_worker()

        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=["conv1", "conv2a", "conv2b", "conv4"])
        for conv in conversations.values():
            self.start_conv(conv)

        self.assert_conversations_bucketed(worker, {})
        yield worker.populate_conversation_buckets()
        yield worker.process_bucket(2)
        self.assert_conversations_bucketed(worker, {
            1: [conversations["conv1"]],
            4: [conversations["conv4"]],
        })

    @inlineCallbacks
    def test_increment_bucket(self):
        worker = yield self.get_metrics_worker()
        self.assertEqual(worker._current_bucket, 0)
        worker.increment_bucket()
        self.assertEqual(worker._current_bucket, 1)
        worker._current_bucket = 59
        worker.increment_bucket()
        self.assertEqual(worker._current_bucket, 0)

    @inlineCallbacks
    def test_metrics_poller(self):
        polls = []
        # Replace metrics worker with one that hasn't started yet.
        worker = yield self.get_metrics_worker(start=False, needs_looping=True)
        worker.metrics_loop_func = lambda: polls.append(None)
        self.assertEqual(0, len(polls))
        # Start worker.
        yield worker.startWorker()
        self.assertEqual(1, len(polls))
        # Pass time, but not enough to trigger a metric run.
        self.clock.advance(4)
        self.assertEqual(1, len(polls))
        # Pass time, trigger a metric run.
        self.clock.advance(1)
        self.assertEqual(2, len(polls))

    @inlineCallbacks
    def test_find_accounts(self):
        worker = yield self.get_metrics_worker()
        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=[], acc2=[], acc3=[])
        yield worker.redis.sadd('disabled_metrics_accounts',
                                accounts["acc3"].key)

        account_keys = yield worker.find_account_keys()
        self.assertEqual(
            sorted([accounts["acc1"].key, accounts["acc2"].key]),
            sorted(account_keys))

    @inlineCallbacks
    def test_find_conversations_for_account(self):
        worker = yield self.get_metrics_worker()
        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=["conv1", "conv2", "conv3"])

        yield self.archive_conv(conversations["conv1"])
        yield self.start_conv(conversations["conv2"])

        active_conversations = yield worker.find_conversations_for_account(
            accounts["acc1"].key)
        self.assertEqual([c.key for c in active_conversations],
                         [conversations["conv2"].key])

    @inlineCallbacks
    def test_send_metrics_command(self):
        worker = yield self.get_metrics_worker()
        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=["conv1"])

        yield self.start_conv(conversations["conv1"])
        yield worker.send_metrics_command(conversations["conv1"])
        [cmd] = self._get_dispatched('vumi.api')

        self.assertEqual(cmd['worker_name'], 'my_conv_application')
        self.assertEqual(cmd.payload['kwargs']['conversation_key'],
                         conversations["conv1"].key)
        self.assertEqual(cmd.payload['kwargs']['user_account_key'],
                         accounts["acc1"].key)

    @inlineCallbacks
    def setup_metric_loop_conversations(self, worker):
        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=["conv0", "conv1"], acc2=["conv2", "conv3"])

        yield self.start_conv(conversations["conv0"])
        yield self.start_conv(conversations["conv1"])
        yield self.start_conv(conversations["conv2"])
        yield self.start_conv(conversations["conv3"])

        returnValue(conversations)

    @inlineCallbacks
    def test_metrics_loop_func_bucket_zero(self):
        worker = yield self.get_metrics_worker()
        conversations = yield self.setup_metric_loop_conversations(worker)

        self.assertEqual(worker._current_bucket, 0)
        yield worker.metrics_loop_func()
        self.assertEqual(worker._current_bucket, 1)

        cmds = self._get_dispatched('vumi.api')
        conv_keys = [c.payload['kwargs']['conversation_key'] for c in cmds]
        self.assertEqual(conv_keys, [conversations["conv0"].key])

        self.assert_conversations_bucketed(worker, {
            1: [conversations["conv1"]],
            2: [conversations["conv2"]],
            3: [conversations["conv3"]],
        })

    @inlineCallbacks
    def test_metrics_loop_func_bucket_nonzero(self):
        worker = yield self.get_metrics_worker()
        conversations = yield self.setup_metric_loop_conversations(worker)
        yield worker.populate_conversation_buckets()

        worker._current_bucket = 1
        yield worker.metrics_loop_func()
        self.assertEqual(worker._current_bucket, 2)

        cmds = self._get_dispatched('vumi.api')
        conv_keys = [c.payload['kwargs']['conversation_key'] for c in cmds]
        self.assertEqual(conv_keys, [conversations["conv1"].key])

        self.assert_conversations_bucketed(worker, {
            0: [conversations["conv0"]],
            2: [conversations["conv2"]],
            3: [conversations["conv3"]],
        })
<commit_msg>Fix tombstone key problem in metrics worker test. (@Hodgestar)<commit_after>"""Tests for go.vumitools.metrics_worker."""

import copy
import re

from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet.task import Clock, LoopingCall

from go.vumitools.tests.utils import GoWorkerTestCase
from go.vumitools import metrics_worker

from vumi.tests.utils import LogCatcher

class GoMetricsWorkerTestCase(GoWorkerTestCase):
    worker_class = metrics_worker.GoMetricsWorker

    def setUp(self):
        super(GoMetricsWorkerTestCase, self).setUp()
        self.clock = Clock()
        self.patch(metrics_worker, 'LoopingCall', self.looping_call)

    def get_metrics_worker(self, config=None, start=True,
                           needs_looping=False, needs_hash=False):
        if not needs_looping:
            self.patch(metrics_worker, 'LoopingCall', self.no_looping)
        if not needs_hash:
            self.patch(metrics_worker.GoMetricsWorker,
                       'bucket_for_conversation',
                       self.dummy_bucket_for_conversation)
        if config is None:
            config = {}
        return self.get_worker(self.mk_config(config), start)

    def rkey(self, name):
        return name

    def looping_call(self, *args, **kwargs):
        looping_call = LoopingCall(*args, **kwargs)
        looping_call.clock = self.clock
        return looping_call

    def no_looping(self, *args, **kw):
        return self.looping_call(lambda: None)

    def dummy_bucket_for_conversation(self, conv):
        digits = re.sub('[^0-9]', '', conv.name) or '0'
        return int(digits) % 60

    def make_account(self, worker, username):
        return self.mk_user(worker.vumi_api, username)

    def make_conv(self, user_api, conv_name, conv_type=u'my_conv'):
        return user_api.new_conversation(conv_type, conv_name, u'', {})

    def start_conv(self, conv):
        conv.set_status_started()
        return conv.save()

    def archive_conv(self, conv):
        conv.set_status_stopped()
        conv.set_status_finished()
        return conv.save()

    @inlineCallbacks
    def make_accounts_and_conversations(self, worker, **account_definitions):
        accounts, conversations = {}, {}
        for username, convs in account_definitions.iteritems():
            account = yield self.make_account(worker, unicode(username))
            accounts[username] = account
            user_api = worker.vumi_api.get_user_api(account.key)
            for conv_name in convs:
                conv = yield self.make_conv(user_api, unicode(conv_name))
                conversations[conv_name] = conv

        returnValue((accounts, conversations))

    @inlineCallbacks
    def test_bucket_for_conversation(self):
        worker = yield self.get_metrics_worker(needs_hash=True)
        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=["conv1"])
        bucket = worker.bucket_for_conversation(conversations["conv1"])
        self.assertEqual(bucket, hash(conversations["conv1"].key) % 60)

    def assert_conversations_bucketed(self, worker, expected):
        expected = copy.deepcopy(expected)
        buckets = copy.deepcopy(worker._buckets)
        for key in range(60):
            buckets[key] = sorted(c.key for c in buckets[key])
            expected[key] = sorted(c.key for c in expected.get(key, []))
            if buckets[key] == expected[key]:
                del buckets[key]
                del expected[key]
        self.assertEqual(buckets, expected)

    @inlineCallbacks
    def test_populate_conversation_buckets(self):
        worker = yield self.get_metrics_worker()

        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=["conv1", "conv2a", "conv2b", "conv4"])
        for conv in conversations.values():
            self.start_conv(conv)

        self.assert_conversations_bucketed(worker, {})
        with LogCatcher(message='Scheduled') as lc:
            yield worker.populate_conversation_buckets()
            [log_msg] = lc.messages()
        self.assert_conversations_bucketed(worker, {
            1: [conversations["conv1"]],
            2: [conversations["conv2a"], conversations["conv2b"]],
            4: [conversations["conv4"]],
        })
        # We may have tombstone keys from accounts created (and deleted) by
        # previous tests, so we replace the account count in the log message
        # we're asserting on.
        log_msg = re.sub(r'in \d account', 'in 1 account', log_msg)
        self.assertEqual(log_msg, "Scheduled metrics commands for"
                         " 4 conversations in 1 accounts.")

    @inlineCallbacks
    def test_process_bucket(self):
        worker = yield self.get_metrics_worker()

        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=["conv1", "conv2a", "conv2b", "conv4"])
        for conv in conversations.values():
            self.start_conv(conv)

        self.assert_conversations_bucketed(worker, {})
        yield worker.populate_conversation_buckets()
        yield worker.process_bucket(2)
        self.assert_conversations_bucketed(worker, {
            1: [conversations["conv1"]],
            4: [conversations["conv4"]],
        })

    @inlineCallbacks
    def test_increment_bucket(self):
        worker = yield self.get_metrics_worker()
        self.assertEqual(worker._current_bucket, 0)
        worker.increment_bucket()
        self.assertEqual(worker._current_bucket, 1)
        worker._current_bucket = 59
        worker.increment_bucket()
        self.assertEqual(worker._current_bucket, 0)

    @inlineCallbacks
    def test_metrics_poller(self):
        polls = []
        # Replace metrics worker with one that hasn't started yet.
        worker = yield self.get_metrics_worker(start=False, needs_looping=True)
        worker.metrics_loop_func = lambda: polls.append(None)
        self.assertEqual(0, len(polls))
        # Start worker.
        yield worker.startWorker()
        self.assertEqual(1, len(polls))
        # Pass time, but not enough to trigger a metric run.
        self.clock.advance(4)
        self.assertEqual(1, len(polls))
        # Pass time, trigger a metric run.
        self.clock.advance(1)
        self.assertEqual(2, len(polls))

    @inlineCallbacks
    def test_find_accounts(self):
        worker = yield self.get_metrics_worker()
        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=[], acc2=[], acc3=[])
        yield worker.redis.sadd('disabled_metrics_accounts',
                                accounts["acc3"].key)

        account_keys = yield worker.find_account_keys()
        self.assertEqual(
            sorted([accounts["acc1"].key, accounts["acc2"].key]),
            sorted(account_keys))

    @inlineCallbacks
    def test_find_conversations_for_account(self):
        worker = yield self.get_metrics_worker()
        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=["conv1", "conv2", "conv3"])

        yield self.archive_conv(conversations["conv1"])
        yield self.start_conv(conversations["conv2"])

        active_conversations = yield worker.find_conversations_for_account(
            accounts["acc1"].key)
        self.assertEqual([c.key for c in active_conversations],
                         [conversations["conv2"].key])

    @inlineCallbacks
    def test_send_metrics_command(self):
        worker = yield self.get_metrics_worker()
        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=["conv1"])

        yield self.start_conv(conversations["conv1"])
        yield worker.send_metrics_command(conversations["conv1"])
        [cmd] = self._get_dispatched('vumi.api')

        self.assertEqual(cmd['worker_name'], 'my_conv_application')
        self.assertEqual(cmd.payload['kwargs']['conversation_key'],
                         conversations["conv1"].key)
        self.assertEqual(cmd.payload['kwargs']['user_account_key'],
                         accounts["acc1"].key)

    @inlineCallbacks
    def setup_metric_loop_conversations(self, worker):
        accounts, conversations = yield self.make_accounts_and_conversations(
            worker, acc1=["conv0", "conv1"], acc2=["conv2", "conv3"])

        yield self.start_conv(conversations["conv0"])
        yield self.start_conv(conversations["conv1"])
        yield self.start_conv(conversations["conv2"])
        yield self.start_conv(conversations["conv3"])

        returnValue(conversations)

    @inlineCallbacks
    def test_metrics_loop_func_bucket_zero(self):
        worker = yield self.get_metrics_worker()
        conversations = yield self.setup_metric_loop_conversations(worker)

        self.assertEqual(worker._current_bucket, 0)
        yield worker.metrics_loop_func()
        self.assertEqual(worker._current_bucket, 1)

        cmds = self._get_dispatched('vumi.api')
        conv_keys = [c.payload['kwargs']['conversation_key'] for c in cmds]
        self.assertEqual(conv_keys, [conversations["conv0"].key])

        self.assert_conversations_bucketed(worker, {
            1: [conversations["conv1"]],
            2: [conversations["conv2"]],
            3: [conversations["conv3"]],
        })

    @inlineCallbacks
    def test_metrics_loop_func_bucket_nonzero(self):
        worker = yield self.get_metrics_worker()
        conversations = yield self.setup_metric_loop_conversations(worker)
        yield worker.populate_conversation_buckets()

        worker._current_bucket = 1
        yield worker.metrics_loop_func()
        self.assertEqual(worker._current_bucket, 2)

        cmds = self._get_dispatched('vumi.api')
        conv_keys = [c.payload['kwargs']['conversation_key'] for c in cmds]
        self.assertEqual(conv_keys, [conversations["conv1"].key])

        self.assert_conversations_bucketed(worker, {
            0: [conversations["conv0"]],
            2: [conversations["conv2"]],
            3: [conversations["conv3"]],
        })
<|endoftext|>