<commit_before># construct groups of connected images.  The inclusion order favors
# images with the most connections (features matches) to neighbors.

import json
import os
import sys

def countFeatureConnections(image_list, matches):
    for image in image_list:
        image.connection_set = set()
    for i, match in enumerate(matches):
        for mi in match[1:]:
            for mj in match[1:]:
                if mi != mj:
                    image_list[mi[0]].connection_set.add(mj[0])
    for i, image in enumerate(image_list):
        # print image.name
        # for j in image.connection_set:
        #     print '  pair len', j, len(image.match_list[j])
        image.connection_count = len(image.connection_set)
        # print image.name, i, image.connection_set
        for j in range(len(image.match_list)):
            size = len(image.match_list[j])
            if size > 0 and not j in image.connection_set:
                print '  matches, but no connection'

def updatePlacedFeatures(placed_images, matches, placed_features):
    for i, match in enumerate(matches):
        for m in match[1:]:
            if m[0] in placed_images:
                placed_features[i] = True

# This is the current, best grouping function to use

def simpleGrouping(image_list, matches):
    countFeatureConnections(image_list, matches)
    print "Start of top level grouping algorithm..."

    # start with no placed images or features
    placed_images = set()
    groups = []
    placed_features = [False] * len(matches)

    # wipe connection order for all images
    for image in image_list:
        image.connection_order = -1

    done = False
    while not done:
        print "Start of new group..."
        # start a fresh group
        group_images = set()

        # find the unplaced image with the most connections to other
        # images
        max_connections = 0
        max_index = -1
        for i, image in enumerate(image_list):
            if image.connection_order < 0 and image.connection_count > max_connections:
                max_connections = image.connection_count
                max_index = i
        max_image = image_list[max_index]
        max_image.connection_order = 0
        print "Image with max connections:", max_image.name
        print "Number of connections:", max_connections
        placed_images.add(max_index)
        group_images.add(max_index)
        updatePlacedFeatures(placed_images, matches, placed_features)

        while True:
            # find the unplaced image with the most connections into
            # the placed set

            # per image counter
            image_counter = [0] * len(image_list)

            # count up the placed feature references to unplaced images
            for i, match in enumerate(matches):
                # only proceed if this feature has been placed (i.e. it
                # connects to two or more placed images)
                if placed_features[i]:
                    for m in match[1:]:
                        if not m[0] in placed_images:
                            image_counter[m[0]] += 1
            # print 'connected image count:', image_counter
            new_index = -1
            max_connections = -1
            for i in range(len(image_counter)):
                if image_counter[i] > max_connections:
                    new_index = i
                    max_connections = image_counter[i]
            if max_connections >= 25:
                # print "New image with max connections:", image_list[new_index].name
                # print "Number of connected features:", max_connections
                placed_images.add(new_index)
                group_images.add(new_index)
            else:
                if len(group_images) > 1:
                    groups.append(list(group_images))
                else:
                    done = True
                break

            updatePlacedFeatures(placed_images, matches, placed_features)

            new_image = image_list[new_index]
            new_image.connection_order = len(placed_images) - 1
            print 'Added:', new_image.name, 'groups:', len(groups)+1, 'in current group', len(group_images), 'total:', len(placed_images)

    # add all unplaced images in their own groups of 1
    for i, image in enumerate(image_list):
        if not i in placed_images:
            groups.append( [i] )

    print groups
    return groups

def save(path, groups):
    file = os.path.join(path, 'Groups.json')
    try:
        fd = open(file, 'w')
        json.dump(groups, fd, indent=4, sort_keys=True)
        fd.close()
    except:
        print file + ": error saving file:", str(sys.exc_info()[1])

def load(path):
    file = os.path.join(path, 'Groups.json')
    try:
        fd = open(file, 'r')
        groups = json.load(fd)
        fd.close()
    except:
        print file + ": error loading file:", str(sys.exc_info()[1])
        groups = []
    return groups

##
## For historical/archival purposes, here is an alternative grouping
## function that uses number of neighbor with matches instead of
## number of total number of matches.  The results should be
## identical, but the order images are added to the group may be
## slightly different...
##

# return the neighbor that is closest to the root node of the
# placement tree (i.e. smallest cycle_depth.
def bestNeighbor(image, image_list):
    best_cycle_depth = len(image_list) + 1
    best_index = None
    for i, pairs in enumerate(image.match_list):
        if len(pairs):
            i2 = image_list[i]
            dist = i2.cycle_depth
            # print "  neighbor check %d = %d" % ( i, dist )
            if dist >= 0 and dist < best_cycle_depth:
                # print '    new best:', dist
                best_cycle_depth = dist
                best_index = i
    return best_index, best_cycle_depth

def groupByConnections(image_list):
    # reset the cycle distance for all images
    for image in image_list:
        image.cycle_depth = -1

    for image in image_list:
        image.connections = 0
        for match in image.match_list:
            if len(match) >= 8:
                image.connections += 1
        if image.connections > 1:
            print "%s connections: %d" % (image.name, image.connections)

    last_cycle_depth = len(image_list) + 1
    group_list = []
    group = []
    done = False
    while not done:
        done = True
        best_index = None
        # find an unplaced image with a placed neighbor that is the
        # closest conection to the root of the placement tree.
        best_cycle_depth = len(image_list) + 1
        for i, image in enumerate(image_list):
            if image.cycle_depth < 0:
                index, cycle_depth = bestNeighbor(image, image_list)
                if cycle_depth >= 0 and (cycle_depth+1 < best_cycle_depth):
                    best_index = i
                    best_cycle_depth = cycle_depth+1
                    done = False
        if best_index == None:
            print "Cannot find an unplaced image with a connected neighbor"
            if len(group):
                # commit the previous group (if it exists)
                group_list.append(group)
                # and start a new group
                group = []
                best_cycle_depth = last_cycle_depth + 1
            else:
                best_cycle_depth = 0
            # now find an unplaced image that has the most connections
            # to other images (new cycle start)
            max_connections = None
            for i, image in enumerate(image_list):
                if image.cycle_depth < 0:
                    if (max_connections == None or image.connections > max_connections):
                        max_connections = image.connections
                        best_index = i
                        done = False
                        print " found image %d connections = %d" % (i, max_connections)
        if best_index != None:
            image = image_list[best_index]
            image.cycle_depth = best_cycle_depth
            last_cycle_depth = best_cycle_depth
            print "Adding %s (cycles = %d)" % (image.name, best_cycle_depth)
            group.append(image)

    print "Group (cycles) report:"
    for group in group_list:
        #if len(group) < 2:
        #    continue
        print "group (size=%d):" % (len(group)),
        for image in group:
            print "%s(%d)" % (image.name, image.cycle_depth),
        print ""

    return group_list

<commit_msg>Experiment with computing the connected area of an image (in pixels).<commit_after># construct groups of connected images.  The inclusion order favors
# images with the most connections (features matches) to neighbors.

import cv2
import json
import numpy as np
import os
import sys

# this builds a simple set structure that records if any image has any
# connection to any other image
def countFeatureConnections(image_list, matches):
    for image in image_list:
        image.connection_set = set()
    for i, match in enumerate(matches):
        for mi in match[1:]:
            for mj in match[1:]:
                if mi != mj:
                    image_list[mi[0]].connection_set.add(mj[0])
    for i, image in enumerate(image_list):
        # print image.name
        # for j in image.connection_set:
        #     print '  pair len', j, len(image.match_list[j])
        image.connection_count = len(image.connection_set)
        # print image.name, i, image.connection_set
        for j in range(len(image.match_list)):
            size = len(image.match_list[j])
            if size > 0 and not j in image.connection_set:
                print '  matches, but no connection'

def updatePlacedFeatures(placed_images, matches, placed_features):
    for i, match in enumerate(matches):
        for m in match[1:]:
            if m[0] in placed_images:
                placed_features[i] = True

# This is the current, best grouping function to use
def groupByFeatureConnections(image_list, matches):
    countFeatureConnections(image_list, matches)
    print "Start of top level grouping algorithm..."

    # start with no placed images or features
    placed_images = set()
    groups = []
    placed_features = [False] * len(matches)

    # wipe connection order for all images
    for image in image_list:
        image.connection_order = -1

    done = False
    while not done:
        print "Start of new group..."
        # start a fresh group
        group_images = set()

        # find the unplaced image with the most connections to other
        # images
        max_connections = 0
        max_index = -1
        for i, image in enumerate(image_list):
            if image.connection_order < 0 and image.connection_count > max_connections:
                max_connections = image.connection_count
                max_index = i
        max_image = image_list[max_index]
        max_image.connection_order = 0
        print "Image with max connections:", max_image.name, "num:", max_connections
        placed_images.add(max_index)
        group_images.add(max_index)
        updatePlacedFeatures(placed_images, matches, placed_features)

        while True:
            # find the unplaced image with the most connections into
            # the placed set

            # per image counter
            image_counter = [0] * len(image_list)

            # count up the placed feature references to unplaced images
            for i, match in enumerate(matches):
                # only proceed if this feature has been placed (i.e. it
                # connects to two or more placed images)
                if placed_features[i]:
                    for m in match[1:]:
                        if not m[0] in placed_images:
                            image_counter[m[0]] += 1
            # print 'connected image count:', image_counter
            new_index = -1
            max_connections = -1
            for i in range(len(image_counter)):
                if image_counter[i] > max_connections:
                    new_index = i
                    max_connections = image_counter[i]
            if max_connections >= 25:
                # print "New image with max connections:", image_list[new_index].name
                # print "Number of connected features:", max_connections
                placed_images.add(new_index)
                group_images.add(new_index)
            else:
                if len(group_images) > 1:
                    groups.append(list(group_images))
                else:
                    done = True
                break

            updatePlacedFeatures(placed_images, matches, placed_features)

            new_image = image_list[new_index]
            new_image.connection_order = len(placed_images) - 1
            print 'Added:', new_image.name, 'groups:', len(groups)+1, 'in current group', len(group_images), 'total:', len(placed_images)

    # add all unplaced images in their own groups of 1
    for i, image in enumerate(image_list):
        if not i in placed_images:
            groups.append( [i] )

    print groups
    return groups

# for the specified image estimate the image area covered by
# connections to placed images.
def estimateConnectionArea(image):
    pass

# speculative ....
def groupByConnectedArea(image_list, matches):
    countFeatureConnections(image_list, matches)
    print "Start of top level grouping algorithm..."

    # start with no placed images or features
    placed_images = set()
    groups = []
    placed_features = [False] * len(matches)

    # wipe connection order for all images
    for image in image_list:
        image.connection_order = -1

    done = False
    while not done:
        print "Start of new group..."
        # start a fresh group
        group_images = set()

        # find the unplaced image with the most connections to other
        # images
        max_connections = 0
        max_index = -1
        for i, image in enumerate(image_list):
            if image.connection_order < 0 and image.connection_count > max_connections:
                max_connections = image.connection_count
                max_index = i
        max_image = image_list[max_index]
        max_image.connection_order = 0
        print "Image with max connections:", max_image.name, "num:", max_connections
        placed_images.add(max_index)
        group_images.add(max_index)
        updatePlacedFeatures(placed_images, matches, placed_features)

        while True:
            # find the unplaced image with the largest connection area
            # into the placed set

            # per image counter
            # image_counter = [0] * len(image_list)

            # clear the placed feature lists
            for i, image in enumerate(image_list):
                image.placed_feature_list = []

            # assemble the placed feature lists for each unplaced image
            for i, match in enumerate(matches):
                # only proceed if this feature has been placed (i.e. it
                # connects to two or more placed images)
                if placed_features[i]:
                    for m in match[1:]:
                        if not m[0] in placed_images:
                            uv = image_list[m[0]].uv_list[m[1]]
                            image_list[m[0]].placed_feature_list.append(uv)

            # find the minarearect bounds for each the connected
            # points in each image
            for image in image_list:
                if len(image.placed_feature_list):
                    center, (w, h), angle = cv2.minAreaRect(np.array(image.placed_feature_list))
                    print w, h, w*h
                    image.connected_area = w*h

            # print 'connected image count:', image_counter
            new_index = -1
            max_area = -1
            for i, image in enumerate(image_list):
                if len(image.placed_feature_list):
                    if image.connected_area > max_area:
                        new_index = i
                        max_area = image.connected_area
            if max_area >= 10000:
                print "New image with max area:", image_list[new_index].name,
                print "area:", image_list[new_index].connected_area
                placed_images.add(new_index)
                group_images.add(new_index)
            else:
                if len(group_images) > 1:
                    groups.append(list(group_images))
                else:
                    done = True
                break

            updatePlacedFeatures(placed_images, matches, placed_features)

            new_image = image_list[new_index]
            new_image.connection_order = len(placed_images) - 1
            print 'Added:', new_image.name, 'groups:', len(groups)+1, 'in current group', len(group_images), 'total:', len(placed_images)

    # add all unplaced images in their own groups of 1
    for i, image in enumerate(image_list):
        if not i in placed_images:
            groups.append( [i] )

    print groups
    return groups

def save(path, groups):
    file = os.path.join(path, 'Groups.json')
    try:
        fd = open(file, 'w')
        json.dump(groups, fd, indent=4, sort_keys=True)
        fd.close()
    except:
        print file + ": error saving file:", str(sys.exc_info()[1])

def load(path):
    file = os.path.join(path, 'Groups.json')
    try:
        fd = open(file, 'r')
        groups = json.load(fd)
        fd.close()
    except:
        print file + ": error loading file:", str(sys.exc_info()[1])
        groups = []
    return groups

##
## For historical/archival purposes, here is an alternative grouping
## function that uses number of neighbor with matches instead of
## number of total number of matches.  The results should be
## identical, but the order images are added to the group may be
## slightly different...
##

# return the neighbor that is closest to the root node of the
# placement tree (i.e. smallest cycle_depth.
def bestNeighbor(image, image_list):
    best_cycle_depth = len(image_list) + 1
    best_index = None
    for i, pairs in enumerate(image.match_list):
        if len(pairs):
            i2 = image_list[i]
            dist = i2.cycle_depth
            # print "  neighbor check %d = %d" % ( i, dist )
            if dist >= 0 and dist < best_cycle_depth:
                # print '    new best:', dist
                best_cycle_depth = dist
                best_index = i
    return best_index, best_cycle_depth

def groupByImageConnections(image_list):
    # reset the cycle distance for all images
    for image in image_list:
        image.cycle_depth = -1

    for image in image_list:
        image.connections = 0
        for match in image.match_list:
            if len(match) >= 8:
                image.connections += 1
        if image.connections > 1:
            print "%s connections: %d" % (image.name, image.connections)

    last_cycle_depth = len(image_list) + 1
    group_list = []
    group = []
    done = False
    while not done:
        done = True
        best_index = None
        # find an unplaced image with a placed neighbor that is the
        # closest conection to the root of the placement tree.
        best_cycle_depth = len(image_list) + 1
        for i, image in enumerate(image_list):
            if image.cycle_depth < 0:
                index, cycle_depth = bestNeighbor(image, image_list)
                if cycle_depth >= 0 and (cycle_depth+1 < best_cycle_depth):
                    best_index = i
                    best_cycle_depth = cycle_depth+1
                    done = False
        if best_index == None:
            print "Cannot find an unplaced image with a connected neighbor"
            if len(group):
                # commit the previous group (if it exists)
                group_list.append(group)
                # and start a new group
                group = []
                best_cycle_depth = last_cycle_depth + 1
            else:
                best_cycle_depth = 0
            # now find an unplaced image that has the most connections
            # to other images (new cycle start)
            max_connections = None
            for i, image in enumerate(image_list):
                if image.cycle_depth < 0:
                    if (max_connections == None or image.connections > max_connections):
                        max_connections = image.connections
                        best_index = i
                        done = False
                        print " found image %d connections = %d" % (i, max_connections)
        if best_index != None:
            image = image_list[best_index]
            image.cycle_depth = best_cycle_depth
            last_cycle_depth = best_cycle_depth
            print "Adding %s (cycles = %d)" % (image.name, best_cycle_depth)
            group.append(image)

    print "Group (cycles) report:"
    for group in group_list:
        #if len(group) < 2:
        #    continue
        print "group (size=%d):" % (len(group)),
        for image in group:
            print "%s(%d)" % (image.name, image.cycle_depth),
        print ""

    return group_list

<|endoftext|>


<commit_before>
import os
import json
import re

### util

def read_json(path):
  with open(path,'r') as f:
    return json.load(f)

def write_json(path, value):
  with open(path,"w") as f:
    json.dump(value, f, sort_keys=True, indent=4)

def group_map(l,f):
  res = {}
  for it in l:
    k,v = f(it)
    if k not in res: res[k] = []
    res[k].append(v)
  return res

def one(it): return it

def ext(f): return lambda arg: f(*arg)

###

def docker_conf(): return "python3 $C4CI_PROTO_DIR/gitlab-docker-conf.py"
def prod(arg):
  return [docker_conf(),f"ssh-agent perl $C4CI_PROTO_DIR/prod.pl {arg}"]
def handle(arg):
  return f"python3 $C4CI_PROTO_DIR/gitlab-ci.py {arg}"
def push_rule(cond):
  return { "if": f"$CI_PIPELINE_SOURCE == \"push\" && {cond}" }
def common_job(cond,when,stage,script):
  return {
    "rules": [{ **push_rule(cond), "when": when }],
    "image": "$C4COMMON_IMAGE", "variables": {"GIT_STRATEGY": "none" },
    "stage": stage, "script": script
  }
def build_path(fn):
  dir = os.environ["C4CI_BUILD_DIR"]
  return f"{dir}/{fn}"
def prefix_cond(v): return f"=~ /^{v}\\//"
def get_aggr_cond(aggr_cond_list):
  aggr_to_cond_list = group_map(aggr_cond_list, ext(lambda aggr, cond: (aggr,cond)))
  aggr_to_cond = { aggr: one(*cond_list) for aggr, cond_list in aggr_to_cond_list.items() }
  return (aggr_cond_list, aggr_to_cond)

build_common_name = "build common"
build_gate_name = "build gate"
build_frp_name = "build frp"
def build_aggr_name(v): return f"{v}.aggr"
def build_rt_name(v): return f"b {v}.rt"
stage_build_rt = "develop"
stage_deploy_de = "develop"
stage_confirm = "confirm"
stage_deploy_sp = "confirm"
stage_deploy_cl = "deploy"

def get_build_jobs(config_statements):
  def build(cond,stage,arg):
    return common_job(f"$CI_COMMIT_BRANCH {cond}","on_success",stage,prod(arg))
  def build_main(cond,arg):
    return { **build(cond,"build_main",arg), "needs": [build_common_name] }
  tag_aggr_list = config_statements["C4TAG_AGGR"]
  (aggr_cond_list, aggr_to_cond) = get_aggr_cond(config_statements["C4AGGR_COND"])
  aggr_to_tags = group_map(tag_aggr_list, ext(lambda tag, aggr: (aggr,tag)))
  aggr_jobs = {
    build_aggr_name(aggr): build_main(
      prefix_cond(cond), f"ci_build_aggr {aggr} " + ":".join(aggr_to_tags[aggr])
    ) for aggr, cond in aggr_cond_list
  }
  fin_jobs = {
    build_rt_name(tag): build(
      prefix_cond(aggr_to_cond[aggr]), stage_build_rt, f"ci_build {tag} {aggr}"
    ) for tag, aggr in tag_aggr_list
  }
  return {
    build_gate_name: build_main("","ci_build def"),
    build_frp_name: build_main("","ci_build_frp"),
    **aggr_jobs, **fin_jobs
  }

def get_deploy_jobs(config_statements):
  tag_aggr_list = config_statements["C4TAG_AGGR"]
  (aggr_cond_list, aggr_to_cond) = get_aggr_cond(config_statements["C4AGGR_COND"])
  cond_to_aggr = group_map(aggr_cond_list, ext(lambda aggr, cond: (cond,aggr)))
  cond_to_tags = group_map(tag_aggr_list, ext(lambda tag, aggr: (aggr_to_cond[aggr], tag)))
  cond_list = sorted(set(cond for aggr, cond in aggr_cond_list))
  def needs_de(cond): return [build_gate_name,build_frp_name] + ([] if cond == "nil" else [build_aggr_name(aggr) for aggr in cond_to_aggr[cond]])
  def needs_rt(cond): return [build_gate_name] + [build_rt_name(tag) for tag in cond_to_tags[cond]]
  return {
    key: value
    for env_mask, caption_mask in config_statements["C4DEPLOY"]
    for mode_mask, arg, proj_mask, opt in [re.findall(r'[^\-]+',env_mask)]
    for proj_name in (cond_list if proj_mask == "$C4PROJ" else [proj_mask])
    for cond_re in ["" if proj_name == "nil" else prefix_cond(f"{proj_name}\/release" if arg == "prod" else proj_name)]
    for cond in [f"$CI_COMMIT_BRANCH {cond_re}"]
    for mode, stage, needs_fun in (
      [(mode_mask,stage_deploy_de,needs_de)] if mode_mask == "de" else
      [(mode_mask,stage_deploy_cl,needs_rt)] if mode_mask == "cl" else
      [(mode_mask,stage_deploy_sp,needs_rt)]
    )
    for key_mask in [caption_mask.replace("$C4PROJ",proj_name)]
    for script in [[
      "export C4SUBJ=$(perl -e 's{[^\w/]}{}g,/(\w+)$/&&print$1 for $ENV{CI_COMMIT_BRANCH}')",
      "export C4USER=$(perl -e 's{[^\w/]}{}g,/(\w+)$/&&print$1 for $ENV{GITLAB_USER_LOGIN}')",
      "env | grep C4 | sort",
      handle(f"deploy {mode}-{arg}-{proj_name}-{opt} $CI_COMMIT_BRANCH")
    ]]
    for confirm_key in [key_mask.replace("$C4CONFIRM","confirm")]
    for key, value in (
      [
        (confirm_key, { **common_job(cond,"manual",stage_confirm,["echo confirming"]), "needs": [] }),
        (key_mask.replace("$C4CONFIRM","deploy"), { **common_job(cond,"manual",stage,script), "needs": [confirm_key] + needs_fun(proj_name) }),
      ] if confirm_key != key_mask else [
        (key_mask, { **common_job(cond,"manual",stage,script), "needs": needs_fun(proj_name) }),
      ]
    )
  }

def get_env_jobs():
  cond_qa = "$CI_COMMIT_TAG =~ /\\/qa-/"
  def stop(cond,when,needs): return {
    **common_job(cond,when,"stop",[handle("down")]), "needs": needs,
    "environment": { "name": "$CI_COMMIT_TAG", "action": "stop" }
  }
  start_name = "start"
  testing_name = "testing"
  return {
    start_name: {
      **common_job("$CI_COMMIT_TAG","on_success","start",[docker_conf(),handle("up $CI_ENVIRONMENT_SLUG")]),
      "environment": { "name": "$CI_COMMIT_TAG", "action": "start", "on_stop": "stop" }
    },
    testing_name: common_job(cond_qa,"on_success","testing",[handle("qa_run /c4/qa")]),
    "check": common_job("$CI_COMMIT_TAG","on_success","check",[handle("check")]),
    "stop": stop("$CI_COMMIT_TAG","manual",[start_name]),
    "auto-stop": stop(cond_qa,"on_success",[testing_name]),
    "forward": common_job("$CI_COMMIT_TAG =~ /\\/de-/","manual","start",[
      handle("deploy fc-$(perl -e '/\\bde-(\w+-\w+-\w+)/&&print$1 for $ENV{CI_COMMIT_TAG}') no-branch")
    ])
  }

def main():
  config_statements = group_map(read_json(build_path("c4dep.main.json")), lambda it: (it[0],it[1:]))
  out = {
    "variables": { "C4CI_DOCKER_CONFIG": "/tmp/c4-docker-config" },
    "stages": ["build_replink","build_common","build_main","develop","confirm","deploy","start","check","testing","stop"],
    "build_replink": {
      "image": {
        "name": "gcr.io/kaniko-project/executor:debug",
        "entrypoint": [ "" ]
      },
      "rules": [{"if":"$C4CI_REPLINK"}],
      "stage": "build_replink",
      "variables": {
        "C4DOCKER_CONF": '{"auths":{"$CI_REGISTRY":{"username":"$CI_REGISTRY_USER","password":"$CI_REGISTRY_PASSWORD"}}}'
      },
      "script": [
        "mkdir -p /kaniko/.docker",
        "echo $C4DOCKER_CONF > /kaniko/.docker/config.json",
        "cat /kaniko/.docker/config.json",
        "/kaniko/executor --context $CI_PROJECT_DIR/replink_extra --dockerfile $CI_PROJECT_DIR/replink_extra/Dockerfile --destination $CI_REGISTRY_IMAGE/replink:v2sshk3"
      ]
    },
    build_common_name: {
      "rules": [push_rule("$CI_COMMIT_BRANCH")],
      "stage": "build_common",
      "image": "$CI_REGISTRY_IMAGE/replink:v2sshk3",
      "script": [
        "export C4CI_BUILD_DIR=$CI_PROJECT_DIR",
        "export C4CI_PROTO_DIR=$C4COMMON_PROTO_DIR",
        "C4REPO_MAIN_CONF=$CI_PROJECT_DIR/c4dep.ci.replink /replink.pl",
        "C4CI_BUILD_DIR=$C4CI_PROTO_DIR C4REPO_MAIN_CONF=$C4CI_PROTO_DIR/c4dep.main.replink /replink.pl",
        *prod("ci_build_common"),
      ],
    },
    **get_build_jobs(config_statements), **get_deploy_jobs(config_statements), **get_env_jobs()
  }
  write_json(build_path("gitlab-ci-generated.yml"), out)

main()

#docker run --rm -v $PWD:/build_dir -e C4CI_BUILD_DIR=/build_dir python:3.8 python3 /build_dir/c4proto/gitlab-gen.py<commit_msg>gitlab fix<commit_after>
import os
import json
import re

### util

def read_json(path):
  with open(path,'r') as f:
    return json.load(f)

def write_json(path, value):
  with open(path,"w") as f:
    json.dump(value, f, sort_keys=True, indent=4)

def group_map(l,f):
  res = {}
  for it in l:
    k,v = f(it)
    if k not in res: res[k] = []
    res[k].append(v)
  return res

def one(it): return it

def ext(f): return lambda arg: f(*arg)

###

def docker_conf(): return "python3 $C4CI_PROTO_DIR/gitlab-docker-conf.py"
def prod(arg):
  return [docker_conf(),f"ssh-agent perl $C4CI_PROTO_DIR/prod.pl {arg}"]
def handle(arg):
  return f"python3 $C4CI_PROTO_DIR/gitlab-ci.py {arg}"
def push_rule(cond):
  return { "if": f"$CI_PIPELINE_SOURCE == \"push\" && {cond}" }
def common_job(cond,when,stage,needs,script):
  return {
    "rules": [{ **push_rule(cond), "when": when }],
    "image": "$C4COMMON_IMAGE", "variables": {"GIT_STRATEGY": "none" },
    "stage": stage, "needs": needs, "script": script
  }
def build_path(fn):
  dir = os.environ["C4CI_BUILD_DIR"]
  return f"{dir}/{fn}"
def prefix_cond(v): return f"=~ /^{v}\\//"
def get_aggr_cond(aggr_cond_list):
  aggr_to_cond_list = group_map(aggr_cond_list, ext(lambda aggr, cond: (aggr,cond)))
  aggr_to_cond = { aggr: one(*cond_list) for aggr, cond_list in aggr_to_cond_list.items() }
  return (aggr_cond_list, aggr_to_cond)

build_common_name = "build common"
build_gate_name = "build gate"
build_frp_name = "build frp"
def build_aggr_name(v): return f"{v}.aggr"
def build_rt_name(v): return f"b {v}.rt"
stage_build_rt = "develop"
stage_deploy_de = "develop"
stage_confirm = "confirm"
stage_deploy_sp = "confirm"
stage_deploy_cl = "deploy"

def get_build_jobs(config_statements):
  def build(cond,stage,needs,arg):
    return common_job(f"$CI_COMMIT_BRANCH {cond}","on_success",stage,needs,prod(arg))
  def build_main(cond,arg):
    return build(cond,"build_main",[build_common_name],arg)
  tag_aggr_list = config_statements["C4TAG_AGGR"]
  (aggr_cond_list, aggr_to_cond) = get_aggr_cond(config_statements["C4AGGR_COND"])
  aggr_to_tags = group_map(tag_aggr_list, ext(lambda tag, aggr: (aggr,tag)))
  aggr_jobs = {
    build_aggr_name(aggr): build_main(
      prefix_cond(cond), f"ci_build_aggr {aggr} " + ":".join(aggr_to_tags[aggr])
    ) for aggr, cond in aggr_cond_list
  }
  fin_jobs = {
    build_rt_name(tag): build(
      prefix_cond(aggr_to_cond[aggr]), stage_build_rt, sorted(aggr_jobs.keys()),
      f"ci_build {tag} {aggr}"
    ) for tag, aggr in tag_aggr_list
  }
  return {
    build_gate_name: build_main("","ci_build def"),
    build_frp_name: build_main("","ci_build_frp"),
    **aggr_jobs, **fin_jobs
  }

def get_deploy_jobs(config_statements):
  tag_aggr_list = config_statements["C4TAG_AGGR"]
  (aggr_cond_list, aggr_to_cond) = get_aggr_cond(config_statements["C4AGGR_COND"])
  cond_to_aggr = group_map(aggr_cond_list, ext(lambda aggr, cond: (cond,aggr)))
  cond_to_tags = group_map(tag_aggr_list, ext(lambda tag, aggr: (aggr_to_cond[aggr], tag)))
  cond_list = sorted(set(cond for aggr, cond in aggr_cond_list))
  def needs_de(cond): return [build_gate_name,build_frp_name] + ([] if cond == "nil" else [build_aggr_name(aggr) for aggr in cond_to_aggr[cond]])
  def needs_rt(cond): return [build_gate_name] + [build_rt_name(tag) for tag in cond_to_tags[cond]]
  return {
    key: value
    for env_mask, caption_mask in config_statements["C4DEPLOY"]
    for mode_mask, arg, proj_mask, opt in [re.findall(r'[^\-]+',env_mask)]
    for proj_name in (cond_list if proj_mask == "$C4PROJ" else [proj_mask])
    for cond_re in ["" if proj_name == "nil" else prefix_cond(f"{proj_name}\/release" if arg == "prod" else proj_name)]
    for cond in [f"$CI_COMMIT_BRANCH {cond_re}"]
    for mode, stage, needs_fun in (
      [(mode_mask,stage_deploy_de,needs_de)] if mode_mask == "de" else
      [(mode_mask,stage_deploy_cl,needs_rt)] if mode_mask == "cl" else
      [(mode_mask,stage_deploy_sp,needs_rt)]
    )
    for key_mask in [caption_mask.replace("$C4PROJ",proj_name)]
    for script in [[
      "export C4SUBJ=$(perl -e 's{[^\w/]}{}g,/(\w+)$/&&print$1 for $ENV{CI_COMMIT_BRANCH}')",
      "export C4USER=$(perl -e 's{[^\w/]}{}g,/(\w+)$/&&print$1 for $ENV{GITLAB_USER_LOGIN}')",
      "env | grep C4 | sort",
      handle(f"deploy {mode}-{arg}-{proj_name}-{opt} $CI_COMMIT_BRANCH")
    ]]
    for confirm_key in [key_mask.replace("$C4CONFIRM","confirm")]
    for key, value in (
      [
        (confirm_key, common_job(cond,"manual",stage_confirm,[],["echo confirming"])),
        (key_mask.replace("$C4CONFIRM","deploy"), common_job(cond,"manual",stage,[confirm_key]+needs_fun(proj_name),script)),
      ] if confirm_key != key_mask else [
        (key_mask, common_job(cond,"manual",stage,needs_fun(proj_name),script)),
      ]
    )
  }

def get_env_jobs():
  cond_qa = "$CI_COMMIT_TAG =~ /\\/qa-/"
  def stop(cond,when,needs): return {
    **common_job(cond,when,"stop",needs,[handle("down")]),
    "environment": { "name": "$CI_COMMIT_TAG", "action": "stop" }
  }
  def forward(when): return common_job("$CI_COMMIT_TAG =~ /\\/de-/",when,"start",[],[
    handle("deploy fc-$(perl -e '/\\bde-(\w+-\w+-\w+)/&&print$1 for $ENV{CI_COMMIT_TAG}') no-branch")
  ])

  start_name = "start"
  check_name = "check"
  testing_name = "testing"
  return {
    start_name: {
      **common_job("$CI_COMMIT_TAG","on_success","start",[],[docker_conf(),handle("up $CI_ENVIRONMENT_SLUG")]),
      "environment": { "name": "$CI_COMMIT_TAG", "action": "start", "on_stop": "stop" }
    },
    check_name: common_job("$CI_COMMIT_TAG","on_success","check",[start_name],[handle("check")]),
    testing_name: common_job(cond_qa,"on_success","testing",[check_name],[handle("qa_run /c4/qa")]),
    "stop": stop("$CI_COMMIT_TAG","manual",[start_name]),
    "auto-stop": stop(cond_qa,"on_success",[testing_name]),
    "forward": forward("manual"),
    "auto-forward": forward("on_success"),
  }

def main():
  config_statements = group_map(read_json(build_path("c4dep.main.json")), lambda it: (it[0],it[1:]))
  out = {
    "variables": { "C4CI_DOCKER_CONFIG": "/tmp/c4-docker-config" },
    "stages": ["build_replink","build_common","build_main","develop","confirm","deploy","start","check","testing","stop"],
    "build_replink": {
      "image": {
        "name": "gcr.io/kaniko-project/executor:debug",
        "entrypoint": [ "" ]
      },
      "rules": [{"if":"$C4CI_REPLINK"}],
      "stage": "build_replink",
      "variables": {
        "C4DOCKER_CONF": '{"auths":{"$CI_REGISTRY":{"username":"$CI_REGISTRY_USER","password":"$CI_REGISTRY_PASSWORD"}}}'
      },
      "script": [
        "mkdir -p /kaniko/.docker",
        "echo $C4DOCKER_CONF > /kaniko/.docker/config.json",
        "cat /kaniko/.docker/config.json",
        "/kaniko/executor --context $CI_PROJECT_DIR/replink_extra --dockerfile $CI_PROJECT_DIR/replink_extra/Dockerfile --destination $CI_REGISTRY_IMAGE/replink:v2sshk3"
      ]
    },
    build_common_name: {
      "rules": [push_rule("$CI_COMMIT_BRANCH")],
      "stage": "build_common",
      "image": "$CI_REGISTRY_IMAGE/replink:v2sshk3",
      "script": [
        "export C4CI_BUILD_DIR=$CI_PROJECT_DIR",
        "export C4CI_PROTO_DIR=$C4COMMON_PROTO_DIR",
        "C4REPO_MAIN_CONF=$CI_PROJECT_DIR/c4dep.ci.replink /replink.pl",
        "C4CI_BUILD_DIR=$C4CI_PROTO_DIR C4REPO_MAIN_CONF=$C4CI_PROTO_DIR/c4dep.main.replink /replink.pl",
        *prod("ci_build_common"),
      ],
    },
    **get_build_jobs(config_statements), **get_deploy_jobs(config_statements), **get_env_jobs()
  }
  write_json(build_path("gitlab-ci-generated.yml"), out)

main()

#docker run --rm -v $PWD:/build_dir -e C4CI_BUILD_DIR=/build_dir python:3.8 python3 /build_dir/c4proto/gitlab-gen.py<|endoftext|>


<commit_before>from sh import git
from requests.auth import HTTPBasicAuth
from globalvars import GlobalVars
import requests
import time
import json

class GitManager:
    @classmethod
    def add_to_blacklist(self, items_to_blacklist, username, code_permissions):
        # Check if we're on master
        if git("rev-parse", "--abbrev-ref", "HEAD").strip() != "master":
            return (False, "Not currently on master.")

        # Check that we're up-to-date with origin (GitHub)
        git.remote.update()
        if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
            return (False, "HEAD isn't at tip of origin's master branch")

        # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
        if "blacklisted_websites.txt" in git.status():  # Also ugly
            return (False, "blacklisted_websites.txt modified locally. This is probably bad.")

        # Store current commit hash
        current_commit = git("rev-parse", "HEAD").strip()

        # Add items to file

        with open("blacklisted_websites.txt", "a") as blacklisted_websites:
            blacklisted_websites.write("\n" + "\n".join(items_to_blacklist))

        # Checkout a new branch (mostly unnecessary, but may help if we create PRs in the future
        branch = "auto-blacklist-%s" % str(time.time())
        git.checkout("-b", branch)

        # Clear HEAD just in case
        git.reset("HEAD")

        git.add("blacklisted_websites.txt")
        git.commit("-m", "Auto blacklist of %s by %s --autopull" % (", ".join(items_to_blacklist), username))

        if code_permissions:
            git.checkout("master")
            git.merge(branch)
            git.push()
        else:
            git.push("origin", branch)
            git.checkout("master")

            if GlobalVars.github_username is None or GlobalVars.github_password is None:
                return (False, "tell someone to set a GH password")

            payload = {"title": "%: Blacklist %" % (username, ", ".join(items_to_blacklist)),
                       "body": "% requests blacklist of domains: \n\n - %" % (username, "\n - ".join(items_to_blacklist)),
                       "head": branch,
                       "base": "master"}
            response = requests.post("https://api.github.com/repos/Undo1/AutoHubTest/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload))
            return (True, "You don't have code privileges, but I've [created a pull request for you](%s)." % response.json()["url"])

        git.checkout(current_commit)  # Return to old commit to await CI. This will make Smokey think it's in reverted mode if it restarts

        if not code_permissions:
            return (False, "Unable to perform action due to lack of code-level permissions. [Branch pushed](https://github.com/Charcoal-SE/SmokeDetector/tree/%s), PR at your leisure." % branch)

        return (True, "Blacklisted {0} - the entry will be applied via autopull if CI succeeds.".format(", ".join(items_to_blacklist)))
<commit_msg>Fix format character --autopull<commit_after>from sh import git
from requests.auth import HTTPBasicAuth
from globalvars import GlobalVars
import requests
import time
import json

class GitManager:
    @classmethod
    def add_to_blacklist(self, items_to_blacklist, username, code_permissions):
        # Check if we're on master
        if git("rev-parse", "--abbrev-ref", "HEAD").strip() != "master":
            return (False, "Not currently on master.")

        # Check that we're up-to-date with origin (GitHub)
        git.remote.update()
        if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
            return (False, "HEAD isn't at tip of origin's master branch")

        # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
        if "blacklisted_websites.txt" in git.status():  # Also ugly
            return (False, "blacklisted_websites.txt modified locally. This is probably bad.")

        # Store current commit hash
        current_commit = git("rev-parse", "HEAD").strip()

        # Add items to file

        with open("blacklisted_websites.txt", "a") as blacklisted_websites:
            blacklisted_websites.write("\n" + "\n".join(items_to_blacklist))

        # Checkout a new branch (mostly unnecessary, but may help if we create PRs in the future
        branch = "auto-blacklist-%s" % str(time.time())
        git.checkout("-b", branch)

        # Clear HEAD just in case
        git.reset("HEAD")

        git.add("blacklisted_websites.txt")
        git.commit("-m", "Auto blacklist of %s by %s --autopull" % (", ".join(items_to_blacklist), username))

        if code_permissions:
            git.checkout("master")
            git.merge(branch)
            git.push()
        else:
            git.push("origin", branch)
            git.checkout("master")

            if GlobalVars.github_username is None or GlobalVars.github_password is None:
                return (False, "tell someone to set a GH password")

            payload = {"title": "%s: Blacklist %s" % (username, ", ".join(items_to_blacklist)),
                       "body": "% requests blacklist of domains: \n\n - %" % (username, "\n - ".join(items_to_blacklist)),
                       "head": branch,
                       "base": "master"}
            response = requests.post("https://api.github.com/repos/Undo1/AutoHubTest/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload))
            return (True, "You don't have code privileges, but I've [created a pull request for you](%s)." % response.json()["url"])

        git.checkout(current_commit)  # Return to old commit to await CI. This will make Smokey think it's in reverted mode if it restarts

        if not code_permissions:
            return (False, "Unable to perform action due to lack of code-level permissions. [Branch pushed](https://github.com/Charcoal-SE/SmokeDetector/tree/%s), PR at your leisure." % branch)

        return (True, "Blacklisted {0} - the entry will be applied via autopull if CI succeeds.".format(", ".join(items_to_blacklist)))
<|endoftext|>


<commit_before>#! /usr/bin/env python2
import argparse
from collections import namedtuple
import os.path
import logging
import sys

from sh import git

logging.basicConfig(format='%(levelname)s: %(message)s')

def make_enum(name, values):
    return namedtuple(name, values)._make(values)

Action = make_enum('Action', ('fetch', 'commit', 'pull', 'push'))
TreeState = make_enum('TreeState', ('clean', 'dirty'))

def fail(msg):
    'Fail program with printed message'
    logging.error(msg)
    sys.exit(1)

def goto(repo_path):
    'find repo and change directory'
    repo = os.path.expanduser(repo)
    logging.debug('going to %s' % repo)
    if not os.path.isdir(repo):
        fail('No directory at %s!' % repo)
    if not os.path.isdir(os.path.join(repo, '.git')):
        fail('No git repo at %s!' % repo)
    os.chdir(repo)

def act(action):
    'perform given git action (with side effects)'
    assert action in Action
    if action == Action.fetch:
        git.fetch()
    else:
        raise NotImplementedError()

def get_tree_state():
    'parse git status and return True if tree is clean'
    for line in git.status(porcelain=True):
        if line.strip():
            return TreeState.dirty
    return TreeState.clean

def update(repo, actions):
    'Update repo according to allowed actions.'
    goto(repo)
    act(Action.fetch)

    tree_state = get_tree_state()
    if tree_state == TreeState.dirty:
        if Action.commit in actions:
            pass # TODO: do commit!
        else:
            logging.info('Skip repo with dirty tree:')
            for line in git.status(porcelain=True):
                logging.info(line.rstrip())
            return
    # TODO: handle pull, push

def parse(config):
    'Parse config and yield repos with actions'
    with open(config) as f:
        for line in f:
            items = line.strip().split(',')
            repo_path, actions = items[0], items[1:]
            for a in actions:
                if a not in Action:
                    raise ValueError('Unknown action: %s' % a)
            yield repo_path, actions

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Keep git repos up-to-date.')
    parser.add_argument('config', type=str, help='config file that lists repos')
    parser.add_argument('-v', '--verbose', default=False, action='store_true')
    args = parser.parse_args()

    if args.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    for repo, actions in parse(args.config):
        update(repo, actions)
<commit_msg>implement 'commit'<commit_after>#! /usr/bin/env python2
import argparse
from collections import namedtuple
import os.path
import logging
import sys

from sh import git

logging.basicConfig(format='%(levelname)s: %(message)s')

def make_enum(name, values):
    return namedtuple(name, values)._make(values)

Action = make_enum('Action', ('fetch', 'commit', 'pull', 'push'))
TreeState = make_enum('TreeState', ('clean', 'dirty'))

def fail(msg):
    'Fail program with printed message'
    logging.error(msg)
    sys.exit(1)

def goto(repo_path):
    'find repo and change directory'
    repo = os.path.expanduser(repo)
    logging.debug('going to %s' % repo)
    if not os.path.isdir(repo):
        fail('No directory at %s!' % repo)
    if not os.path.isdir(os.path.join(repo, '.git')):
        fail('No git repo at %s!' % repo)
    os.chdir(repo)

def act(action):
    'perform given git action (with side effects)'
    assert action in Action
    if action == Action.fetch:
        git.fetch()
    elif action == Action.commit:
        git.add('.', all=True)
        msg = 'auto-commit by gitthemall'
        git.commit(message=msg)
    else:
        raise NotImplementedError()

def get_tree_state():
    'parse git status and return True if tree is clean'
    for line in git.status(porcelain=True):
        if line.strip():
            return TreeState.dirty
    return TreeState.clean

def update(repo, actions):
    'Update repo according to allowed actions.'
    goto(repo)
    act(Action.fetch)

    tree_state = get_tree_state()
    if tree_state == TreeState.dirty:
        if Action.commit in actions:
            act(Action.commit)
        else:
            logging.info('Skip repo with dirty tree:')
            for line in git.status(porcelain=True):
                logging.info(line.rstrip())
            return
    # TODO: handle pull, push

def parse(config):
    'Parse config and yield repos with actions'
    with open(config) as f:
        for line in f:
            items = line.strip().split(',')
            repo_path, actions = items[0], items[1:]
            for a in actions:
                if a not in Action:
                    raise ValueError('Unknown action: %s' % a)
            yield repo_path, actions

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Keep git repos up-to-date.')
    parser.add_argument('config', type=str, help='config file that lists repos')
    parser.add_argument('-v', '--verbose', default=False, action='store_true')
    args = parser.parse_args()

    if args.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    for repo, actions in parse(args.config):
        update(repo, actions)
<|endoftext|>


<commit_before>#!/usr/bin/env
"""Common utils."""
__author__ = 'Volodymyr Varchuk'
__email__ = "vladimir.varchuk@rackspace.com"

import bson

DELETE_TMPLT = 'DELETE FROM {table} WHERE {conditions};'
UPDATE_TMPLT = 'UPDATE {table} SET {statements} WHERE {conditions};'
INSERT_TMPLT = 'INSERT INTO {table} ({columns}) VALUES({values});'
SELECT_TMPLT = 'SELECT MAX(idx) FROM {table} WHERE {conditions};'
UPSERT_TMLPT = """\
LOOP
    {update}
    IF found THEN
        RETURN;
    END IF;
    BEGIN
        {insert}
        RETURN;
    EXCEPTION WHEN unique_violation THEN
    END;
END LOOP;
"""

def get_field_name_without_underscore(field_name):
    for i in range(len(field_name)):
        if field_name[i].isalpha():
            s = field_name[i:]
            break
    return s

def isIdField(field_name):
    if field_name in ['id', 'oid', '_id', '_oid', '_id_oid', 'id_oid']:
        return True
    else:
        return False

def get_postgres_type(type_name):
    # TODO should be case insensitive
    # TODO should be replaced
    return {
        'STRING': 'text',
        'INT': 'integer',
        'BOOL': 'boolean',
        'LONG': 'bigint',
        'TIMESTAMP': 'text',
        'string': 'text',
        'int': 'integer',
        'bool': 'boolean',
        'long': 'bigint',
        'timestamp': 'text'
    }[type_name]

def get_table_name_from_list(spath):
    spathl = spath[:]
    for it in spathl:
        if it.isdigit():
            spathl.remove(it)
    if len(spathl) > 1:
        return '_'.join(['_'.join((el[:-1]) for el in spathl[:-1]), spathl[-1]])
    else:
        return spathl[-1]

def get_idx_column_name_from_list(spath):
    spathl = spath[:]
    for it in spathl:
        if it.isdigit():
            spathl.remove(it)
    if len(spathl) > 1:
        return '_'.join(['_'.join((el) for el in spathl[:-1]), spathl[-1]])
    else:
        return spathl[-1]

def get_root_table_from_path(path):
    spath = path.split('.')
    if len(spath) == 0:
        return path
    else:
        return spath[0]
    return get_table_name_from_list(collection_path)

def get_indexes_dictionary(path):
    index_dict = {}
    spath = path.split('.')
    iter_i = reversed(xrange(len(spath)))
    for i in iter_i:
        if spath[i].isdigit():
            table_name = get_table_name_from_list(spath)
            index_dict[table_name] = spath[i]
            del spath[i]
            del spath[i - 1]
            next(iter_i)
        else:
            del spath[i]
    return index_dict

def get_indexes_dictionary_idx(path):
    index_dict = {}
    spath = path.split('.')
    iter_i = reversed(xrange(len(spath)))
    for i in iter_i:
        if spath[i].isdigit():
            table_name = get_idx_column_name_from_list(spath)
            index_dict[table_name] = spath[i]
            del spath[i]
            del spath[i - 1]
            next(iter_i)
        else:
            del spath[i]
    return index_dict

def get_last_idx_from_path(path):
    spath = path.split('.')
    if spath[-1].isdigit():
        return spath[-1]
    else:
        return None

def get_ids_list(lst, is_root):
    if type(lst) is list:
        list_it = lst[0]
    else:
        list_it = lst
    ids_to_add = {}
    for it in list_it:
        if isIdField(it) and is_root:
            if type(list_it[it]) is dict:
                for id_item in list_it[it]:
                    if isIdField(id_item) and is_root:
                        ids_to_add[get_field_name_without_underscore(
                            it + '_' + get_field_name_without_underscore(id_item))] = get_postgres_type(
                            list_it[it][id_item])
            else:
                ids_to_add[get_field_name_without_underscore(it)] = get_postgres_type(list_it[it])
    if len(ids_to_add) == 0:
        ids_to_add['idx'] = 'bigint'
    return ids_to_add

def get_tables_structure(schema, table, table_mappings, parent_tables_ids, root_table):
    if type(schema) is list:
        table_struct = schema[0]
    else:
        table_struct = schema

    table_mappings[table] = {}

    for ids in parent_tables_ids:
        table_mappings[table][ids] = parent_tables_ids[ids]

    if not root_table:
        table_mappings[table][u'idx'] = u'bigint'
        parent_tables_ids[table + u'_idx'] = u'bigint'
    else:
        root_ids = get_ids_list(schema, 1)
        root_id_key = root_ids.iterkeys().next()
        parent_tables_ids[table + '_' + root_id_key] = root_ids[root_id_key].decode('utf-8')
    root_table = 0

    if not type(table_struct) is dict:
        table_mappings[table][u'data'] = get_postgres_type(table_struct)
        return table_mappings

    for element in table_struct:
        if type(table_struct[element]) is list:
            get_tables_structure(table_struct[element], table[:-1] + '_' + get_field_name_without_underscore(element),
                                 table_mappings, parent_tables_ids.copy(), root_table)
        elif type(table_struct[element]) is dict:
            get_table_struct_from_dict(table_struct[element], table, table_mappings, parent_tables_ids.copy(),
                                       get_field_name_without_underscore(element))
        else:
            table_mappings[table][get_field_name_without_underscore(element)] = get_postgres_type(table_struct[element])
    return table_mappings

def get_table_struct_from_dict(schema, table, table_mappings, parent_tables_ids, parent_name):
    for column in schema:
        if type(schema[column]) is dict:
            get_table_struct_from_dict(schema[column], table, table_mappings, parent_tables_ids,
                                       parent_name + '_' + get_field_name_without_underscore(column))
        elif type(schema[column]) is list:
            get_tables_structure(schema[column], table[:-1] + '_' + parent_name + '_' + column, table_mappings,
                                 parent_tables_ids, 0)
        else:
            table_mappings[table][parent_name + '_' + get_field_name_without_underscore(column)] = get_postgres_type(
                schema[column])

def get_column_type(schema, table, field_name, collection_name):
    return get_tables_structure(schema, collection_name, {}, {}, 1)[table][field_name]

def get_quotes_using(schema, table, field_name, collection_name):
    quotes_not_needed = ['int', 'bigint', 'integer']
    if get_column_type(schema, table, field_name, collection_name) in quotes_not_needed:
        return False
    else:
        return True
<commit_msg>added database name and scema support<commit_after>#!/usr/bin/env
"""Common utils."""
__author__ = 'Volodymyr Varchuk'
__email__ = "vladimir.varchuk@rackspace.com"

import bson

DELETE_TMPLT = 'DELETE FROM {table} WHERE {conditions};'
UPDATE_TMPLT = 'UPDATE {table} SET {statements} WHERE {conditions};'
INSERT_TMPLT = 'INSERT INTO {table} ({columns}) VALUES({values});'
SELECT_TMPLT = 'SELECT MAX(idx) FROM {table} WHERE {conditions};'
UPSERT_TMLPT = """\
LOOP
    {update}
    IF found THEN
        RETURN;
    END IF;
    BEGIN
        {insert}
        RETURN;
    EXCEPTION WHEN unique_violation THEN
    END;
END LOOP;
"""

def get_field_name_without_underscore(field_name):
    for i in range(len(field_name)):
        if field_name[i].isalpha():
            s = field_name[i:]
            break
    return s

def isIdField(field_name):
    if field_name in ['id', 'oid', '_id', '_oid', '_id_oid', 'id_oid']:
        return True
    else:
        return False

def get_table_name_schema(str_list):
    return '.'.join(filter(None,str_list))

def get_postgres_type(type_name):
    # TODO should be case insensitive
    # TODO should be replaced
    return {
        'STRING': 'text',
        'INT': 'integer',
        'BOOL': 'boolean',
        'LONG': 'bigint',
        'TIMESTAMP': 'text',
        'string': 'text',
        'int': 'integer',
        'bool': 'boolean',
        'long': 'bigint',
        'timestamp': 'text'
    }[type_name]

def get_table_name_from_list(spath):
    spathl = spath[:]
    for it in spathl:
        if it.isdigit():
            spathl.remove(it)
    if len(spathl) > 1:
        return '_'.join(['_'.join((el[:-1]) for el in spathl[:-1]), spathl[-1]])
    else:
        return spathl[-1]

def get_idx_column_name_from_list(spath):
    spathl = spath[:]
    for it in spathl:
        if it.isdigit():
            spathl.remove(it)
    if len(spathl) > 1:
        return '_'.join(['_'.join((el) for el in spathl[:-1]), spathl[-1]])
    else:
        return spathl[-1]

def get_root_table_from_path(path):
    spath = path.split('.')
    if len(spath) == 0:
        return path
    else:
        return spath[0]
    return get_table_name_from_list(collection_path)

def get_indexes_dictionary(path):
    index_dict = {}
    spath = path.split('.')
    iter_i = reversed(xrange(len(spath)))
    for i in iter_i:
        if spath[i].isdigit():
            table_name = get_table_name_from_list(spath)
            index_dict[table_name] = spath[i]
            del spath[i]
            del spath[i - 1]
            next(iter_i)
        else:
            del spath[i]
    return index_dict

def get_indexes_dictionary_idx(path):
    index_dict = {}
    spath = path.split('.')
    iter_i = reversed(xrange(len(spath)))
    for i in iter_i:
        if spath[i].isdigit():
            table_name = get_idx_column_name_from_list(spath)
            index_dict[table_name] = spath[i]
            del spath[i]
            del spath[i - 1]
            next(iter_i)
        else:
            del spath[i]
    return index_dict

def get_last_idx_from_path(path):
    spath = path.split('.')
    if spath[-1].isdigit():
        return spath[-1]
    else:
        return None

def get_ids_list(lst, is_root):
    if type(lst) is list:
        list_it = lst[0]
    else:
        list_it = lst
    ids_to_add = {}
    for it in list_it:
        if isIdField(it) and is_root:
            if type(list_it[it]) is dict:
                for id_item in list_it[it]:
                    if isIdField(id_item) and is_root:
                        ids_to_add[get_field_name_without_underscore(
                            it + '_' + get_field_name_without_underscore(id_item))] = get_postgres_type(
                            list_it[it][id_item])
            else:
                ids_to_add[get_field_name_without_underscore(it)] = get_postgres_type(list_it[it])
    if len(ids_to_add) == 0:
        ids_to_add['idx'] = 'bigint'
    return ids_to_add

def get_tables_structure(schema, table, table_mappings, parent_tables_ids, root_table):
    if type(schema) is list:
        table_struct = schema[0]
    else:
        table_struct = schema

    table_mappings[table] = {}

    for ids in parent_tables_ids:
        table_mappings[table][ids] = parent_tables_ids[ids]

    if not root_table:
        table_mappings[table][u'idx'] = u'bigint'
        parent_tables_ids[table + u'_idx'] = u'bigint'
    else:
        root_ids = get_ids_list(schema, 1)
        root_id_key = root_ids.iterkeys().next()
        parent_tables_ids[table + '_' + root_id_key] = root_ids[root_id_key].decode('utf-8')
    root_table = 0

    if not type(table_struct) is dict:
        table_mappings[table][u'data'] = get_postgres_type(table_struct)
        return table_mappings

    for element in table_struct:
        if type(table_struct[element]) is list:
            get_tables_structure(table_struct[element], table[:-1] + '_' + get_field_name_without_underscore(element),
                                 table_mappings, parent_tables_ids.copy(), root_table)
        elif type(table_struct[element]) is dict:
            get_table_struct_from_dict(table_struct[element], table, table_mappings, parent_tables_ids.copy(),
                                       get_field_name_without_underscore(element))
        else:
            table_mappings[table][get_field_name_without_underscore(element)] = get_postgres_type(table_struct[element])
    return table_mappings

def get_table_struct_from_dict(schema, table, table_mappings, parent_tables_ids, parent_name):
    for column in schema:
        if type(schema[column]) is dict:
            get_table_struct_from_dict(schema[column], table, table_mappings, parent_tables_ids,
                                       parent_name + '_' + get_field_name_without_underscore(column))
        elif type(schema[column]) is list:
            get_tables_structure(schema[column], table[:-1] + '_' + parent_name + '_' + column, table_mappings,
                                 parent_tables_ids, 0)
        else:
            table_mappings[table][parent_name + '_' + get_field_name_without_underscore(column)] = get_postgres_type(
                schema[column])

def get_column_type(schema, table, field_name, collection_name):
    return get_tables_structure(schema, collection_name, {}, {}, 1)[table][field_name]

def get_quotes_using(schema, table, field_name, collection_name):
    quotes_not_needed = ['int', 'bigint', 'integer']
    if get_column_type(schema, table, field_name, collection_name) in quotes_not_needed:
        return False
    else:
        return True
<|endoftext|>


<commit_before>try:
    import xml.etree.cElementTree as etree
except ImportError:
    import xml.etree.ElementTree as etree

from collections import defaultdict, OrderedDict
from contextlib import closing
from itertools import chain
import sys
import re

if sys.version_info >= (3, 0):
    from urllib.request import urlopen
else:
    from urllib2 import urlopen

_ARRAY_RE = re.compile(r'\[\d+\]')

class Spec(object):
    API = 'https://cvs.khronos.org/svn/repos/ogl/trunk/doc/registry/public/api/'
    NAME = ''

    def __init__(self, root):
        self.root = root

        self._types = None
        self._groups = None
        self._enums = None
        self._commands = None
        self._features = None
        self._extensions = None

    @classmethod
    def from_url(cls, url):
        with closing(urlopen(url)) as f:
            raw = f.read()

        return cls(etree.fromstring(raw))

    @classmethod
    def from_svn(cls):
        return cls.from_url(cls.API + cls.NAME + '.xml')

    @classmethod
    def fromstring(cls, string):
        return cls(etree.fromstring(string))

    @classmethod
    def from_file(cls, path):
        return cls(etree.parse(path).getroot())

    @property
    def comment(self):
        return self.root.find('comment').text

    @property
    def types(self):
        if self._types is None:
            self._types = [Type(element) for element in
                           self.root.find('types').iter('type')]
        return self._types

    @property
    def groups(self):
        if self._groups is None:
            self._groups = dict([(element.attrib['name'], Group(element))
                                 for element in self.root.find('groups')])
        return self._groups

    @property
    def commands(self):
        if self._commands is None:
            self._commands = dict([(element.find('proto').find('name').text,
                                    Command(element, self))
                                   for element in self.root.find('commands')])
        return self._commands

    @property
    def enums(self):
        if self._enums is not None:
            return self._enums

        self._enums = dict()
        for element in self.root.iter('enums'):
            namespace = element.attrib['namespace']
            type_ = element.get('type')
            group = element.get('group')
            vendor = element.get('vendor')
            comment = element.get('comment', '')

            for enum in element:
                if enum.tag == 'unused':
                    continue
                assert enum.tag == 'enum'

                name = enum.attrib['name']
                self._enums[name] = Enum(name, enum.attrib['value'], namespace,
                                         type_, group, vendor, comment)

        return self._enums

    @property
    def features(self):
        if self._features is not None:
            return self._features

        self._features = defaultdict(OrderedDict)
        for element in self.root.iter('feature'):
            num = tuple(map(int, element.attrib['number'].split('.')))
            self._features[element.attrib['api']][num] = Feature(element, self)

        return self._features

    @property
    def extensions(self):
        if self._extensions is not None:
            return self._extensions

        self._extensions = defaultdict(dict)
        for element in self.root.find('extensions'):
            for api in element.attrib['supported'].split('|'):
                self._extensions[api][element.attrib['name']] = Extension(element, self)

        return self._extensions

class Type(object):
    def __init__(self, element):
        apientry = element.find('apientry')
        if apientry is not None:
            apientry.text = 'APIENTRY'
        self.raw = ''.join(element.itertext())
        self.api = element.get('api')
        self.name = element.get('name')

    @property
    def is_preprocessor(self):
        return '#' in self.raw

class Group(object):
    def __init__(self, element):
        self.name = element.attrib['name']
        self.enums = [enum.attrib['name'] for enum in element]

class Enum(object):
    def __init__(self, name, value, namespace, type_=None,
                 group=None, vendor=None, comment=''):
        self.name = name
        self.value = value
        self.namespace = namespace
        self.type = type_
        self.group = group
        self.vendor = vendor
        self.comment = comment

    def __hash__(self):
        return hash(self.name)

    def __str__(self):
        return self.name

    __repr__ = __str__

class Command(object):
    def __init__(self, element, spec):
        self.proto = Proto(element.find('proto'))
        self.params = [Param(ele, spec) for ele in element.iter('param')]

    def __hash__(self):
        return hash(self.proto.name)

    def __str__(self):
        return '{self.proto.name}'.format(self=self)

    __repr__ = __str__

class Proto(object):
    def __init__(self, element):
        self.name = element.find('name').text
        self.ret = OGLType(element)

    def __str__(self):
        return '{self.ret} {self.name}'.format(self=self)

class Param(object):
    def __init__(self, element, spec):
        self.group = element.get('group')
        self.type = OGLType(element)
        self.name = element.find('name').text

    def __str__(self):
        return '{0!r} {1}'.format(self.type, self.name)

class OGLType(object):
    def __init__(self, element):
        text = ''.join(element.itertext())
        self.type = (text.replace('const', '').replace('unsigned', '')
                     .replace('struct', '').strip().split(None, 1)[0]
                     if element.find('ptype') is None else element.find('ptype').text)
        # 0 if no pointer, 1 if *, 2 if **
        self.is_pointer = 0 if text is None else text.count('*')
        # it can be a pointer to an array, or just an array
        self.is_pointer += len(_ARRAY_RE.findall(text))
        self.is_const = False if text is None else 'const' in text
        self.is_unsigned = False if text is None else 'unsigned' in text

        if 'struct' in text and 'struct' not in self.type:
            self.type = 'struct {}'.format(self.type)

    def to_d(self):

        if self.is_pointer > 1 and self.is_const:
            s = 'const({}{}*)'.format('u' if self.is_unsigned else '', self.type)
            s += '*' * (self.is_pointer - 1)
        else:
            t = '{}{}'.format('u' if self.is_unsigned else '', self.type)
            s = 'const({})'.format(t) if self.is_const else t
            s += '*' * self.is_pointer
        return s.replace('struct ', '')

    to_volt = to_d

    def to_c(self):
        ut = 'unsigned {}'.format(self.type) if self.is_unsigned else self.type
        s = '{}const {}'.format('unsigned ' if self.is_unsigned else '', self.type) \
            if self.is_const else ut
        s += '*' * self.is_pointer
        return s

    __str__ = to_d
    __repr__ = __str__

class Extension(object):
    def __init__(self, element, spec):
        self.name = element.attrib['name']

        self.require = []
        for required in chain.from_iterable(element.findall('require')):
            if required.tag == 'type':
                continue

            data = {'enum': spec.enums, 'command': spec.commands}[required.tag]
            try:
                self.require.append(data[required.attrib['name']])
            except KeyError:
                pass  # TODO

    @property
    def enums(self):
        for r in self.require:
            if isinstance(r, Enum):
                yield r

    @property
    def functions(self):
        for r in self.require:
            if isinstance(r, Command):
                yield r

    def __hash__(self):
        return hash(self.name)

    def __str__(self):
        return self.name

    __repr__ = __str__

class Feature(Extension):
    def __init__(self, element, spec):
        Extension.__init__(self, element, spec)
        self.spec = spec

        # not every spec has a ._remove member, but there shouldn't be a remove
        # tag without that member, if there is, blame me!
        for removed in chain.from_iterable(element.findall('remove')):
            if removed.tag == 'type':
                continue

            data = {'enum': spec.enums, 'command': spec.commands}[removed.tag]
            try:
                spec._remove.add(data[removed.attrib['name']])
            except KeyError:
                pass  # TODO

        self.number = tuple(map(int, element.attrib['number'].split('.')))
        self.api = element.attrib['api']

    def __str__(self):
        return '{self.name}@{self.number!r}'.format(self=self)

    @property
    def enums(self):
        for enum in super(Feature, self).enums:
            if enum not in getattr(self.spec, 'removed', []):
                yield enum

    @property
    def functions(self):
        for func in super(Feature, self).functions:
            if func not in getattr(self.spec, 'removed', []):
                yield func

    __repr__ = __str__
<commit_msg>parse: use lxml if available<commit_after>try:
    from lxml import etree
    from lxml.etree import ETCompatXMLParser as parser
except ImportError:
    try:
        import xml.etree.cElementTree as etree
    except ImportError:
        import xml.etree.ElementTree as etree
    parser = etree.XMLParser

from collections import defaultdict, OrderedDict
from contextlib import closing
from itertools import chain
import sys
import re

if sys.version_info >= (3, 0):
    from urllib.request import urlopen
else:
    from urllib2 import urlopen

_ARRAY_RE = re.compile(r'\[\d+\]')

class Spec(object):
    API = 'https://cvs.khronos.org/svn/repos/ogl/trunk/doc/registry/public/api/'
    NAME = ''

    def __init__(self, root):
        self.root = root

        self._types = None
        self._groups = None
        self._enums = None
        self._commands = None
        self._features = None
        self._extensions = None

    @classmethod
    def from_url(cls, url):
        with closing(urlopen(url)) as f:
            raw = f.read()

        return cls(etree.fromstring(raw, parser=parser()))

    @classmethod
    def from_svn(cls):
        return cls.from_url(cls.API + cls.NAME + '.xml')

    @classmethod
    def fromstring(cls, string):
        return cls(etree.fromstring(string, parser=parser()))

    @classmethod
    def from_file(cls, path):
        return cls(etree.parse(path, parser=parser()).getroot())

    @property
    def comment(self):
        return self.root.find('comment').text

    @property
    def types(self):
        if self._types is None:
            self._types = [Type(element) for element in
                           self.root.find('types').iter('type')]
        return self._types

    @property
    def groups(self):
        if self._groups is None:
            self._groups = dict([(element.attrib['name'], Group(element))
                                 for element in self.root.find('groups')])
        return self._groups

    @property
    def commands(self):
        if self._commands is None:
            self._commands = dict([(element.find('proto').find('name').text,
                                    Command(element, self))
                                   for element in self.root.find('commands')])
        return self._commands

    @property
    def enums(self):
        if self._enums is not None:
            return self._enums

        self._enums = dict()
        for element in self.root.iter('enums'):
            namespace = element.attrib['namespace']
            type_ = element.get('type')
            group = element.get('group')
            vendor = element.get('vendor')
            comment = element.get('comment', '')

            for enum in element:
                if enum.tag == 'unused':
                    continue
                assert enum.tag == 'enum'

                name = enum.attrib['name']
                self._enums[name] = Enum(name, enum.attrib['value'], namespace,
                                         type_, group, vendor, comment)

        return self._enums

    @property
    def features(self):
        if self._features is not None:
            return self._features

        self._features = defaultdict(OrderedDict)
        for element in self.root.iter('feature'):
            num = tuple(map(int, element.attrib['number'].split('.')))
            self._features[element.attrib['api']][num] = Feature(element, self)

        return self._features

    @property
    def extensions(self):
        if self._extensions is not None:
            return self._extensions

        self._extensions = defaultdict(dict)
        for element in self.root.find('extensions'):
            for api in element.attrib['supported'].split('|'):
                self._extensions[api][element.attrib['name']] = Extension(element, self)

        return self._extensions

class Type(object):
    def __init__(self, element):
        apientry = element.find('apientry')
        if apientry is not None:
            apientry.text = 'APIENTRY'
        self.raw = ''.join(element.itertext())
        self.api = element.get('api')
        self.name = element.get('name')

    @property
    def is_preprocessor(self):
        return '#' in self.raw

class Group(object):
    def __init__(self, element):
        self.name = element.attrib['name']
        self.enums = [enum.attrib['name'] for enum in element]

class Enum(object):
    def __init__(self, name, value, namespace, type_=None,
                 group=None, vendor=None, comment=''):
        self.name = name
        self.value = value
        self.namespace = namespace
        self.type = type_
        self.group = group
        self.vendor = vendor
        self.comment = comment

    def __hash__(self):
        return hash(self.name)

    def __str__(self):
        return self.name

    __repr__ = __str__

class Command(object):
    def __init__(self, element, spec):
        self.proto = Proto(element.find('proto'))
        self.params = [Param(ele, spec) for ele in element.iter('param')]

    def __hash__(self):
        return hash(self.proto.name)

    def __str__(self):
        return '{self.proto.name}'.format(self=self)

    __repr__ = __str__

class Proto(object):
    def __init__(self, element):
        self.name = element.find('name').text
        self.ret = OGLType(element)

    def __str__(self):
        return '{self.ret} {self.name}'.format(self=self)

class Param(object):
    def __init__(self, element, spec):
        self.group = element.get('group')
        self.type = OGLType(element)
        self.name = element.find('name').text

    def __str__(self):
        return '{0!r} {1}'.format(self.type, self.name)

class OGLType(object):
    def __init__(self, element):
        text = ''.join(element.itertext())
        self.type = (text.replace('const', '').replace('unsigned', '')
                     .replace('struct', '').strip().split(None, 1)[0]
                     if element.find('ptype') is None else element.find('ptype').text)
        # 0 if no pointer, 1 if *, 2 if **
        self.is_pointer = 0 if text is None else text.count('*')
        # it can be a pointer to an array, or just an array
        self.is_pointer += len(_ARRAY_RE.findall(text))
        self.is_const = False if text is None else 'const' in text
        self.is_unsigned = False if text is None else 'unsigned' in text

        if 'struct' in text and 'struct' not in self.type:
            self.type = 'struct {}'.format(self.type)

    def to_d(self):

        if self.is_pointer > 1 and self.is_const:
            s = 'const({}{}*)'.format('u' if self.is_unsigned else '', self.type)
            s += '*' * (self.is_pointer - 1)
        else:
            t = '{}{}'.format('u' if self.is_unsigned else '', self.type)
            s = 'const({})'.format(t) if self.is_const else t
            s += '*' * self.is_pointer
        return s.replace('struct ', '')

    to_volt = to_d

    def to_c(self):
        ut = 'unsigned {}'.format(self.type) if self.is_unsigned else self.type
        s = '{}const {}'.format('unsigned ' if self.is_unsigned else '', self.type) \
            if self.is_const else ut
        s += '*' * self.is_pointer
        return s

    __str__ = to_d
    __repr__ = __str__

class Extension(object):
    def __init__(self, element, spec):
        self.name = element.attrib['name']

        self.require = []
        for required in chain.from_iterable(element.findall('require')):
            if required.tag == 'type':
                continue

            data = {'enum': spec.enums, 'command': spec.commands}[required.tag]
            try:
                self.require.append(data[required.attrib['name']])
            except KeyError:
                pass  # TODO

    @property
    def enums(self):
        for r in self.require:
            if isinstance(r, Enum):
                yield r

    @property
    def functions(self):
        for r in self.require:
            if isinstance(r, Command):
                yield r

    def __hash__(self):
        return hash(self.name)

    def __str__(self):
        return self.name

    __repr__ = __str__

class Feature(Extension):
    def __init__(self, element, spec):
        Extension.__init__(self, element, spec)
        self.spec = spec

        # not every spec has a ._remove member, but there shouldn't be a remove
        # tag without that member, if there is, blame me!
        for removed in chain.from_iterable(element.findall('remove')):
            if removed.tag == 'type':
                continue

            data = {'enum': spec.enums, 'command': spec.commands}[removed.tag]
            try:
                spec._remove.add(data[removed.attrib['name']])
            except KeyError:
                pass  # TODO

        self.number = tuple(map(int, element.attrib['number'].split('.')))
        self.api = element.attrib['api']

    def __str__(self):
        return '{self.name}@{self.number!r}'.format(self=self)

    @property
    def enums(self):
        for enum in super(Feature, self).enums:
            if enum not in getattr(self.spec, 'removed', []):
                yield enum

    @property
    def functions(self):
        for func in super(Feature, self).functions:
            if func not in getattr(self.spec, 'removed', []):
                yield func

    __repr__ = __str__
<|endoftext|>


<commit_before>import imp
import os
import sys
from dotenv import load_dotenv

here_dir = os.path.dirname(os.path.abspath(__file__))
module_path = os.path.join(here_dir, '../../')
config_file = os.path.join(here_dir, "../configuration/local.env")
load_dotenv(dotenv_path=config_file)
sys.path.append(module_path)
# Load cortexlib
fp, pathname, description = imp.find_module('cortexlib')
cortexlib = imp.load_module('cortexlib', fp, pathname, description)
# Load donlib
fp, pathname, description = imp.find_module('donlib')
donlib = imp.load_module('donlib', fp, pathname, description)

safe_event = {"server_group_name": "NOTME",
              "type": "NOT_A_BAD_TYPE",
              "critical": True}

quar_event = {"critical": True}

class TestIntegrationCortexlibQuarantine:
    def instantiate_donlib_config_good(self, monkeypatch):
        monkeypatch.setenv('HALO_API_HOSTNAME', 'api.cloudpassage.com')
        monkeypatch.setenv('HALO_API_PORT', 443)
        monkeypatch.setenv('SLACK_API_TOKEN', 'some_token_')
        config = donlib.ConfigHelper()
        return config

    def instantiate_donlib_config_invalid_halo(self, monkeypatch):
        monkeypatch.setenv('HALO_API_KEY', 'bad_key')
        monkeypatch.setenv('HALO_API_SECRET_KEY', 'bad_secret')
        monkeypatch.setenv('HALO_API_HOSTNAME', 'api.cloudpassage.com')
        monkeypatch.setenv('HALO_API_PORT', 443)
        monkeypatch.setenv('SLACK_API_TOKEN', 'some_token_')
        config = donlib.ConfigHelper()
        return config

    def instantiate_donlib_config_bad_q_config(self, monkeypatch):
        monkeypatch.setenv('HALO_API_HOSTNAME', 'api.cloudpassage.com')
        monkeypatch.setenv('HALO_API_PORT', 443)
        monkeypatch.setenv('SLACK_API_TOKEN', 'some_token_')
        monkeypatch.setenv('QUARANTINE_TRIGGER_GROUP_NAMES', 'duplicate')
        config = donlib.ConfigHelper()
        return config

    def instantiate_donlib_config_q_disabled(self, monkeypatch):
        monkeypatch.setenv('HALO_API_HOSTNAME', 'api.cloudpassage.com')
        monkeypatch.setenv('HALO_API_PORT', 443)
        monkeypatch.setenv('SLACK_API_TOKEN', 'some_token_')
        monkeypatch.setenv('QUARANTINE_ENABLED', 'false')
        config = donlib.ConfigHelper()
        return config

    def instantiate_cortexlib_quarantine(self, monkeypatch):
        config = self.instantiate_donlib_config_good(monkeypatch)
        q_obj = cortexlib.Quarantine(config)
        return q_obj

    def instantiate_cortexlib_quarantine_bad_conf(self, monkeypatch):
        config = self.instantiate_donlib_config_bad_q_config(monkeypatch)
        q_obj = cortexlib.Quarantine(config)
        return q_obj

    def instantiate_cortexlib_quarantine_bad_keys(self, monkeypatch):
        config = self.instantiate_donlib_config_invalid_halo(monkeypatch)
        q_obj = cortexlib.Quarantine(config)
        return q_obj

    def instantiate_cortexlib_quarantine_disabled(self, monkeypatch):
        config = self.instantiate_donlib_config_q_disabled(monkeypatch)
        q_obj = cortexlib.Quarantine(config)
        return q_obj

    def test_instantiate_cortexlib_quarantine(self, monkeypatch):
        assert self.instantiate_cortexlib_quarantine(monkeypatch)

    def test_trigger_validation_fail_invalid_trigger_group(self, monkeypatch):
        """Walk through scenarios where live validation will fail because of a
        missing group."""
        q = self.instantiate_cortexlib_quarantine(monkeypatch)
        q.quarantine_enable = True
        # Invalid group name causes validation failure
        q.quarantine_trigger_group_names = ["invalidus maximus"]
        q.quarantine_group_name = "donbot testing"
        assert q.config_is_unambiguous() is False

    def test_trigger_validation_fail_ambiguous_quarantine_group(self,
                                                                monkeypatch):
        """Walk through scenario where live validation will fail over duplicate
        groups."""
        q = self.instantiate_cortexlib_quarantine(monkeypatch)
        q.quarantine_enable = True
        # Multiple groups with the same name causes validation failure
        q.quarantine_trigger_group_names = ["duplicate"]
        q.quarantine_group_name = "donbot testing"
        assert q.config_is_unambiguous() is False

    def test_quarantine_event_trigger(self, monkeypatch):
        """Successfully match a quarantine event."""
        q = self.instantiate_cortexlib_quarantine(monkeypatch)
        q_event = quar_event.copy()
        q_event["server_group_name"] = q.quarantine_trigger_group_names[0]
        q_event["type"] = q.quarantine_trigger_events[0]
        q_grp = q.quarantine_group_name
        print(q.quarantine_enable)
        print(q_grp)
        result = q.should_quarantine(q_event)
        print result
        assert result["quarantine_group"] == q_grp

    def test_quarantine_event_crit_no_trigger(self, monkeypatch):
        """Don't trigger quarantine because criticality doesn't pass test."""
        q = self.instantiate_cortexlib_quarantine(monkeypatch)
        q_event = quar_event.copy()
        q_event["server_group_name"] = q.quarantine_trigger_group_names[0]
        q_event["type"] = q.quarantine_trigger_events[0]
        q_event["critical"] = False
        assert q.should_quarantine(q_event) is False

    def test_quarantine_event_no_trigger(self, monkeypatch):
        """This should NOT match as a quarantine event."""
        q = self.instantiate_cortexlib_quarantine(monkeypatch)
        assert q.should_quarantine(safe_event) is False

    def test_quarantine_event_no_trigger_due_to_bad_config(self, monkeypatch):
        """Valid quarantine event should not trigger because config is bad."""
        q = self.instantiate_cortexlib_quarantine_bad_conf(monkeypatch)
        q_event = quar_event.copy()
        q_event["server_group_name"] = q.quarantine_trigger_group_names[0]
        q_event["type"] = q.quarantine_trigger_events[0]
        assert q.should_quarantine(q_event) is False

    def test_quarantine_event_no_trigger_because_disabled(self, monkeypatch):
        """Valid quarantine event should not trigger because Q is disabled."""
        q = self.instantiate_cortexlib_quarantine_disabled(monkeypatch)
        q_event = quar_event.copy()
        q_event["server_group_name"] = q.quarantine_trigger_group_names[0]
        q_event["type"] = q.quarantine_trigger_events[0]
        assert q.should_quarantine(quar_event) is False

    def test_quarantine_event_no_trigger_because_bad_keys(self, monkeypatch):
        """Valid quarantine event should not trigger b/c API keys are bad."""
        q = self.instantiate_cortexlib_quarantine_bad_keys(monkeypatch)
        assert q.should_quarantine(quar_event) is False
<commit_msg>chg: test: Better definition of trigger event in test. !minor<commit_after>import imp
import os
import sys
from dotenv import load_dotenv

here_dir = os.path.dirname(os.path.abspath(__file__))
module_path = os.path.join(here_dir, '../../')
config_file = os.path.join(here_dir, "../configuration/local.env")
load_dotenv(dotenv_path=config_file)
sys.path.append(module_path)
# Load cortexlib
fp, pathname, description = imp.find_module('cortexlib')
cortexlib = imp.load_module('cortexlib', fp, pathname, description)
# Load donlib
fp, pathname, description = imp.find_module('donlib')
donlib = imp.load_module('donlib', fp, pathname, description)

safe_event = {"server_group_name": "NOTME",
              "type": "NOT_A_BAD_TYPE",
              "critical": True}

quar_event = {"critical": True}

class TestIntegrationCortexlibQuarantine:
    def instantiate_donlib_config_good(self, monkeypatch):
        monkeypatch.setenv('HALO_API_HOSTNAME', 'api.cloudpassage.com')
        monkeypatch.setenv('HALO_API_PORT', 443)
        monkeypatch.setenv('SLACK_API_TOKEN', 'some_token_')
        config = donlib.ConfigHelper()
        return config

    def instantiate_donlib_config_invalid_halo(self, monkeypatch):
        monkeypatch.setenv('HALO_API_KEY', 'bad_key')
        monkeypatch.setenv('HALO_API_SECRET_KEY', 'bad_secret')
        monkeypatch.setenv('HALO_API_HOSTNAME', 'api.cloudpassage.com')
        monkeypatch.setenv('HALO_API_PORT', 443)
        monkeypatch.setenv('SLACK_API_TOKEN', 'some_token_')
        config = donlib.ConfigHelper()
        return config

    def instantiate_donlib_config_bad_q_config(self, monkeypatch):
        monkeypatch.setenv('HALO_API_HOSTNAME', 'api.cloudpassage.com')
        monkeypatch.setenv('HALO_API_PORT', 443)
        monkeypatch.setenv('SLACK_API_TOKEN', 'some_token_')
        monkeypatch.setenv('QUARANTINE_TRIGGER_GROUP_NAMES', 'duplicate')
        config = donlib.ConfigHelper()
        return config

    def instantiate_donlib_config_q_disabled(self, monkeypatch):
        monkeypatch.setenv('HALO_API_HOSTNAME', 'api.cloudpassage.com')
        monkeypatch.setenv('HALO_API_PORT', 443)
        monkeypatch.setenv('SLACK_API_TOKEN', 'some_token_')
        monkeypatch.setenv('QUARANTINE_ENABLED', 'false')
        config = donlib.ConfigHelper()
        return config

    def instantiate_cortexlib_quarantine(self, monkeypatch):
        config = self.instantiate_donlib_config_good(monkeypatch)
        q_obj = cortexlib.Quarantine(config)
        return q_obj

    def instantiate_cortexlib_quarantine_bad_conf(self, monkeypatch):
        config = self.instantiate_donlib_config_bad_q_config(monkeypatch)
        q_obj = cortexlib.Quarantine(config)
        return q_obj

    def instantiate_cortexlib_quarantine_bad_keys(self, monkeypatch):
        config = self.instantiate_donlib_config_invalid_halo(monkeypatch)
        q_obj = cortexlib.Quarantine(config)
        return q_obj

    def instantiate_cortexlib_quarantine_disabled(self, monkeypatch):
        config = self.instantiate_donlib_config_q_disabled(monkeypatch)
        q_obj = cortexlib.Quarantine(config)
        return q_obj

    def test_instantiate_cortexlib_quarantine(self, monkeypatch):
        assert self.instantiate_cortexlib_quarantine(monkeypatch)

    def test_trigger_validation_fail_invalid_trigger_group(self, monkeypatch):
        """Walk through scenarios where live validation will fail because of a
        missing group."""
        q = self.instantiate_cortexlib_quarantine(monkeypatch)
        q.quarantine_enable = True
        # Invalid group name causes validation failure
        q.quarantine_trigger_group_names = ["invalidus maximus"]
        q.quarantine_group_name = "donbot testing"
        assert q.config_is_unambiguous() is False

    def test_trigger_validation_fail_ambiguous_quarantine_group(self,
                                                                monkeypatch):
        """Walk through scenario where live validation will fail over duplicate
        groups."""
        q = self.instantiate_cortexlib_quarantine(monkeypatch)
        q.quarantine_enable = True
        # Multiple groups with the same name causes validation failure
        q.quarantine_trigger_group_names = ["duplicate"]
        q.quarantine_group_name = "donbot testing"
        assert q.config_is_unambiguous() is False

    def test_quarantine_event_trigger(self, monkeypatch):
        """Successfully match a quarantine event."""
        q = self.instantiate_cortexlib_quarantine(monkeypatch)
        q_event = quar_event.copy()
        q_event["server_group_name"] = q.quarantine_trigger_group_names[0]
        q_event["type"] = q.quarantine_trigger_events[0]
        q_grp = q.quarantine_group_name
        print(q.quarantine_enable)
        print(q_grp)
        result = q.should_quarantine(q_event)
        print result
        assert result["quarantine_group"] == q_grp

    def test_quarantine_event_crit_no_trigger(self, monkeypatch):
        """Don't trigger quarantine because criticality doesn't pass test."""
        q = self.instantiate_cortexlib_quarantine(monkeypatch)
        q_event = quar_event.copy()
        q_event["server_group_name"] = q.quarantine_trigger_group_names[0]
        q_event["type"] = q.quarantine_trigger_events[0]
        q_event["critical"] = False
        assert q.should_quarantine(q_event) is False

    def test_quarantine_event_no_trigger(self, monkeypatch):
        """This should NOT match as a quarantine event."""
        q = self.instantiate_cortexlib_quarantine(monkeypatch)
        assert q.should_quarantine(safe_event) is False

    def test_quarantine_event_no_trigger_due_to_bad_config(self, monkeypatch):
        """Valid quarantine event should not trigger because config is bad."""
        q = self.instantiate_cortexlib_quarantine_bad_conf(monkeypatch)
        q_event = quar_event.copy()
        q_event["server_group_name"] = q.quarantine_trigger_group_names[0]
        q_event["type"] = q.quarantine_trigger_events[0]
        assert q.should_quarantine(q_event) is False

    def test_quarantine_event_no_trigger_because_disabled(self, monkeypatch):
        """Valid quarantine event should not trigger because Q is disabled."""
        q = self.instantiate_cortexlib_quarantine_disabled(monkeypatch)
        q_event = quar_event.copy()
        q_event["server_group_name"] = q.quarantine_trigger_group_names[0]
        q_event["type"] = q.quarantine_trigger_events[0]
        assert q.should_quarantine(quar_event) is False

    def test_quarantine_event_no_trigger_because_bad_keys(self, monkeypatch):
        """Valid quarantine event should not trigger b/c API keys are bad."""
        q = self.instantiate_cortexlib_quarantine_bad_keys(monkeypatch)
        q_event = quar_event.copy()
        q_event["server_group_name"] = q.quarantine_trigger_group_names[0]
        q_event["type"] = q.quarantine_trigger_events[0]
        assert q.should_quarantine(quar_event) is False
<|endoftext|>


<commit_before>from __future__ import division
import numpy as np
from scikits.audiolab import play

def testsignal(hz,seconds, amplitude = .5,sr=44100.):
    '''
    Create a sine wave at hz for n seconds
    '''
    # cycles per sample
    cps = hz / sr
    # total samples
    ts = seconds * sr
    return amplitude * np.sin(np.arange(0,ts*cps,cps) * (2*np.pi))
<commit_msg>basic synth setup with piano.py and scikit audiolab<commit_after>from __future__ import division
import math
import numpy as np
from scikits.audiolab import play

bpm = 160
noteFrequencies = []
noteNamesWithSharps = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"]
noteNamesWithFlats = ["C", "Db", "D", "Eb", "E", "F", "Gb", "G", "Ab", "A", "Bb", "B"]
referenceFrequency = referenceFrequency = 130.81278265*3 # C

for i in range(0, 12):
    noteFrequencies.append(referenceFrequency*math.pow(2, i/12))

def sig(hz,seconds, amplitude = .5,sr=44100.):
    cps = hz / sr
    ts = seconds * sr
    return amplitude * np.sin(np.arange(0,ts*cps,cps) * (2*np.pi))

# sync playNote function using frequency lookup
def playNote(note, beats):
    seconds = beats*60/bpm
    freq = 800
    if (note in noteNamesWithSharps):
        freq = noteFrequencies[noteNamesWithSharps.index(note)]
    else:
        freq = noteFrequencies[noteNamesWithFlats.index(note)]
    s = sig(freq, seconds)
    play(s)

playNote("C", 1/2)
playNote("C#", 1/2)
playNote("D", 1)
playNote("G", 4)
<|endoftext|>