<commit_before># -*- coding: utf-8 -*-

"""
	Project Bluebox

	Copyright (C) <2015> <University of Stuttgart>

	This software may be modified and distributed under the terms
	of the MIT license.  See the LICENSE file for details.
"""

import collections
import json
import logging
from io import StringIO
from urllib import parse as urlParse

import pandas
import psycopg2
import requests
from bokeh.embed import components
from bokeh.models import TickFormatter
from bokeh.palettes import small_palettes
from bokeh.plotting import figure
from bokeh.charts import Donut

from bokeh.properties import Dict, Int, String
from bokeh.resources import EMPTY
from bokeh.util.compiler import CoffeeScript
from flask import request, Response
from pandas.indexes.range import RangeIndex

from mcm.Bluebox import app, accountServer
from mcm.Bluebox import configuration
from mcm.Bluebox.exceptions import HttpError

###############################################################################
# HTTP API
###############################################################################
@app.route("/api_analytics/table", methods=["GET"])
def doTable():
    accountServer.assert_token_tenant_validity(request)
    nrDataSource = json.loads(urlParse.unquote(request.args.get("nrDataSource")))
    logging.info("producing table for: {}".format(nrDataSource))
    data = getDataFromNodeRed(nrDataSource=nrDataSource)
    # log.debug("our pandas data frame is: {}".format(data.to_json(orient="records")))
    info = StringIO()
    data.info(verbose=False, buf=info)
    r = {
        "table": data[:50].to_json(orient="records"),
        "info": info.getvalue(),
        "truncated": (len(data) > 50)
    }
    # print(r)
    return Response(json.dumps(r), mimetype="application/json")

@app.route("/api_analytics/plot", methods=["GET"])
def doPlot():
    accountServer.assert_token_tenant_validity(request)
    nrDataSource = json.loads(urlParse.unquote(request.args.get("nrDataSource")))
    plotType = request.args.get("plotType")
    logging.info("producing plot: {} for: {}".format(plotType, nrDataSource))

    df = getDataFromNodeRed(nrDataSource=nrDataSource)
    try:
        logging.info(df)
        if ('bar' == plotType):
            c = bokeh_plot_bar(data=df, nrDataSource=nrDataSource)
        elif ('bar_log' == plotType):
            c = bokeh_plot_bar(data=df, nrDataSource=nrDataSource, logScale="log")
        elif ('line' == plotType):
            c = bokeh_plot_line(data=df, nrDataSource=nrDataSource)
        elif ('line_log' == plotType):
            c = bokeh_plot_line(data=df, nrDataSource=nrDataSource, logScale="log")
        elif ('pie' == plotType):
            c = bokeh_plot_pie(data=df, nrDataSource=nrDataSource)
        else:
            return Response("Plot type unknown", status=500)
        return Response(json.dumps(c), mimetype="application/json")
    except Exception as e:
        logging.exception("plotting error:")
        raise HttpError(
            "the Node-RED result could not be plotted. Maybe wrong data format for the plot type? Check result table: {}".format(
                str(e)), 500)

@app.route("/api_analytics/nrsources", methods=["GET"])
def getNodeRedEnpointList():
    accountServer.assert_token_tenant_validity(request)
    n = requests.get(configuration.nodered_url + "/flows").json()
    sources = []
    for s in n:
        # Node-RED has a strange API... we can't reconstruct node/flow relationships...
        # if ('tab' == s['type'] and 'label' in s):
        # 	thisFlowName = s['label'] + "->"
        if ('http in' == s['type'] and 'url' in s):
            sources.append({"url": s['url'], "name": s['name']})
    return Response(json.dumps(sources), mimetype="application/json")

###############################################################################
# bokeh support functions
###############################################################################
class FixedTickFormatter(TickFormatter):
    """
    Class used to allow custom axis tick labels on a bokeh chart
    Extends bokeh.model.formatters.TickFormatter
    """
    labels = Dict(Int, String, help="""
            A mapping of integer ticks values to their labels.
            """)
    COFFEESCRIPT = """
        import {_} from "underscore"
        import {Model} from "model"
        import * as p from "core/properties"
        export class FixedTickFormatter extends Model
          type: 'FixedTickFormatter'
          @define {
            labels: [ p.Any ]
          }
          doFormat: (ticks) ->
                labels = @labels
                return (labels[tick] ? "" for tick in ticks)
    """

    __implementation__ = CoffeeScript(COFFEESCRIPT)

def __col_to_label_dict(col, offset=1):
    r = dict()
    for k, v in dict(col).items():
        r[(int(k) * offset)] = v
    return r

###############################################################################
# Plots
###############################################################################

###############################################################################
# WORKING
###############################################################################
def bokeh_plot_line(data, nrDataSource, logScale="linear"):
    title = "Line graph: " + nrDataSource['name']
    value_col_names = [d for d in data.columns[1:]]
    # print(data)

    # small_palettes only works with 2+ items; so we increase size by two
    #   and later reduce back down
    colors = small_palettes['Dark2'][len(value_col_names) + 2]
    colors = colors[:len(value_col_names)]

    plot = figure(plot_width=1000, plot_height=600, y_axis_type=logScale)
    plot.title.text = title
    plot.yaxis.axis_label = "values"
    plot.xaxis.axis_label = data.columns[0]

    for (col_name, color) in zip(value_col_names, colors):
        plot.line(data.index, data[col_name], name=col_name, legend=col_name, color=color, line_width=3)

    plot.xaxis[0].formatter = FixedTickFormatter(labels=__col_to_label_dict(data[data.columns[0]]))

    script, div = components(plot, resources=None, wrap_script=False, wrap_plot_info=True)

    js = EMPTY.js_raw[0] + script

    return (js, div)

def bokeh_plot_bar(data, nrDataSource, logScale="linear"):
    title = "Bar graph: " + nrDataSource['name']
    value_col_names = [d for d in data.columns[1:]]
    # print(data)

    # small_palettes only works with 2+ items; so we increase size by two
    #   and later reduce back down
    colors = small_palettes['Dark2'][len(value_col_names) + 2]
    colors = colors[:len(value_col_names)]

    plot = figure(plot_width=1000, plot_height=600, y_axis_type=logScale)
    plot.title.text = title
    plot.yaxis.axis_label = "values"
    plot.xaxis.axis_label = data.columns[0]

    num_series = len(value_col_names)
    for (col_name, color, idx) in zip(value_col_names, colors, range(0, num_series)):
        this_index = RangeIndex(start=idx, stop=len(data.index) * num_series, step=num_series)
        print(data[col_name])
        print(col_name)
        print(this_index)
        plot.vbar(x=this_index, width=0.5, top=data[col_name], name=col_name, legend=col_name, color=color)

    print(__col_to_label_dict(data[data.columns[0]], offset=num_series))
    plot.xaxis[0].formatter = FixedTickFormatter(labels=__col_to_label_dict(data[data.columns[0]], offset=num_series))

    script, div = components(plot, resources=None, wrap_script=False, wrap_plot_info=True)

    js = EMPTY.js_raw[0] + script

    return (js, div)

def bokeh_plot_pie(data, nrDataSource):
    value_col_names = [d for d in data.columns[1:]]
    # print(data)

    plot = Donut(data, label=['abbr', 'medal'], values='medal_count',
          text_font_size='8pt', color=small_palettes['Dark2'])

    script, div = components(plot, resources=None, wrap_script=False, wrap_plot_info=True)

    return (script, div)

###############################################################################
# nodered integration
###############################################################################

def getListOfKeys(d):
    keys = []
    for k in d.keys():
        keys.append(k)
    return keys

def getDataFromNodeRed(nrDataSource):
    url = configuration.nodered_url + nrDataSource['url']
    logging.info("getting data from node red at: {}".format(url))
    r = requests.get(url)
    if r.status_code == 404:
        raise HttpError("the Node-RED data source is not reachable: {}".format(url), 420)
    try:
        data = json.JSONDecoder(object_pairs_hook=collections.OrderedDict).decode(r.content.decode())
        dataKeys = getListOfKeys(data[0])
        df = pandas.DataFrame(data, columns=dataKeys)
        df[dataKeys[0]] = df[dataKeys[0]].map(lambda x: str(x)[:20])
        return df

    except:
        logging.exception("JSON parse error:")
        raise HttpError("the Node-RED result is no valid JSON", 500)

###############################################################################
# Database connection
###############################################################################
def __get_db_connection_for_tenant(tenant):
    d = configuration.metadata_warehouse_endpoint.copy()
    d["database"] = d["database"].format(tenant)
    return d

@app.route("/api_analytics/tablestructure", methods=["GET"])
def getTableStructure():
    # check if user is logged in
    t = accountServer.get_and_assert_tenant_from_request(request)
    d = __get_db_connection_for_tenant(t)
    # Establish connection to PostgreSQL database
    # conn = sqlite3.connect("/tmp/metadata.sqlite") #SQLITE
    with psycopg2.connect(**d) as conn:
        with conn.cursor() as cursor:
            # Retrieve all table names
            # cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name ASC") #SQLITE
            cursor.execute("SELECT table_name FROM information_schema.tables WHERE table_schema=%s;", ("public",))
            tableNames = cursor.fetchall()

            tableData = {}

            # Retrieve the column names and first 5 rows for each table
            for table in tableNames:
                # cursor.execute("PRAGMA table_info(" + table[0] + ")") #SQLITE
                cursor.execute("SELECT column_name,data_type FROM information_schema.columns WHERE table_name = %s;",
                               (table[0],))
                columnNames = cursor.fetchall()

                # cursor.execute("SELECT * FROM " + table[0] + " LIMIT 5") #SQLITE
                cursor.execute("SELECT * FROM {} LIMIT 5".format(table[0]))
                rowEntries = cursor.fetchall()

                # Dictionary which combines column names and row entries
                columnStructure = {}

                # format the column names and types
                columnList = ["{} ({})".format(c[0], c[1]) for c in columnNames]

                # Populate final dictionary table Data
                columnStructure['columnNames'] = columnList
                columnStructure['rowEntries'] = rowEntries
                tableData[table[0]] = columnStructure

            return Response(json.dumps(tableData), mimetype="application/json")
<commit_msg>new analytics<commit_after># -*- coding: utf-8 -*-

"""
	Project Bluebox

	Copyright (C) <2015> <University of Stuttgart>

	This software may be modified and distributed under the terms
	of the MIT license.  See the LICENSE file for details.
"""

import collections
import json
import logging
from io import StringIO
from math import pi
from urllib import parse as urlParse

import pandas
import psycopg2
import requests
from bokeh.embed import components
from bokeh.models import TickFormatter
from bokeh.palettes import small_palettes
from bokeh.plotting import figure
from bokeh.charts import Donut

from bokeh.properties import Dict, Int, String
from bokeh.resources import EMPTY
from bokeh.util.compiler import CoffeeScript
from flask import request, Response
from pandas.indexes.range import RangeIndex

from mcm.Bluebox import app, accountServer
from mcm.Bluebox import configuration
from mcm.Bluebox.exceptions import HttpError

###############################################################################
# HTTP API
###############################################################################
@app.route("/api_analytics/table", methods=["GET"])
def doTable():
    accountServer.assert_token_tenant_validity(request)
    nrDataSource = json.loads(urlParse.unquote(request.args.get("nrDataSource")))
    logging.info("producing table for: {}".format(nrDataSource))
    data = getDataFromNodeRed(nrDataSource=nrDataSource)
    # log.debug("our pandas data frame is: {}".format(data.to_json(orient="records")))
    info = StringIO()
    data.info(verbose=False, buf=info)
    r = {
        "table": data[:50].to_json(orient="records"),
        "info": info.getvalue(),
        "truncated": (len(data) > 50)
    }
    # print(r)
    return Response(json.dumps(r), mimetype="application/json")

@app.route("/api_analytics/plot", methods=["GET"])
def doPlot():
    accountServer.assert_token_tenant_validity(request)
    nrDataSource = json.loads(urlParse.unquote(request.args.get("nrDataSource")))
    plotType = request.args.get("plotType")
    logging.info("producing plot: {} for: {}".format(plotType, nrDataSource))

    df = getDataFromNodeRed(nrDataSource=nrDataSource)
    try:
        logging.info(df)
        if ('bar' == plotType):
            c = bokeh_plot_bar(data=df, nrDataSource=nrDataSource)
        elif ('bar_log' == plotType):
            c = bokeh_plot_bar(data=df, nrDataSource=nrDataSource, logScale="log")
        elif ('line' == plotType):
            c = bokeh_plot_line(data=df, nrDataSource=nrDataSource)
        elif ('line_log' == plotType):
            c = bokeh_plot_line(data=df, nrDataSource=nrDataSource, logScale="log")
        elif ('pie' == plotType):
            c = bokeh_plot_pie(data=df, nrDataSource=nrDataSource)
        else:
            return Response("Plot type unknown", status=500)
        return Response(json.dumps(c), mimetype="application/json")
    except Exception as e:
        logging.exception("plotting error:")
        raise HttpError(
            "the Node-RED result could not be plotted. Maybe wrong data format for the plot type? Check result table: {}".format(
                str(e)), 500)

@app.route("/api_analytics/nrsources", methods=["GET"])
def getNodeRedEnpointList():
    accountServer.assert_token_tenant_validity(request)
    n = requests.get(configuration.nodered_url + "/flows").json()
    sources = []
    for s in n:
        # Node-RED has a strange API... we can't reconstruct node/flow relationships...
        # if ('tab' == s['type'] and 'label' in s):
        # 	thisFlowName = s['label'] + "->"
        if ('http in' == s['type'] and 'url' in s):
            sources.append({"url": s['url'], "name": s['name']})
    return Response(json.dumps(sources), mimetype="application/json")

###############################################################################
# bokeh support functions
###############################################################################
class FixedTickFormatter(TickFormatter):
    """
    Class used to allow custom axis tick labels on a bokeh chart
    Extends bokeh.model.formatters.TickFormatter
    """
    labels = Dict(Int, String, help="""
            A mapping of integer ticks values to their labels.
            """)
    COFFEESCRIPT = """
        import {_} from "underscore"
        import {Model} from "model"
        import * as p from "core/properties"
        export class FixedTickFormatter extends Model
          type: 'FixedTickFormatter'
          @define {
            labels: [ p.Any ]
          }
          doFormat: (ticks) ->
                labels = @labels
                return (labels[tick] ? "" for tick in ticks)
    """

    __implementation__ = CoffeeScript(COFFEESCRIPT)

def __col_to_label_dict(col, offset=1):
    r = dict()
    for k, v in dict(col).items():
        r[(int(k) * offset)] = v
    return r

###############################################################################
# Plots
###############################################################################

###############################################################################
# WORKING
###############################################################################
def bokeh_plot_line(data, nrDataSource, logScale="linear"):
    title = "Line graph: " + nrDataSource['name']
    value_col_names = [d for d in data.columns[1:]]
    # print(data)

    # small_palettes only works with 2+ items; so we increase size by two
    #   and later reduce back down
    colors = small_palettes['Dark2'][len(value_col_names) + 2]
    colors = colors[:len(value_col_names)]

    plot = figure(plot_width=1000, plot_height=600, y_axis_type=logScale)
    plot.title.text = title
    plot.yaxis.axis_label = "values"
    plot.xaxis.axis_label = data.columns[0]

    for (col_name, color) in zip(value_col_names, colors):
        plot.line(data.index, data[col_name], name=col_name, legend=col_name, color=color, line_width=3)

    plot.xaxis[0].formatter = FixedTickFormatter(labels=__col_to_label_dict(data[data.columns[0]]))
    plot.xaxis[0].ticker.desired_num_ticks = len(data[data.columns[0]])
    plot.xaxis[0].major_label_orientation = pi / 4

    script, div = components(plot, resources=None, wrap_script=False, wrap_plot_info=True)
    js = EMPTY.js_raw[0] + script

    return (js, div)

def bokeh_plot_bar(data, nrDataSource, logScale="linear"):
    title = "Bar graph: " + nrDataSource['name']
    value_col_names = [d for d in data.columns[1:]]
    # print(data)

    # small_palettes only works with 2+ items; so we increase size by two
    #   and later reduce back down
    colors = small_palettes['Dark2'][len(value_col_names) + 2]
    colors = colors[:len(value_col_names)]

    plot = figure(plot_width=1000, plot_height=600, y_axis_type=logScale)
    plot.title.text = title
    plot.yaxis.axis_label = "values"
    plot.xaxis.axis_label = data.columns[0]

    num_series = len(value_col_names)
    max_idx = len(data.index) * (num_series + 1)
    for (col_name, color, idx) in zip(value_col_names, colors, range(0, num_series)):
        this_index = RangeIndex(start=idx, stop=max_idx, step=num_series + 1)
        plot.vbar(x=this_index, width=0.8, top=data[col_name], name=col_name, legend=col_name, color=color)

    plot.xaxis[0].formatter = FixedTickFormatter(
        labels=__col_to_label_dict(data[data.columns[0]], offset=num_series + 1))
    plot.xaxis[0].ticker.desired_num_ticks = max_idx
    plot.xaxis[0].major_label_orientation = pi / 4

    script, div = components(plot, resources=None, wrap_script=False, wrap_plot_info=True)
    js = EMPTY.js_raw[0] + script

    return (js, div)

def bokeh_plot_pie(data, nrDataSource):
    value_col_names = [d for d in data.columns[1:]]
    # print(data)

    plot = Donut(data, label=['abbr', 'medal'], values='medal_count',
                 text_font_size='8pt', color=small_palettes['Dark2'])

    script, div = components(plot, resources=None, wrap_script=False, wrap_plot_info=True)

    return (script, div)

###############################################################################
# nodered integration
###############################################################################

def getListOfKeys(d):
    keys = []
    for k in d.keys():
        keys.append(k)
    return keys

def getDataFromNodeRed(nrDataSource):
    url = configuration.nodered_url + nrDataSource['url']
    logging.info("getting data from node red at: {}".format(url))
    r = requests.get(url)
    if r.status_code == 404:
        raise HttpError("the Node-RED data source is not reachable: {}".format(url), 420)
    try:
        data = json.JSONDecoder(object_pairs_hook=collections.OrderedDict).decode(r.content.decode())
        dataKeys = getListOfKeys(data[0])
        df = pandas.DataFrame(data, columns=dataKeys)
        df[dataKeys[0]] = df[dataKeys[0]].map(lambda x: str(x)[:20])
        return df

    except:
        logging.exception("JSON parse error:")
        raise HttpError("the Node-RED result is no valid JSON", 500)

###############################################################################
# Database connection
###############################################################################
def __get_db_connection_for_tenant(tenant):
    d = configuration.metadata_warehouse_endpoint.copy()
    d["database"] = d["database"].format(tenant)
    return d

@app.route("/api_analytics/tablestructure", methods=["GET"])
def getTableStructure():
    # check if user is logged in
    t = accountServer.get_and_assert_tenant_from_request(request)
    d = __get_db_connection_for_tenant(t)
    # Establish connection to PostgreSQL database
    # conn = sqlite3.connect("/tmp/metadata.sqlite") #SQLITE
    with psycopg2.connect(**d) as conn:
        with conn.cursor() as cursor:
            # Retrieve all table names
            # cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name ASC") #SQLITE
            cursor.execute("SELECT table_name FROM information_schema.tables WHERE table_schema=%s;", ("public",))
            tableNames = cursor.fetchall()

            tableData = {}

            # Retrieve the column names and first 5 rows for each table
            for table in tableNames:
                # cursor.execute("PRAGMA table_info(" + table[0] + ")") #SQLITE
                cursor.execute("SELECT column_name,data_type FROM information_schema.columns WHERE table_name = %s;",
                               (table[0],))
                columnNames = cursor.fetchall()

                # cursor.execute("SELECT * FROM " + table[0] + " LIMIT 5") #SQLITE
                cursor.execute("SELECT * FROM {} LIMIT 5".format(table[0]))
                rowEntries = cursor.fetchall()

                # Dictionary which combines column names and row entries
                columnStructure = {}

                # format the column names and types
                columnList = ["{} ({})".format(c[0], c[1]) for c in columnNames]

                # Populate final dictionary table Data
                columnStructure['columnNames'] = columnList
                columnStructure['rowEntries'] = rowEntries
                tableData[table[0]] = columnStructure

            return Response(json.dumps(tableData), mimetype="application/json")
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Take N students and split them up into G groups, A times (for different activities), so that no two students are in the same group more than once"""

import math
import pprint

ORDINAL_SUFFIXES = ["th", "st", "nd", "rd"] + ["th"]*16

def ordinal(n):
    return "%d%s" % (n, ORDINAL_SUFFIXES[n%20])

class PartitionSet(object):
    def __init__(self, N, S):
        """Sets up a partition set for N students divided into groups of S"""
        self.N = N
        self.S = S
        self.G = int(math.ceil(N/S))
        self.partitions = []

    def add_partition(self, partition):
        self.partitions.append(partition)

    def count_overlaps(self):
        worked_with = {}
        for partition in self.partitions:
            for pp in partition:
                for n in pp:
                    so_far = worked_with.setdefault(n, {})
                    for i in pp.difference({n}):
                        so_far.setdefault(i, 0)
                        so_far[i] += 1
        return worked_with

    def get_duplicates(self):
        duplicates = {n: {k: v-1 for k, v in together.items() if v > 1} for n, together in self.count_overlaps().items()}
        return {n: dups for n, dups in duplicates.items() if dups}

    def get_missing(self):
        for p, partition in enumerate(self.partitions):
            missing = set(range(self.N)).difference(set().union(*partition))
            if missing:
                yield p, missing

    def get_badly_sized(self):
        for p, partition in enumerate(self.partitions):
            l = [len(pp) for pp in partition]
            if set(l) != {self.S}:
                yield p, l

    def show_partition(self, partition):
        print(" ".join(",".join("%2d" % n for n in sorted(pp)) for pp in sorted(partition, key=min)))

    def show_and_test(self):
        for partition in self.partitions:
            self.show_partition(partition)

        duplicates = self.get_duplicates()
        missing = dict(self.get_missing())
        badly_sized = dict(self.get_badly_sized())
        if duplicates:
            print ("Duplicates:")
            pprint.pprint(duplicates)
        if missing:
            print ("Missing:")
            pprint.pprint(missing)
        if badly_sized:
            print ("Badly Sized:")
            pprint.pprint(badly_sized)

class PartitionMaker(PartitionSet):
    def __init__(self, N, S):
        super(PartitionMaker, self).__init__(N, S)
        self.worked_with = {}

    def add_partition(self, partition):
        super(PartitionMaker, self).add_partition(partition)
        for group in partition:
            for n in group:
                self.worked_with.setdefault(n, set()).update(group.difference({n}))

    def add_initial_partition(self):
        self.add_partition([{n for n in range(self.N) if n/self.S == g} for g in range(self.G)])

    def find_next_partition(self):
        """simple partition generation that just slots the first things it can find in place"""
        partition = [set() for g in range(self.G)]
        for n in range(self.N):
            avoid = self.worked_with.get(n, set())
            for group in partition:
                if len(group) < self.S and not group.intersection(avoid):
                    group.add(n)
                    partition.sort(key=len)
                    break
            else:
                self.show_and_test()
                self.show_partition(partition)
                raise ValueError("No place for %d in %s partition" % (n, ordinal(len(self.partitions)+1)))
        self.add_partition(partition)

def simple_spread(N, S):
    G = int(math.ceil(N/S)) # number of groups
    for a in [1, 5, 7, 11, 17, 29]:
        map_to_group = [(n, ((a * n) % N)//S) for n in range(N)]
        partition = [{n for n, gg in map_to_group if gg == g} for g in range(G)]
        yield partition

if __name__ == '__main__':
    A = 8   # number of activities
    P = PartitionMaker(52, 4)
    for partition in simple_spread(52, 4):
        P.add_partition(partition)
    for a in range(A-len(P.partitions)):
        P.find_next_partition()
    # P = PartitionSet(52, 4)
    P.show_and_test()

<commit_msg>Switch to trying things in a random order, and not using the original naive algorithm I made<commit_after>#!/usr/bin/env python

"""Take N students and split them up into G groups, A times (for different activities), so that no two students are in the same group more than once"""

import math
import pprint
import random

ORDINAL_SUFFIXES = ["th", "st", "nd", "rd"] + ["th"]*16

def ordinal(n):
    return "%d%s" % (n, ORDINAL_SUFFIXES[n%20])

class PartitionSet(object):
    def __init__(self, N, S):
        """Sets up a partition set for N students divided into groups of S"""
        self.N = N
        self.S = S
        self.G = int(math.ceil(N/S))
        self.partitions = []

    def add_partition(self, partition):
        self.partitions.append(partition)

    def count_overlaps(self):
        worked_with = {}
        for partition in self.partitions:
            for pp in partition:
                for n in pp:
                    so_far = worked_with.setdefault(n, {})
                    for i in pp.difference({n}):
                        so_far.setdefault(i, 0)
                        so_far[i] += 1
        return worked_with

    def get_duplicates(self):
        duplicates = {n: {k: v-1 for k, v in together.items() if v > 1} for n, together in self.count_overlaps().items()}
        return {n: dups for n, dups in duplicates.items() if dups}

    def get_missing(self):
        for p, partition in enumerate(self.partitions):
            missing = set(range(self.N)).difference(set().union(*partition))
            if missing:
                yield p, missing

    def get_badly_sized(self):
        for p, partition in enumerate(self.partitions):
            l = [len(pp) for pp in partition]
            if set(l) != {self.S}:
                yield p, l

    def show_partition(self, partition):
        print(" ".join(",".join("%2d" % n for n in sorted(pp)) for pp in sorted(partition, key=min)))

    def show_and_test(self):
        for partition in self.partitions:
            self.show_partition(partition)

        duplicates = self.get_duplicates()
        missing = dict(self.get_missing())
        badly_sized = dict(self.get_badly_sized())
        if duplicates:
            print ("Duplicates:")
            pprint.pprint(duplicates)
        if missing:
            print ("Missing:")
            pprint.pprint(missing)
        if badly_sized:
            print ("Badly Sized:")
            pprint.pprint(badly_sized)
        if not (duplicates or missing or badly_sized):
            print ("Valid solution")

class PartitionMaker(PartitionSet):
    def __init__(self, N, S):
        super(PartitionMaker, self).__init__(N, S)
        self.worked_with = {}

    def add_partition(self, partition):
        super(PartitionMaker, self).add_partition(partition)
        for group in partition:
            for n in group:
                self.worked_with.setdefault(n, set()).update(group.difference({n}))

    def add_initial_partition(self):
        self.add_partition([{n for n in range(self.N) if n/self.S == g} for g in range(self.G)])

    def find_next_partition(self, n_sequence=None):
        """simple partition generation that just slots the given numbers into the first place it can find for them"""
        partition = [set() for g in range(self.G)]
        n_sequence = n_sequence or range(self.N)
        for n in n_sequence:
            avoid = self.worked_with.get(n, set())
            for group in partition:
                if len(group) < self.S and not group.intersection(avoid):
                    group.add(n)
                    partition.sort(key=len)
                    break
            else:
                # self.show_and_test()
                # self.show_partition(partition)
                raise ValueError("No place for %d in %s partition" % (n, ordinal(len(self.partitions)+1)))
        self.add_partition(partition)

    def try_random_order(self):
        for i in range(100):
            n_sequence = range(self.N)
            random.shuffle(n_sequence)
            try:
                self.find_next_partition(n_sequence)
                print "!"
                return
            except ValueError, e:
                print "x",

def simple_spread(N, S):
    G = int(math.ceil(N/S)) # number of groups
    for a in [1, 5, 7, 11, 17, 29]:
        map_to_group = [(n, ((a * n) % N)//S) for n in range(N)]
        partition = [{n for n, gg in map_to_group if gg == g} for g in range(G)]
        yield partition

if __name__ == '__main__':
    A = 8   # number of activities
    P = PartitionMaker(52, 4)
    # for partition in simple_spread(52, 4):
    #     P.add_partition(partition)
    for a in range(A-len(P.partitions)+1):
        P.try_random_order()
    # P = PartitionSet(52, 4)
    P.show_and_test()

<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Take N students and split them up into G groups, A times (for different activities), so that no two students are in the same group more than once"""

# Copyright 2014 David Fraser <https://github.com/davidfraser/> and Matthew Hampton <https://github.com/matthewhampton/>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import math
import pprint
import random
import csv

ORDINAL_SUFFIXES = ["th", "st", "nd", "rd"] + ["th"]*16

def ordinal(n):
    return "%d%s" % (n, ORDINAL_SUFFIXES[n%20])

rand_instance  = random.Random()

class PartitionSet(object):
    def __init__(self, N, S, students, leaders):
        """Sets up a partition set for N students divided into groups of S"""
        self.N = N
        self.S = S
        self.G = int(math.ceil(N/S))
        self.partitions = []
        self.students = students
        self.leaders = leaders

    def add_partition(self, partition):
        self.partitions.append(partition)

    def count_overlaps(self):
        worked_with = {}
        for partition in self.partitions:
            for pp in partition:
                for n in pp:
                    so_far = worked_with.setdefault(n, {})
                    for i in pp.difference({n}):
                        so_far.setdefault(i, 0)
                        so_far[i] += 1
        return worked_with

    def get_duplicates(self):
        duplicates = {n: {k: v-1 for k, v in together.items() if v > 1} for n, together in self.count_overlaps().items()}
        return {n: dups for n, dups in duplicates.items() if dups}

    def get_missing(self):
        for p, partition in enumerate(self.partitions):
            missing = set(range(self.N)).difference(set().union(*partition))
            if missing:
                yield p, missing

    def get_badly_sized(self):
        for p, partition in enumerate(self.partitions):
            l = [len(pp) for pp in partition]
            if set(l) != {self.S}:
                yield p, l

    def get_leaders_with_duplicate_students(self):
        leaders_kids = {}
        for p, partition in enumerate(self.partitions):
            for i, group in enumerate(partition):
                kidset = leaders_kids.setdefault(i, set())
                for n in group:
                    if n in kidset:
                        yield self.leaders[i], self.students[n]
                    kidset.add(n)

    def get_groups_with_only_one_gender(self):
        for p, partition in enumerate(self.partitions):
            for i, group in enumerate(partition):
                genders = set([self.students[gn]['Gender'] for gn in group])
                if len(genders) < 2:
                    yield self.group_to_string(i, group)

    def group_to_string(self, leader_num, group):
        return '%(First Name)s %(Surname)s: ' % self.leaders[leader_num] + ", ".join('%(First Name)s %(Surname)s (%(Gender)s)' % self.students[n] for n in group)

    def group_to_row(self, leader_num, group):
        return ['%(First Name)s %(Surname)s' % self.leaders[leader_num]] + ['%(First Name)s %(Surname)s' % self.students[n] for n in group]

    def show_partition(self, partition):
        print("\n".join(self.group_to_string(i, pp) for i, pp in enumerate(partition)))

    def save_de_rangement(self):
        with open('derangement.csv', 'wb') as f:
            csv_file = csv.writer(f)
            for p, partition in enumerate(self.partitions):
                csv_file.writerow(['Exercise %02d'%(p+1)] + ['']*self.S)
                for i, group in enumerate(partition):
                    csv_file.writerow(self.group_to_row(i, group))
                csv_file.writerow(['']*(self.S+1))

    def show_and_test(self):

        for i, partition in enumerate(self.partitions):
            print '\nExercise %02d\n' % (i+1)
            self.show_partition(partition)

        duplicates = self.get_duplicates()
        missing = dict(self.get_missing())
        badly_sized = dict(self.get_badly_sized())
        leaders_with_duplicates = list(self.get_leaders_with_duplicate_students())
        groups_with_one_gender = list(self.get_groups_with_only_one_gender())
        if duplicates:
            print ("Duplicates:")
            pprint.pprint(duplicates)
        if missing:
            print ("Missing:")
            pprint.pprint(missing)
        if badly_sized:
            print ("Badly Sized:")
            pprint.pprint(badly_sized)
        if leaders_with_duplicates:
            print ("Leaders with duplicates:")
            pprint.pprint(leaders_with_duplicates)
        if groups_with_one_gender:
            print ("Groups with one gender:")
            pprint.pprint(groups_with_one_gender)
        if not (duplicates or missing or badly_sized or leaders_with_duplicates):
            print ("Valid solution")
            return True
        return False

class PartitionMaker(PartitionSet):
    def __init__(self, N, S, students, leaders):
        super(PartitionMaker, self).__init__(N, S, students, leaders)
        self.worked_with = {}
        self.worked_with_leaders = {}

    def add_partition(self, partition):
        super(PartitionMaker, self).add_partition(partition)
        for i, group in enumerate(partition):
            for n in group:
                self.worked_with.setdefault(n, set()).update(group.difference({n}))
                a = self.worked_with_leaders.setdefault(n, set())
                a.add(i)

    def add_initial_partition(self):
        self.add_partition([{n for n in range(self.N) if n/self.S == g} for g in range(self.G)])

    def find_next_partition(self, n_sequence=None):
        """simple partition generation that just slots the given numbers into the first place it can find for them"""
        partition = [set() for g in range(self.G)]
        n_sequence = n_sequence or range(self.N)
        for n in n_sequence:
            avoid = self.worked_with.get(n, set())
            avoid_leader = self.worked_with_leaders.get(n, set())
            for i, group in enumerate(sorted(partition, key=len)):
                group = partition[i]
                genders = set([self.students[gn]['Gender'] for gn in group])
                if (len(group) < (self.S-1) or (len(group) < self.S and (len(genders)>1 or self.students[n]['Gender'] not in genders))) and not group.intersection(avoid) and not i in avoid_leader:
                    group.add(n)
                    break
            else:
                # self.show_and_test()
                # self.show_partition(partition)
                raise ValueError("No place for %d in %s partition" % (n, ordinal(len(self.partitions)+1)))
        self.add_partition(partition)

    def try_random_order(self, max_tries=10000):
        for i in range(max_tries):
            n_sequence = range(self.N)
            rand_instance.shuffle(n_sequence)
            try:
                self.find_next_partition(n_sequence)
                print "! (%d tries)" % i
                return True
            except ValueError, e:
                if i % 100 == 0:
                    print ".",
        else:
            raise ValueError("Could not find solution to add to %d items" % len(self.partitions))

def simple_spread(N, S):
    G = int(math.ceil(N/S)) # number of groups
    for a in [1, 5, 7, 11, 17, 29]:
        map_to_group = [(n, ((a * n) % N)//S) for n in range(N)]
        partition = [{n for n, gg in map_to_group if gg == g} for g in range(G)]
        yield partition

if __name__ == '__main__':

    with open('students.csv') as f:
        students = list(sorted(csv.DictReader(f), key=lambda s:(s['Gender'],s['Race'])))

    with open('leaders.csv') as f:
        leaders = list(csv.DictReader(f))

    if len(students) % len(leaders) != 0:
        raise ValueError("Students must be an exact multiple of the number of leaders")
    A = 6   # number of activities
    P = PartitionMaker(len(students), len(students)/len(leaders), students, leaders)
    for a in range(A-len(P.partitions)+1):
        P.try_random_order()
    print
    P.show_and_test()
    P.save_de_rangement()

<commit_msg>Implement way to go back if the search is proving fruitless, and a little heuristic about how far to back out (which could use some improvement)<commit_after>#!/usr/bin/env python

"""Take N students and split them up into G groups, A times (for different activities), so that no two students are in the same group more than once"""

# Copyright 2014 David Fraser <https://github.com/davidfraser/> and Matthew Hampton <https://github.com/matthewhampton/>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import math
import pprint
import random
import csv
import sys

ORDINAL_SUFFIXES = ["th", "st", "nd", "rd"] + ["th"]*16

def ordinal(n):
    return "%d%s" % (n, ORDINAL_SUFFIXES[n%20])

rand_instance  = random.Random()

class PartitionSet(object):
    def __init__(self, N, S, students, leaders):
        """Sets up a partition set for N students divided into groups of S"""
        self.N = N
        self.S = S
        self.G = int(math.ceil(N/S))
        self.partitions = []
        self.students = students
        self.leaders = leaders

    def add_partition(self, partition):
        self.partitions.append(partition)

    def count_overlaps(self):
        worked_with = {}
        for partition in self.partitions:
            for pp in partition:
                for n in pp:
                    so_far = worked_with.setdefault(n, {})
                    for i in pp.difference({n}):
                        so_far.setdefault(i, 0)
                        so_far[i] += 1
        return worked_with

    def get_duplicates(self):
        duplicates = {n: {k: v-1 for k, v in together.items() if v > 1} for n, together in self.count_overlaps().items()}
        return {n: dups for n, dups in duplicates.items() if dups}

    def get_missing(self):
        for p, partition in enumerate(self.partitions):
            missing = set(range(self.N)).difference(set().union(*partition))
            if missing:
                yield p, missing

    def get_badly_sized(self):
        for p, partition in enumerate(self.partitions):
            l = [len(pp) for pp in partition]
            if set(l) != {self.S}:
                yield p, l

    def get_leaders_with_duplicate_students(self):
        leaders_kids = {}
        for p, partition in enumerate(self.partitions):
            for i, group in enumerate(partition):
                kidset = leaders_kids.setdefault(i, set())
                for n in group:
                    if n in kidset:
                        yield self.leaders[i], self.students[n]
                    kidset.add(n)

    def get_groups_with_only_one_gender(self):
        for p, partition in enumerate(self.partitions):
            for i, group in enumerate(partition):
                genders = set([self.students[gn]['Gender'] for gn in group])
                if len(genders) < 2:
                    yield self.group_to_string(i, group)

    def group_to_string(self, leader_num, group):
        return '%(First Name)s %(Surname)s: ' % self.leaders[leader_num] + ", ".join('%(First Name)s %(Surname)s (%(Gender)s)' % self.students[n] for n in group)

    def group_to_row(self, leader_num, group):
        return ['%(First Name)s %(Surname)s' % self.leaders[leader_num]] + ['%(First Name)s %(Surname)s' % self.students[n] for n in group]

    def show_partition(self, partition):
        print("\n".join(self.group_to_string(i, pp) for i, pp in enumerate(partition)))

    def save_de_rangement(self):
        with open('derangement.csv', 'wb') as f:
            csv_file = csv.writer(f)
            for p, partition in enumerate(self.partitions):
                csv_file.writerow(['Exercise %02d'%(p+1)] + ['']*self.S)
                for i, group in enumerate(partition):
                    csv_file.writerow(self.group_to_row(i, group))
                csv_file.writerow(['']*(self.S+1))

    def show_and_test(self):

        for i, partition in enumerate(self.partitions):
            print '\nExercise %02d\n' % (i+1)
            self.show_partition(partition)

        duplicates = self.get_duplicates()
        missing = dict(self.get_missing())
        badly_sized = dict(self.get_badly_sized())
        leaders_with_duplicates = list(self.get_leaders_with_duplicate_students())
        groups_with_one_gender = list(self.get_groups_with_only_one_gender())
        if duplicates:
            print ("Duplicates:")
            pprint.pprint(duplicates)
        if missing:
            print ("Missing:")
            pprint.pprint(missing)
        if badly_sized:
            print ("Badly Sized:")
            pprint.pprint(badly_sized)
        if leaders_with_duplicates:
            print ("Leaders with duplicates:")
            pprint.pprint(leaders_with_duplicates)
        if groups_with_one_gender:
            print ("Groups with one gender:")
            pprint.pprint(groups_with_one_gender)
        if not (duplicates or missing or badly_sized or leaders_with_duplicates):
            print ("Valid solution")
            return True
        return False

class PartitionMaker(PartitionSet):
    def __init__(self, N, S, students, leaders):
        super(PartitionMaker, self).__init__(N, S, students, leaders)
        self.worked_with = {}
        self.worked_with_leaders = {}

    def add_partition(self, partition):
        super(PartitionMaker, self).add_partition(partition)
        for i, group in enumerate(partition):
            for n in group:
                self.worked_with.setdefault(n, set()).update(group.difference({n}))
                a = self.worked_with_leaders.setdefault(n, set())
                a.add(i)

    def clear(self):
        self.partitions = []
        self.worked_with = {}
        self.worked_with_leaders = {}

    def pop_partition(self):
        new_partitions = self.partitions[:-1]
        self.clear()
        for partition in new_partitions:
            self.add_partition(partition)

    def add_initial_partition(self):
        self.add_partition([{n for n in range(self.N) if n/self.S == g} for g in range(self.G)])

    def find_next_partition(self, n_sequence=None):
        """simple partition generation that just slots the given numbers into the first place it can find for them"""
        partition = [set() for g in range(self.G)]
        n_sequence = n_sequence or range(self.N)
        for n in n_sequence:
            avoid = self.worked_with.get(n, set())
            avoid_leader = self.worked_with_leaders.get(n, set())
            for i, group in enumerate(sorted(partition, key=len)):
                group = partition[i]
                genders = set([self.students[gn]['Gender'] for gn in group])
                if (len(group) < (self.S-1) or (len(group) < self.S and (len(genders)>1 or self.students[n]['Gender'] not in genders))) and not group.intersection(avoid) and not i in avoid_leader:
                    group.add(n)
                    break
            else:
                # self.show_and_test()
                # self.show_partition(partition)
                raise ValueError("No place for %d in %s partition" % (n, ordinal(len(self.partitions)+1)))
        self.add_partition(partition)

    def try_random_order(self, max_tries=5000):
        for i in range(max_tries):
            n_sequence = range(self.N)
            rand_instance.shuffle(n_sequence)
            try:
                self.find_next_partition(n_sequence)
                print "! (%d tries)" % i
                return True
            except ValueError, e:
                if i % 100 == 0:
                    sys.stdout.write(".")
                    sys.stdout.flush()
        else:
            raise ValueError("Could not find solution to add to %d items" % len(self.partitions))

def simple_spread(N, S):
    G = int(math.ceil(N/S)) # number of groups
    for a in [1, 5, 7, 11, 17, 29]:
        map_to_group = [(n, ((a * n) % N)//S) for n in range(N)]
        partition = [{n for n, gg in map_to_group if gg == g} for g in range(G)]
        yield partition

if __name__ == '__main__':

    with open('students.csv') as f:
        students = list(sorted(csv.DictReader(f), key=lambda s:(s['Gender'],s['Race'])))

    with open('leaders.csv') as f:
        leaders = list(csv.DictReader(f))

    if len(students) % len(leaders) != 0:
        raise ValueError("Students must be an exact multiple of the number of leaders")
    A = 6   # number of activities
    P = PartitionMaker(len(students), len(students)/len(leaders), students, leaders)
    repeated_failures = 0
    while len(P.partitions) < A:
        try:
            P.try_random_order()
            repeated_failures = 0
        except ValueError:
            repeated_failures += 1
            items_to_pop = 1 + (repeated_failures // 5)
            print ("Going back %d..." % items_to_pop)
            for i in range(items_to_pop):
                P.pop_partition()
    print
    P.show_and_test()
    P.save_de_rangement()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

""" Models registries.

"""
import logging
import threading

import openerp.sql_db
import openerp.osv.orm
import openerp.cron
import openerp.tools
import openerp.modules.db
import openerp.tools.config

_logger = logging.getLogger(__name__)

class Registry(object):
    """ Model registry for a particular database.

    The registry is essentially a mapping between model names and model
    instances. There is one registry instance per database.

    """

    def __init__(self, db_name):
        self.models = {} # model name/model instance mapping
        self._sql_error = {}
        self._store_function = {}
        self._init = True
        self._init_parent = {}
        self.db_name = db_name
        self.db = openerp.sql_db.db_connect(db_name)

        # Inter-process signaling (used only when openerp.multi_process is True):
        # The `base_registry_signaling` sequence indicates the whole registry
        # must be reloaded.
        # The `base_cache_signaling sequence` indicates all caches must be
        # invalidated (i.e. cleared).
        self.base_registry_signaling_sequence = 1
        self.base_cache_signaling_sequence = 1

        # Flag indicating if at least one model cache has been cleared.
        # Useful only in a multi-process context.
        self._any_cache_cleared = False

        cr = self.db.cursor()
        has_unaccent = openerp.modules.db.has_unaccent(cr)
        if openerp.tools.config['unaccent'] and not has_unaccent:
            _logger.warning("The option --unaccent was given but no unaccent() function was found in database.")
        self.has_unaccent = openerp.tools.config['unaccent'] and has_unaccent
        cr.close()

    def do_parent_store(self, cr):
        for o in self._init_parent:
            self.get(o)._parent_store_compute(cr)
        self._init = False

    def obj_list(self):
        """ Return the list of model names in this registry."""
        return self.models.keys()

    def add(self, model_name, model):
        """ Add or replace a model in the registry."""
        self.models[model_name] = model

    def get(self, model_name):
        """ Return a model for a given name or None if it doesn't exist."""
        return self.models.get(model_name)

    def __getitem__(self, model_name):
        """ Return a model for a given name or raise KeyError if it doesn't exist."""
        return self.models[model_name]

    def load(self, cr, module):
        """ Load a given module in the registry.

        At the Python level, the modules are already loaded, but not yet on a
        per-registry level. This method populates a registry with the given
        modules, i.e. it instanciates all the classes of a the given module
        and registers them in the registry.

        """

        res = []

        # Instantiate registered classes (via the MetaModel automatic discovery
        # or via explicit constructor call), and add them to the pool.
        for cls in openerp.osv.orm.MetaModel.module_to_models.get(module.name, []):
            res.append(cls.create_instance(self, cr))

        return res

    def schedule_cron_jobs(self):
        """ Make the cron thread care about this registry/database jobs.
        This will initiate the cron thread to check for any pending jobs for
        this registry/database as soon as possible. Then it will continuously
        monitor the ir.cron model for future jobs. See openerp.cron for
        details.
        """
        openerp.cron.schedule_wakeup(openerp.cron.WAKE_UP_NOW, self.db.dbname)

    def clear_caches(self):
        """ Clear the caches
        This clears the caches associated to methods decorated with
        ``tools.ormcache`` or ``tools.ormcache_multi`` for all the models.
        """
        for model in self.models.itervalues():
            model.clear_caches()
        # Special case for ir_ui_menu which does not use openerp.tools.ormcache.
        ir_ui_menu = self.models.get('ir.ui.menu')
        if ir_ui_menu:
            ir_ui_menu.clear_cache()

    # Useful only in a multi-process context.
    def reset_any_cache_cleared(self):
        self._any_cache_cleared = False

    # Useful only in a multi-process context.
    def any_cache_cleared(self):
        return self._any_cache_cleared

class RegistryManager(object):
    """ Model registries manager.

        The manager is responsible for creation and deletion of model
        registries (essentially database connection/model registry pairs).

    """
    # Mapping between db name and model registry.
    # Accessed through the methods below.
    registries = {}
    registries_lock = threading.RLock()

    @classmethod
    def get(cls, db_name, force_demo=False, status=None, update_module=False,
            pooljobs=True):
        """ Return a registry for a given database name."""
        try:
            return cls.registries[db_name]
        except KeyError:
            return cls.new(db_name, force_demo, status,
                           update_module, pooljobs)

    @classmethod
    def new(cls, db_name, force_demo=False, status=None,
            update_module=False, pooljobs=True):
        """ Create and return a new registry for a given database name.

        The (possibly) previous registry for that database name is discarded.

        """
        import openerp.modules
        with cls.registries_lock:
            registry = Registry(db_name)

            # Initializing a registry will call general code which will in turn
            # call registries.get (this object) to obtain the registry being
            # initialized. Make it available in the registries dictionary then
            # remove it if an exception is raised.
            cls.delete(db_name)
            cls.registries[db_name] = registry
            try:
                # This should be a method on Registry
                openerp.modules.load_modules(registry.db, force_demo, status, update_module)
            except Exception:
                del cls.registries[db_name]
                raise

            cr = registry.db.cursor()
            try:
                registry.do_parent_store(cr)
                registry.get('ir.actions.report.xml').register_all(cr)
                cr.commit()
            finally:
                cr.close()

        if pooljobs:
            registry.schedule_cron_jobs()

        return registry

    @classmethod
    def delete(cls, db_name):
        """Delete the registry linked to a given database.

        This also cleans the associated caches. For good measure this also
        cancels the associated cron job. But please note that the cron job can
        be running and take some time before ending, and that you should not
        remove a registry if it can still be used by some thread. So it might
        be necessary to call yourself openerp.cron.Agent.cancel(db_name) and
        and join (i.e. wait for) the thread.
        """
        with cls.registries_lock:
            if db_name in cls.registries:
                cls.registries[db_name].clear_caches()
                del cls.registries[db_name]
                openerp.cron.cancel(db_name)

    @classmethod
    def delete_all(cls):
        """Delete all the registries. """
        with cls.registries_lock:
            for db_name in cls.registries.keys():
                cls.delete(db_name)

    @classmethod
    def clear_caches(cls, db_name):
        """Clear caches

        This clears the caches associated to methods decorated with
        ``tools.ormcache`` or ``tools.ormcache_multi`` for all the models
        of the given database name.

        This method is given to spare you a ``RegistryManager.get(db_name)``
        that would loads the given database if it was not already loaded.
        """
        with cls.registries_lock:
            if db_name in cls.registries:
                cls.registries[db_name].clear_caches()

    @classmethod
    def check_registry_signaling(cls, db_name):
        if openerp.multi_process and db_name in cls.registries:
            # Check if the model registry must be reloaded (e.g. after the
            # database has been updated by another process).
            registry = cls.get(db_name, pooljobs=False)
            cr = registry.db.cursor()
            registry_reloaded = False
            try:
                cr.execute('SELECT last_value FROM base_registry_signaling')
                r = cr.fetchone()[0]
                if registry.base_registry_signaling_sequence != r:
                    _logger.info("Reloading the model registry after database signaling.")
                    # Don't run the cron in the Gunicorn worker.
                    registry = cls.new(db_name, pooljobs=False)
                    registry.base_registry_signaling_sequence = r
                    registry_reloaded = True
            finally:
                cr.close()

            # Check if the model caches must be invalidated (e.g. after a write
            # occured on another process). Don't clear right after a registry
            # has been reload.
            cr = openerp.sql_db.db_connect(db_name).cursor()
            try:
                cr.execute('SELECT last_value FROM base_cache_signaling')
                r = cr.fetchone()[0]
                if registry.base_cache_signaling_sequence != r and not registry_reloaded:
                    _logger.info("Invalidating all model caches after database signaling.")
                    registry.base_cache_signaling_sequence = r
                    registry.clear_caches()
            finally:
                cr.close()

    @classmethod
    def signal_caches_change(cls, db_name):
        if openerp.multi_process and db_name in cls.registries:
            # Check the registries if any cache has been cleared and signal it
            # through the database to other processes.
            registry = cls.get(db_name, pooljobs=False)
            if registry.any_cache_cleared():
                _logger.info("At least one model cache has been cleare, signaling through the database.")
                cr = registry.db.cursor()
                try:
                    pass
                    # cr.execute("select nextval('base_cache_signaling')")
                    # cls.base_cache_signaling_sequence to = result
                finally:
                    cr.close()
                registry.reset_any_cache_cleared()

    @classmethod
    def signal_registry_change(cls, db_name):
        if openerp.multi_process and db_name in cls.registries:
            registry = cls.get(db_name, pooljobs=False)
            cr = registry.db.cursor()
            r = 1
            try:
                cr.execute("select nextval('base_registry_signaling')")
                r = cr.fetchone()[0]
            finally:
                cr.close()
            registry.base_registry_signaling_sequence = r

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[IMP] gunicorn: uncomment the nextval(base_cache_signaling).<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

""" Models registries.

"""
import logging
import threading

import openerp.sql_db
import openerp.osv.orm
import openerp.cron
import openerp.tools
import openerp.modules.db
import openerp.tools.config

_logger = logging.getLogger(__name__)

class Registry(object):
    """ Model registry for a particular database.

    The registry is essentially a mapping between model names and model
    instances. There is one registry instance per database.

    """

    def __init__(self, db_name):
        self.models = {} # model name/model instance mapping
        self._sql_error = {}
        self._store_function = {}
        self._init = True
        self._init_parent = {}
        self.db_name = db_name
        self.db = openerp.sql_db.db_connect(db_name)

        # Inter-process signaling (used only when openerp.multi_process is True):
        # The `base_registry_signaling` sequence indicates the whole registry
        # must be reloaded.
        # The `base_cache_signaling sequence` indicates all caches must be
        # invalidated (i.e. cleared).
        self.base_registry_signaling_sequence = 1
        self.base_cache_signaling_sequence = 1

        # Flag indicating if at least one model cache has been cleared.
        # Useful only in a multi-process context.
        self._any_cache_cleared = False

        cr = self.db.cursor()
        has_unaccent = openerp.modules.db.has_unaccent(cr)
        if openerp.tools.config['unaccent'] and not has_unaccent:
            _logger.warning("The option --unaccent was given but no unaccent() function was found in database.")
        self.has_unaccent = openerp.tools.config['unaccent'] and has_unaccent
        cr.close()

    def do_parent_store(self, cr):
        for o in self._init_parent:
            self.get(o)._parent_store_compute(cr)
        self._init = False

    def obj_list(self):
        """ Return the list of model names in this registry."""
        return self.models.keys()

    def add(self, model_name, model):
        """ Add or replace a model in the registry."""
        self.models[model_name] = model

    def get(self, model_name):
        """ Return a model for a given name or None if it doesn't exist."""
        return self.models.get(model_name)

    def __getitem__(self, model_name):
        """ Return a model for a given name or raise KeyError if it doesn't exist."""
        return self.models[model_name]

    def load(self, cr, module):
        """ Load a given module in the registry.

        At the Python level, the modules are already loaded, but not yet on a
        per-registry level. This method populates a registry with the given
        modules, i.e. it instanciates all the classes of a the given module
        and registers them in the registry.

        """

        res = []

        # Instantiate registered classes (via the MetaModel automatic discovery
        # or via explicit constructor call), and add them to the pool.
        for cls in openerp.osv.orm.MetaModel.module_to_models.get(module.name, []):
            res.append(cls.create_instance(self, cr))

        return res

    def schedule_cron_jobs(self):
        """ Make the cron thread care about this registry/database jobs.
        This will initiate the cron thread to check for any pending jobs for
        this registry/database as soon as possible. Then it will continuously
        monitor the ir.cron model for future jobs. See openerp.cron for
        details.
        """
        openerp.cron.schedule_wakeup(openerp.cron.WAKE_UP_NOW, self.db.dbname)

    def clear_caches(self):
        """ Clear the caches
        This clears the caches associated to methods decorated with
        ``tools.ormcache`` or ``tools.ormcache_multi`` for all the models.
        """
        for model in self.models.itervalues():
            model.clear_caches()
        # Special case for ir_ui_menu which does not use openerp.tools.ormcache.
        ir_ui_menu = self.models.get('ir.ui.menu')
        if ir_ui_menu:
            ir_ui_menu.clear_cache()

    # Useful only in a multi-process context.
    def reset_any_cache_cleared(self):
        self._any_cache_cleared = False

    # Useful only in a multi-process context.
    def any_cache_cleared(self):
        return self._any_cache_cleared

class RegistryManager(object):
    """ Model registries manager.

        The manager is responsible for creation and deletion of model
        registries (essentially database connection/model registry pairs).

    """
    # Mapping between db name and model registry.
    # Accessed through the methods below.
    registries = {}
    registries_lock = threading.RLock()

    @classmethod
    def get(cls, db_name, force_demo=False, status=None, update_module=False,
            pooljobs=True):
        """ Return a registry for a given database name."""
        try:
            return cls.registries[db_name]
        except KeyError:
            return cls.new(db_name, force_demo, status,
                           update_module, pooljobs)

    @classmethod
    def new(cls, db_name, force_demo=False, status=None,
            update_module=False, pooljobs=True):
        """ Create and return a new registry for a given database name.

        The (possibly) previous registry for that database name is discarded.

        """
        import openerp.modules
        with cls.registries_lock:
            registry = Registry(db_name)

            # Initializing a registry will call general code which will in turn
            # call registries.get (this object) to obtain the registry being
            # initialized. Make it available in the registries dictionary then
            # remove it if an exception is raised.
            cls.delete(db_name)
            cls.registries[db_name] = registry
            try:
                # This should be a method on Registry
                openerp.modules.load_modules(registry.db, force_demo, status, update_module)
            except Exception:
                del cls.registries[db_name]
                raise

            cr = registry.db.cursor()
            try:
                registry.do_parent_store(cr)
                registry.get('ir.actions.report.xml').register_all(cr)
                cr.commit()
            finally:
                cr.close()

        if pooljobs:
            registry.schedule_cron_jobs()

        return registry

    @classmethod
    def delete(cls, db_name):
        """Delete the registry linked to a given database.

        This also cleans the associated caches. For good measure this also
        cancels the associated cron job. But please note that the cron job can
        be running and take some time before ending, and that you should not
        remove a registry if it can still be used by some thread. So it might
        be necessary to call yourself openerp.cron.Agent.cancel(db_name) and
        and join (i.e. wait for) the thread.
        """
        with cls.registries_lock:
            if db_name in cls.registries:
                cls.registries[db_name].clear_caches()
                del cls.registries[db_name]
                openerp.cron.cancel(db_name)

    @classmethod
    def delete_all(cls):
        """Delete all the registries. """
        with cls.registries_lock:
            for db_name in cls.registries.keys():
                cls.delete(db_name)

    @classmethod
    def clear_caches(cls, db_name):
        """Clear caches

        This clears the caches associated to methods decorated with
        ``tools.ormcache`` or ``tools.ormcache_multi`` for all the models
        of the given database name.

        This method is given to spare you a ``RegistryManager.get(db_name)``
        that would loads the given database if it was not already loaded.
        """
        with cls.registries_lock:
            if db_name in cls.registries:
                cls.registries[db_name].clear_caches()

    @classmethod
    def check_registry_signaling(cls, db_name):
        if openerp.multi_process and db_name in cls.registries:
            # Check if the model registry must be reloaded (e.g. after the
            # database has been updated by another process).
            registry = cls.get(db_name, pooljobs=False)
            cr = registry.db.cursor()
            registry_reloaded = False
            try:
                cr.execute('SELECT last_value FROM base_registry_signaling')
                r = cr.fetchone()[0]
                if registry.base_registry_signaling_sequence != r:
                    _logger.info("Reloading the model registry after database signaling.")
                    # Don't run the cron in the Gunicorn worker.
                    registry = cls.new(db_name, pooljobs=False)
                    registry.base_registry_signaling_sequence = r
                    registry_reloaded = True
            finally:
                cr.close()

            # Check if the model caches must be invalidated (e.g. after a write
            # occured on another process). Don't clear right after a registry
            # has been reload.
            cr = openerp.sql_db.db_connect(db_name).cursor()
            try:
                cr.execute('SELECT last_value FROM base_cache_signaling')
                r = cr.fetchone()[0]
                if registry.base_cache_signaling_sequence != r and not registry_reloaded:
                    _logger.info("Invalidating all model caches after database signaling.")
                    registry.base_cache_signaling_sequence = r
                    registry.clear_caches()
                    registry.reset_any_cache_cleared()
            finally:
                cr.close()

    @classmethod
    def signal_caches_change(cls, db_name):
        if openerp.multi_process and db_name in cls.registries:
            # Check the registries if any cache has been cleared and signal it
            # through the database to other processes.
            registry = cls.get(db_name, pooljobs=False)
            if registry.any_cache_cleared():
                _logger.info("At least one model cache has been cleare, signaling through the database.")
                cr = registry.db.cursor()
                r = 1
                try:
                    pass
                    cr.execute("select nextval('base_cache_signaling')")
                    r = cr.fetchone()[0]
                finally:
                    cr.close()
                registry.base_cache_signaling_sequence = r
                registry.reset_any_cache_cleared()

    @classmethod
    def signal_registry_change(cls, db_name):
        if openerp.multi_process and db_name in cls.registries:
            registry = cls.get(db_name, pooljobs=False)
            cr = registry.db.cursor()
            r = 1
            try:
                cr.execute("select nextval('base_registry_signaling')")
                r = cr.fetchone()[0]
            finally:
                cr.close()
            registry.base_registry_signaling_sequence = r

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>#! /usr/bin/env python

'''
This script extracts the SIFT4G annotation (http://sift.bii.a-star.edu.sg/sift4g/AnnotateVariants.html) for a given set of samples according to their genotypes.

http://sift.bii.a-star.edu.sg/sift4g/AnnotateVariants.html

# tab file:

CHROM	POS	REF	sample1	sample2	sample3	sample4
chr_1	2068	T	A	N	T	A
chr_1	2069	A	C	C	T	N

# SIFT4G annotation:

chr_1	2068	T	A	gene1	gene1.g	gene1.g	CDS	SYNONYMOUS	I	I	167	1.000	3.31	85	novel	TOLERATED
chr_1	2068	T	T	gene1	gene1.g	gene1.g	CDS	SYNONYMOUS	I	I	167	1.000	3.31	85	ref	TOLERATED
chr_1	2068	T	G	gene1	gene1.g	gene1.g	CDS	SYNONYMOUS	I	I	167	1.000	3.31	85	novel	TOLERATED
chr_1	2068	T	C	gene1	gene1.g	gene1.g	CDS	NONSYNONYMOUS	I	M	167	0.001	3.31	85	novel	DELETERIOUS
chr_1	2069	A	A	gene1	gene1.g	gene1.g	CDS	SYNONYMOUS	I	I	167	1.000	3.31	85	ref	TOLERATED
chr_1	2069	A	T	gene1	gene1.g	gene1.g	CDS	NONSYNONYMOUS	I	K	167	0.000	3.31	85	novel	DELETERIOUS
chr_1	2069	A	G	gene1	gene1.g	gene1.g	CDS	NONSYNONYMOUS	I	T	167	0.001	3.31	85	novel	DELETERIOUS
chr_1	2069	A	C	gene1	gene1.g	gene1.g	CDS	NONSYNONYMOUS	I	R	167	0.000	3.31	85	novel	DELETERIOUS

# output:

CHROM   POS sample1 sample2 sample3 sample4
chr_1   2068    1.000|TOLERATED NA  1.000|TOLERATED 1.000|TOLERATED
chr_1   2069    0.000|DELETERIOUS   0.000|DELETERIOUS   0.000|DELETERIOUS   NA

Note! Chromosome number must be separated by _.
For example, chr_1 - correct, chr1 - incorrect.

# command:

$ python extractSIFT4Gannotation.py -i test.sift4g -t test.tab -o test.output -s "sample1,sample2,sample3,sample4" -f "SIFT_SCORE,PREDICTION"

#contact:

Dmytro Kryvokhyzha dmytro.kryvokhyzha@evobio.eu

'''

############################# modules #############################

import calls # my custom module

############################# options #############################

parser = calls.MyParser()
parser.add_argument('-i', '--input', help = 'name of the input file', type=str, required=True)
parser.add_argument('-t', '--tab', help = 'tab delimited genotype file', type=str, required=True)
parser.add_argument('-o', '--output', help = 'name of the output file', type=str, required=True)
parser.add_argument('-f', '--fields', help = 'annotation fields to extract. Possible options: CHROM, POS, REF_ALLELE, ALT_ALLELE, TRANSCRIPT_ID, GENE_ID, GENE_NAME, REGION, VARIANT_TYPE, REF_AMINO, ALT_AMINO, AMINO_POS, SIFT_SCORE, SIFT_MEDIAN, NUM_SEQS, dbSNP, SIFT_PREDICTION', type=str, required=True)
parser.add_argument('-s', '--samples', help = 'column names of the samples to process (optional)', type=str, required=False)
args = parser.parse_args()

# check if samples names are given and if all sample names are present in a header
sampleNames = calls.checkSampleNames(args.samples, args.tab)

############################# program #############################

print('Opening the file...')

counter = 0

siftFile = open(args.input, 'r')
annotOptions = siftFile.readline().split()
fieldsIndex = calls.indexSamples([args.fields], annotOptions)

sift_words = siftFile.readline().split()
sift_chr = int(sift_words[0].split('_')[1])
sift_pos = int(sift_words[1])

with open(args.tab) as datafile:
  header_words = datafile.readline().split()

  # index samples
  sampCol = calls.indexSamples(sampleNames, header_words)

  # make output header
  print('Creating the output file...')
  output = open(args.output, 'w')
  ouput_header = header_words[0:2] + calls.selectSamples(sampCol, header_words)
  ouput_headerP = '\t'.join(str(el) for el in ouput_header)
  output.write('%s\n' % ouput_headerP)

############################### perform counting ####################

  for line in datafile:
    words = line.split()
    ch = int(words[0].split('_')[1])
    pos = int(words[1])

    # select samples
    tab_charaters = calls.selectSamples(sampCol, words)

    # find overlap in genomic position
    while (ch > sift_chr) or (ch == sift_chr and pos > sift_pos):
      sift_words = siftFile.readline().split()
      if sift_words == []:
        break
      sift_chr = int(sift_words[0].split('_')[1])
      sift_pos = int(sift_words[1])

    # extract all annotations of the position
    sift_alt = []
    sift_annot = []
    while (ch == sift_chr and pos == sift_pos):
      sift_alt.append(sift_words[3])
      sift_annot.append([sift_words[a] for a in fieldsIndex])
      #print words[0:2], sift_words[0:4] # for debugging
      sift_words = siftFile.readline().split()
      if sift_words == []:
        break
      sift_chr = int(sift_words[0].split('_')[1])
      sift_pos = int(sift_words[1])

    # find overlap in tab genotypes and sift ALT
    tab_annot = []
    for s in tab_charaters:
      if s in sift_alt: # check if genotype is A,T,G,C (not N)
        annotIndex = sift_alt.index(s)
        s_annot = sift_annot[annotIndex]
        s_annotP = '|'.join(str(el) for el in s_annot)
        tab_annot.append(s_annotP)
      else:
        tab_annot.append('NA')

    # output annotation
    tab_annotP = '\t'.join(str(el) for el in tab_annot)
    chr_posP = '\t'.join(str(el) for el in words[0:2])
    output.write('%s\t%s\n' % (chr_posP, tab_annotP))

    # track progress
    counter += 1
    if counter % 100000 == 0:
      print str(counter), "lines processed"

datafile.close()
siftFile.close()
output.close()

print('Done!')
<commit_msg>updated<commit_after>#! /usr/bin/env python

'''
This script extracts the SIFT4G annotation (http://sift.bii.a-star.edu.sg/sift4g/AnnotateVariants.html) for a given set of samples according to their genotypes.

http://sift.bii.a-star.edu.sg/sift4g/AnnotateVariants.html

# tab file:

CHROM	POS	REF	sample1	sample2	sample3	sample4
chr_1	2068	T	A	N	T	A
chr_1	2069	A	C	C	T	N

# SIFT4G annotation:

chr_1	2068	T	A	gene1	gene1.g	gene1.g	CDS	SYNONYMOUS	I	I	167	1.000	3.31	85	novel	TOLERATED
chr_1	2068	T	T	gene1	gene1.g	gene1.g	CDS	SYNONYMOUS	I	I	167	1.000	3.31	85	ref	TOLERATED
chr_1	2068	T	G	gene1	gene1.g	gene1.g	CDS	SYNONYMOUS	I	I	167	1.000	3.31	85	novel	TOLERATED
chr_1	2068	T	C	gene1	gene1.g	gene1.g	CDS	NONSYNONYMOUS	I	M	167	0.001	3.31	85	novel	DELETERIOUS
chr_1	2069	A	A	gene1	gene1.g	gene1.g	CDS	SYNONYMOUS	I	I	167	1.000	3.31	85	ref	TOLERATED
chr_1	2069	A	T	gene1	gene1.g	gene1.g	CDS	NONSYNONYMOUS	I	K	167	0.000	3.31	85	novel	DELETERIOUS
chr_1	2069	A	G	gene1	gene1.g	gene1.g	CDS	NONSYNONYMOUS	I	T	167	0.001	3.31	85	novel	DELETERIOUS
chr_1	2069	A	C	gene1	gene1.g	gene1.g	CDS	NONSYNONYMOUS	I	R	167	0.000	3.31	85	novel	DELETERIOUS

# output:

CHROM   POS sample1 sample2 sample3 sample4
chr_1   2068    1.000|TOLERATED NA  1.000|TOLERATED 1.000|TOLERATED
chr_1   2069    0.000|DELETERIOUS   0.000|DELETERIOUS   0.000|DELETERIOUS   NA

Note! Chromosome number must be separated by _.
For example, chr_1 - correct, chr1 - incorrect.

# command:

$ python extractSIFT4Gannotation.py -i test.sift4g -t test.tab -o test.output -s "sample1,sample2,sample3,sample4" -f "SIFT_SCORE,PREDICTION"

#contact:

Dmytro Kryvokhyzha dmytro.kryvokhyzha@evobio.eu

'''

############################# modules #############################

import calls # my custom module

############################# options #############################

parser = calls.MyParser()
parser.add_argument('-i', '--input', help = 'name of the input file', type=str, required=True)
parser.add_argument('-t', '--tab', help = 'tab delimited genotype file', type=str, required=True)
parser.add_argument('-o', '--output', help = 'name of the output file', type=str, required=True)
parser.add_argument('-f', '--fields', help = 'annotation fields to extract. Possible options: CHROM, POS, REF_ALLELE, ALT_ALLELE, TRANSCRIPT_ID, GENE_ID, GENE_NAME, REGION, VARIANT_TYPE, REF_AMINO, ALT_AMINO, AMINO_POS, SIFT_SCORE, SIFT_MEDIAN, NUM_SEQS, dbSNP, SIFT_PREDICTION', type=str, required=True)
parser.add_argument('-s', '--samples', help = 'column names of the samples to process (optional)', type=str, required=False)
args = parser.parse_args()

# check if samples names are given and if all sample names are present in a header
sampleNames = calls.checkSampleNames(args.samples, args.tab)

############################# program #############################

print('Opening the file...')

counter = 0

siftFile = open(args.input, 'r')
annotOptions = siftFile.readline().split()
fieldsIndex = calls.indexSamples([args.fields], annotOptions)

sift_words = siftFile.readline().split()
sift_chr = int(sift_words[0].split('_')[1])
sift_pos = int(sift_words[1])

with open(args.tab) as datafile:
  header_words = datafile.readline().split()

  # index samples
  sampCol = calls.indexSamples(sampleNames, header_words)

  # make output header
  print('Creating the output file...')
  output = open(args.output, 'w')
  ouput_header = header_words[0:2] + calls.selectSamples(sampCol, header_words)
  ouput_headerP = '\t'.join(str(el) for el in ouput_header)
  output.write('%s\n' % ouput_headerP)

############################### perform counting ####################

  for line in datafile:
    words = line.split()
    ch = int(words[0].split('_')[1])
    pos = int(words[1])

    # select samples
    tab_charaters = calls.selectSamples(sampCol, words)

    # find overlap in genomic position
    while (ch > sift_chr) or (ch == sift_chr and pos > sift_pos):
      sift_words = siftFile.readline().split()
      if sift_words == []:
        break
      sift_chr = int(sift_words[0].split('_')[1])
      sift_pos = int(sift_words[1])

    # extract all annotations of the position
    sift_alt = []
    sift_annot = []
    while (ch == sift_chr and pos == sift_pos):
      sift_alt.append(sift_words[3])
      sift_annot.append([sift_words[a] for a in fieldsIndex])
      #print words[0:2], sift_words[0:4] # for debugging
      sift_words = siftFile.readline().split()
      if sift_words == []:
        break
      sift_chr = int(sift_words[0].split('_')[1])
      sift_pos = int(sift_words[1])

    # find overlap in tab genotypes and sift ALT
    tab_annot = []
    for s in tab_charaters:
      if s in sift_alt: # check if genotype is A,T,G,C (not N)
        annotIndex = sift_alt.index(s)
        s_annot = sift_annot[annotIndex]
        s_annotP = '|'.join(str(el) for el in s_annot)
        tab_annot.append(s_annotP)
      else:
        tab_annot.append('NA')

    # output annotation
    if any(x != 'NA' for x in tab_annot):
      tab_annotP = '\t'.join(str(el) for el in tab_annot)
      chr_posP = '\t'.join(str(el) for el in words[0:2])
      output.write('%s\t%s\n' % (chr_posP, tab_annotP))

    # track progress
    counter += 1
    if counter % 100000 == 0:
      print str(counter), "lines processed"

datafile.close()
siftFile.close()
output.close()

print('Done!')
<|endoftext|>


<commit_before>import couchdb, datetime
from django.conf import settings
from openid.consumer import consumer
from django_openid.consumer import signed
from openid_consumer.models import server_uri, openid_db_uri
from django_openid.auth import AuthConsumer as DjangoOpenidAuthConsumer
# I know this long naming sucks! But couldn't think of anything better...
from django_openid.consumer import Consumer as DjangoOpenidConsumer, \
                                   LoginConsumer as DjangoOpenidLoginConsumer, \
                                   SessionConsumer as DjangoOpenidSessionConsumer, \
                                   CookieConsumer as DjangoOpenidCookieConsumer
from openid_consumer.models import DjangoCouchDBOpenIDStore, get_or_create, get_values

class Consumer(DjangoOpenidConsumer):
    def get_consumer(self, request, session_store):
        return consumer.Consumer(session_store, DjangoCouchDBOpenIDStore())

class LoginConsumer(Consumer, DjangoOpenidLoginConsumer):
    pass

class SessionConsumer(LoginConsumer, DjangoOpenidSessionConsumer):
    pass

class CookieConsumer(LoginConsumer, DjangoOpenidCookieConsumer):
    pass

class AuthConsumer(SessionConsumer, DjangoOpenidAuthConsumer):
    def __init__(self):
        self.auth_db = get_or_create(server_uri, "auth")
        super(AuthConsumer, self).__init__()

    def user_can_login(self, request, user):
        "Over-ride for things like user bans or account e-mail validation"
        return user.is_active

    def log_in_user(self, request, user):
        from auth import login
        user.backend = 'auth.backends.CouchDBAuthBackend'
        login(request, user)

    def do_associate(self, request):
        if request.method == 'POST':
            try:
                openid = signed.loads(
                    request.POST.get('openid_token', ''),
                    extra_salt = self.associate_salt + str(request.user.id)
                )
            except signed.BadSignature:
                return self.show_error(request, self.csrf_failed_message)
            # Associate openid with their account, if it isn't already
            temp_db = get_or_create(server_uri, "user_openid")
            if not len(get_values(temp_db.view('openid_view/all', key = openid))):
                from openid_consumer.models import UserOpenidAssociation
                uoa = UserOpenidAssociation(user_id = request.user.id,
                                            openid  = openid,
                                            created = datetime.datetime.now())
                uoa.store(temp_db)
            return self.show_associate_done(request, openid)

        return self.show_error(request, 'Should POST to here')

    def do_associations(self, request):
        "Interface for managing your account's associated OpenIDs"
        if not request.user.is_authenticated():
            return self.need_authenticated_user(request)
        message = None
        if request.method == 'POST':
            if 'todelete' in request.POST:
                # Something needs deleting; find out what
                try:
                    todelete = signed.loads(
                        request.POST['todelete'],
                        extra_salt = self.associate_delete_salt
                    )
                    if todelete['user_id'] != request.user.id:
                        message = self.associate_tampering_message
                    else:
                        # It matches! Delete the OpenID relationship
                        temp_db = get_or_create(server_uri, "user_openid")
                        from openid_consumer.models import UserOpenidAssociation
                        rows = get_values(temp_db.view('openid_view/all', key=todelete['openid']))
                        temp_db.delete(rows[0])
                        message = self.association_deleted_message % (
                            todelete['openid']
                        )
                except signed.BadSignature:
                    message = self.associate_tampering_message
        # We construct a button to delete each existing association
        openids = []
        temp_db = get_or_create(server_uri, "user_openid")
        for association in get_values(temp_db.view('openid_view/all')):
            openids.append({
                'openid': association['openid'],
                'button': signed.dumps({
                    'user_id': request.user.id,
                    'association_id': association['_id'],
                    'openid': association['openid'],
                }, extra_salt = self.associate_delete_salt),
            })
        return self.render(request, self.associations_template, {
            'openids': openids,
            'user': request.user,
            'action': request.path,
            'message': message,
            'action_new': '../',
            'associate_next': self.sign_next(request.path),
        })

    def lookup_openid(self, request, identity_url):
        from auth.models import User
        temp_db = get_or_create(server_uri, "user_openid")
        try:
            openid = get_values(temp_db.view('openid_view/all', key=identity_url))[0]
        except IndexError:
            return []
        return [User.load(self.auth_db, openid['user_id']),]

    def lookup_users_by_email(self, email):
        return get_values(self.auth_db.view('auth_email/all', key=email))

    def lookup_user_by_username(self, username):
        try:
            return get_values(self.auth_db.view('auth_id/all', key=username))
        except IndexError:
            return None

    def lookup_user_by_id(self, id):
        return lookup_by_username(id)
<commit_msg>Ability to host multiple openid DBs of a single couchdb instance<commit_after>import couchdb, datetime
from django.conf import settings
from openid.consumer import consumer
from django_openid.consumer import signed
from openid_consumer.models import server_uri, DB_PREFIX
from django_openid.auth import AuthConsumer as DjangoOpenidAuthConsumer
# I know this long naming sucks! But couldn't think of anything better...
from django_openid.consumer import Consumer as DjangoOpenidConsumer, \
                                   LoginConsumer as DjangoOpenidLoginConsumer, \
                                   SessionConsumer as DjangoOpenidSessionConsumer, \
                                   CookieConsumer as DjangoOpenidCookieConsumer
from openid_consumer.models import DjangoCouchDBOpenIDStore, get_or_create, get_values

class Consumer(DjangoOpenidConsumer):
    def get_consumer(self, request, session_store):
        return consumer.Consumer(session_store, DjangoCouchDBOpenIDStore())

class LoginConsumer(Consumer, DjangoOpenidLoginConsumer):
    pass

class SessionConsumer(LoginConsumer, DjangoOpenidSessionConsumer):
    pass

class CookieConsumer(LoginConsumer, DjangoOpenidCookieConsumer):
    pass

class AuthConsumer(SessionConsumer, DjangoOpenidAuthConsumer):
    def __init__(self):
        auth_db_prefix = getattr(settings, "COUCHDB_AUTH_PREFIX", "")
        self.auth_db = get_or_create(server_uri, "%s%s" %(auth_db_prefix, "auth"))
        super(AuthConsumer, self).__init__()

    def user_can_login(self, request, user):
        "Over-ride for things like user bans or account e-mail validation"
        return user.is_active

    def log_in_user(self, request, user):
        from auth import login
        user.backend = 'auth.backends.CouchDBAuthBackend'
        login(request, user)

    def do_associate(self, request):
        if request.method == 'POST':
            try:
                openid = signed.loads(
                    request.POST.get('openid_token', ''),
                    extra_salt = self.associate_salt + str(request.user.id)
                )
            except signed.BadSignature:
                return self.show_error(request, self.csrf_failed_message)
            # Associate openid with their account, if it isn't already
            temp_db = get_or_create(server_uri, "%s%s" %(DB_PREFIX, "user_openid"))
            if not len(get_values(temp_db.view('openid_view/all', key = openid))):
                from openid_consumer.models import UserOpenidAssociation
                uoa = UserOpenidAssociation(user_id = request.user.id,
                                            openid  = openid,
                                            created = datetime.datetime.now())
                uoa.store(temp_db)
            return self.show_associate_done(request, openid)

        return self.show_error(request, 'Should POST to here')

    def do_associations(self, request):
        "Interface for managing your account's associated OpenIDs"
        if not request.user.is_authenticated():
            return self.need_authenticated_user(request)
        message = None
        if request.method == 'POST':
            if 'todelete' in request.POST:
                # Something needs deleting; find out what
                try:
                    todelete = signed.loads(
                        request.POST['todelete'],
                        extra_salt = self.associate_delete_salt
                    )
                    if todelete['user_id'] != request.user.id:
                        message = self.associate_tampering_message
                    else:
                        # It matches! Delete the OpenID relationship
                        temp_db = get_or_create(server_uri, "%s%s" %(DB_PREFIX, "user_openid"))
                        from openid_consumer.models import UserOpenidAssociation
                        rows = get_values(temp_db.view('openid_view/all', key=todelete['openid']))
                        temp_db.delete(rows[0])
                        message = self.association_deleted_message % (
                            todelete['openid']
                        )
                except signed.BadSignature:
                    message = self.associate_tampering_message
        # We construct a button to delete each existing association
        openids = []
        temp_db = get_or_create(server_uri, "%s%s" %(DB_PREFIX, "user_openid"))
        for association in get_values(temp_db.view('openid_view/all')):
            openids.append({
                'openid': association['openid'],
                'button': signed.dumps({
                    'user_id': request.user.id,
                    'association_id': association['_id'],
                    'openid': association['openid'],
                }, extra_salt = self.associate_delete_salt),
            })
        return self.render(request, self.associations_template, {
            'openids': openids,
            'user': request.user,
            'action': request.path,
            'message': message,
            'action_new': '../',
            'associate_next': self.sign_next(request.path),
        })

    def lookup_openid(self, request, identity_url):
        from auth.models import User
        temp_db = get_or_create(server_uri, "%s%s" %(DB_PREFIX, "user_openid"))
        try:
            openid = get_values(temp_db.view('openid_view/all', key=identity_url))[0]
        except IndexError:
            return []
        return [User.load(self.auth_db, openid['user_id']),]

    def lookup_users_by_email(self, email):
        return get_values(self.auth_db.view('auth_email/all', key=email))

    def lookup_user_by_username(self, username):
        try:
            return get_values(self.auth_db.view('auth_id/all', key=username))
        except IndexError:
            return None

    def lookup_user_by_id(self, id):
        return lookup_by_username(id)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Extract Table of Content
========================

A Pelican plugin to extract table of contents (ToC) from `article.content` and
place it in its own `article.toc` variable for use in templates.
"""

from os import path
from bs4 import BeautifulSoup
from pelican import signals, readers, contents

def extract_toc(content):
    if isinstance(content, contents.Static):
        return

    soup = BeautifulSoup(content._content,'html.parser')
    filename = content.source_path
    extension = path.splitext(filename)[1][1:]
    toc = None
    # if it is a Markdown file
    if extension in readers.MarkdownReader.file_extensions:
        toc = soup.find('div', class_='toc')
        toc.extract()
    # else if it is a reST file
    elif extension in readers.RstReader.file_extensions:
        toc = soup.find('div', class_='contents topic')
        if toc: toc.extract()
        if toc:
            tag=BeautifulSoup(str(toc))
            tag.div['class']='toc'
            tag.div['id']=''
            p=tag.find('p', class_='topic-title first')
            if p:p.extract()
            toc=tag
    elif not toc:  # Pandoc reader
        toc = soup.find('nav', id='TOC')
    if toc:
        toc.extract()
        content._content = soup.decode()
        content.toc = toc.decode()

def register():
    signals.content_object_init.connect(extract_toc)
<commit_msg>extract_toc: fix markdown case when toc not found<commit_after># -*- coding: utf-8 -*-
"""
Extract Table of Content
========================

A Pelican plugin to extract table of contents (ToC) from `article.content` and
place it in its own `article.toc` variable for use in templates.
"""

from os import path
from bs4 import BeautifulSoup
from pelican import signals, readers, contents

def extract_toc(content):
    if isinstance(content, contents.Static):
        return

    soup = BeautifulSoup(content._content,'html.parser')
    filename = content.source_path
    extension = path.splitext(filename)[1][1:]
    toc = None
    # if it is a Markdown file
    if extension in readers.MarkdownReader.file_extensions:
        toc = soup.find('div', class_='toc')
        if toc: toc.extract()
    # else if it is a reST file
    elif extension in readers.RstReader.file_extensions:
        toc = soup.find('div', class_='contents topic')
        if toc: toc.extract()
        if toc:
            tag=BeautifulSoup(str(toc))
            tag.div['class']='toc'
            tag.div['id']=''
            p=tag.find('p', class_='topic-title first')
            if p:p.extract()
            toc=tag
    elif not toc:  # Pandoc reader
        toc = soup.find('nav', id='TOC')
    if toc:
        toc.extract()
        content._content = soup.decode()
        content.toc = toc.decode()

def register():
    signals.content_object_init.connect(extract_toc)
<|endoftext|>