<commit_before>from PandaModules import *
import Particles
import ForceGroup
import DirectNotifyGlobal

class ParticleEffect(NodePath):

    notify = DirectNotifyGlobal.directNotify.newCategory('ParticleEffect')
    pid = 1

    def __init__(self, name=None, particles=None):
        """__init__()"""
        NodePath.__init__(self)
        if (name == None):
            name = 'particle-effect-%d' % ParticleEffect.pid
            ParticleEffect.pid += 1
        self.assign(hidden.attachNewNode(name))
        # Record particle effect name
        self.name = name
        # Enabled flag
        self.fEnabled = 0
        # Dictionary of particles and forceGroups
        self.particlesDict = {}
        self.forceGroupDict = {}
        # The effect's particle system
        if (particles != None):
            self.addParticles(particles)
        self.worldRelative = 0

    def start(self, parent=None, worldRelative=1):
        assert(self.notify.debug('start() - name: %s' % self.name))
        self.worldRelative = worldRelative
        self.enable()
        if (parent != None):
            self.reparentTo(parent)

    def cleanup(self):
        self.reparentTo(hidden)
        self.disable()
        for f in self.forceGroupDict.values():
            f.cleanup()
        for p in self.particlesDict.values():
            p.cleanup()
        forceGroupDict = {}
        particlesDict = {}
        self.removeNode()

    def reset(self):
        self.removeAllForces()
        self.removeAllParticles()
        self.forceGroupDict = {}
        self.particlesDict = {}

    def enable(self):
        """enable()"""
        if (self.worldRelative == 1):
            for p in self.particlesDict.values():
                p.setRenderParent(render.node())
        for f in self.forceGroupDict.values():
            f.enable()
        for p in self.particlesDict.values():
            p.enable()
        self.fEnabled = 1

    def disable(self):
        """disable()"""
        if (self.worldRelative == 1):
            for p in self.particlesDict.values():
                p.setRenderParent(hidden.node())
        for f in self.forceGroupDict.values():
            f.disable()
        for p in self.particlesDict.values():
            p.disable()
        self.fEnabled = 0

    def isEnabled(self):
        """
        isEnabled()
        Note: this may be misleading if enable(),disable() not used
        """
        return self.fEnabled

    def addForceGroup(self, forceGroup):
        """addForceGroup(forceGroup)"""
        forceGroup.nodePath.reparentTo(self)
        forceGroup.particleEffect = self
        self.forceGroupDict[forceGroup.getName()] = forceGroup

        # Associate the force group with all particles
        for f in forceGroup.asList():
            self.addForce(f)

    def addForce(self, force):
        """addForce(force)"""
        for p in self.particlesDict.values():
            p.addForce(force)

    def removeForceGroup(self, forceGroup):
        """removeForceGroup(forceGroup)"""
        # Remove forces from all particles
        for f in forceGroup.asList():
            self.removeForce(f)

        forceGroup.nodePath.reparentTo(hidden)
        forceGroup.particleEffect = None
        del self.forceGroupDict[forceGroup.getName()]

    def removeForce(self, force):
        """removeForce(force)"""
        for p in self.particlesDict.values():
            p.removeForce(force)

    def removeAllForces(self):
        for fg in self.forceGroupDict.values():
            self.removeForceGroup(fg)

    def addParticles(self, particles):
        """addParticles(particles)"""
        particles.nodePath.reparentTo(self)
        self.particlesDict[particles.getName()] = particles

        # Associate all forces in all force groups with the particles
        for fg in self.forceGroupDict.values():
            for f in fg.asList():
                particles.addForce(f)

    def removeParticles(self, particles):
        """removeParticles(particles)"""
        if (particles == None):
            self.notify.warning('removeParticles() - particles == None!')
            return
        particles.nodePath.reparentTo(hidden)
        del self.particlesDict[particles.getName()]

        # Remove all forces from the particles
        for fg in self.forceGroupDict.values():
            for f in fg.asList():
                particles.removeForce(f)

    def removeAllParticles(self):
        for p in self.particlesDict.values():
            self.removeParticles(p)

    def getParticlesList(self):
        """getParticles()"""
        return self.particlesDict.values()

    def getParticlesNamed(self, name):
        """getParticlesNamed(name)"""
        return self.particlesDict.get(name, None)

    def getParticlesDict(self):
        """getParticlesDict()"""
        return self.particlesDict

    def getForceGroupList(self):
        """getForceGroup()"""
        return self.forceGroupDict.values()

    def getForceGroupNamed(self, name):
        """getForceGroupNamed(name)"""
        return self.forceGroupDict.get(name, None)

    def getForceGroupDict(self):
        """getForceGroup()"""
        return self.forceGroupDict

    def saveConfig(self, filename):
        """saveFileData(filename)"""
        f = open(filename.toOsSpecific(), 'wb')
        # Add a blank line
        f.write('\n')

        # Make sure we start with a clean slate
        f.write('self.reset()\n')

        pos = self.getPos()
        hpr = self.getHpr()
        scale = self.getScale()
        f.write('self.setPos(%0.3f, %0.3f, %0.3f)\n' %
                (pos[0], pos[1], pos[2]))
        f.write('self.setHpr(%0.3f, %0.3f, %0.3f)\n' %
                (hpr[0], hpr[1], hpr[2]))
        f.write('self.setScale(%0.3f, %0.3f, %0.3f)\n' %
                (scale[0], scale[1], scale[2]))

        # Save all the particles to file
        num = 0
        for p in self.particlesDict.values():
            target = 'p%d' % num
            num = num + 1
            f.write(target + ' = Particles.Particles(\'%s\')\n' % p.getName())
            p.printParams(f, target)
            f.write('self.addParticles(%s)\n' % target)

        # Save all the forces to file
        num = 0
        for fg in self.forceGroupDict.values():
            target = 'f%d' % num
            num = num + 1
            f.write(target + ' = ForceGroup.ForceGroup(\'%s\')\n' % \
                                                fg.getName())
            fg.printParams(f, target)
            f.write('self.addForceGroup(%s)\n' % target)

        # Close the file
        f.close()

    def loadConfig(self, filename):
        """loadConfig(filename)"""
        if vfs:
            exec vfs.readFile(filename)
        else:
            execfile(filename.toOsSpecific())

<commit_msg>stop the snow in the igloo<commit_after>from PandaModules import *
import Particles
import ForceGroup
import DirectNotifyGlobal

class ParticleEffect(NodePath):

    notify = DirectNotifyGlobal.directNotify.newCategory('ParticleEffect')
    pid = 1

    def __init__(self, name=None, particles=None):
        """__init__()"""
        NodePath.__init__(self)
        if (name == None):
            name = 'particle-effect-%d' % ParticleEffect.pid
            ParticleEffect.pid += 1
        self.assign(hidden.attachNewNode(name))
        # Record particle effect name
        self.name = name
        # Enabled flag
        self.fEnabled = 0
        # Dictionary of particles and forceGroups
        self.particlesDict = {}
        self.forceGroupDict = {}
        # The effect's particle system
        if (particles != None):
            self.addParticles(particles)
        self.renderParent = None

    def start(self, parent=None, renderParent=None):
        assert(self.notify.debug('start() - name: %s' % self.name))
        self.renderParent = renderParent
        self.enable()
        if (parent != None):
            self.reparentTo(parent)

    def cleanup(self):
        self.reparentTo(hidden)
        self.disable()
        for f in self.forceGroupDict.values():
            f.cleanup()
        for p in self.particlesDict.values():
            p.cleanup()
        forceGroupDict = {}
        particlesDict = {}
        self.removeNode()

    def reset(self):
        self.removeAllForces()
        self.removeAllParticles()
        self.forceGroupDict = {}
        self.particlesDict = {}

    def enable(self):
        """enable()"""
        if (self.renderParent != None):
            for p in self.particlesDict.values():
                p.setRenderParent(self.renderParent.node())
        for f in self.forceGroupDict.values():
            f.enable()
        for p in self.particlesDict.values():
            p.enable()
        self.fEnabled = 1

    def disable(self):
        """disable()"""
        for p in self.particlesDict.values():
            p.setRenderParent(p.node)
        for f in self.forceGroupDict.values():
            f.disable()
        for p in self.particlesDict.values():
            p.disable()
        self.fEnabled = 0

    def isEnabled(self):
        """
        isEnabled()
        Note: this may be misleading if enable(),disable() not used
        """
        return self.fEnabled

    def addForceGroup(self, forceGroup):
        """addForceGroup(forceGroup)"""
        forceGroup.nodePath.reparentTo(self)
        forceGroup.particleEffect = self
        self.forceGroupDict[forceGroup.getName()] = forceGroup

        # Associate the force group with all particles
        for f in forceGroup.asList():
            self.addForce(f)

    def addForce(self, force):
        """addForce(force)"""
        for p in self.particlesDict.values():
            p.addForce(force)

    def removeForceGroup(self, forceGroup):
        """removeForceGroup(forceGroup)"""
        # Remove forces from all particles
        for f in forceGroup.asList():
            self.removeForce(f)

        forceGroup.nodePath.reparentTo(hidden)
        forceGroup.particleEffect = None
        del self.forceGroupDict[forceGroup.getName()]

    def removeForce(self, force):
        """removeForce(force)"""
        for p in self.particlesDict.values():
            p.removeForce(force)

    def removeAllForces(self):
        for fg in self.forceGroupDict.values():
            self.removeForceGroup(fg)

    def addParticles(self, particles):
        """addParticles(particles)"""
        particles.nodePath.reparentTo(self)
        self.particlesDict[particles.getName()] = particles

        # Associate all forces in all force groups with the particles
        for fg in self.forceGroupDict.values():
            for f in fg.asList():
                particles.addForce(f)

    def removeParticles(self, particles):
        """removeParticles(particles)"""
        if (particles == None):
            self.notify.warning('removeParticles() - particles == None!')
            return
        particles.nodePath.reparentTo(hidden)
        del self.particlesDict[particles.getName()]

        # Remove all forces from the particles
        for fg in self.forceGroupDict.values():
            for f in fg.asList():
                particles.removeForce(f)

    def removeAllParticles(self):
        for p in self.particlesDict.values():
            self.removeParticles(p)

    def getParticlesList(self):
        """getParticles()"""
        return self.particlesDict.values()

    def getParticlesNamed(self, name):
        """getParticlesNamed(name)"""
        return self.particlesDict.get(name, None)

    def getParticlesDict(self):
        """getParticlesDict()"""
        return self.particlesDict

    def getForceGroupList(self):
        """getForceGroup()"""
        return self.forceGroupDict.values()

    def getForceGroupNamed(self, name):
        """getForceGroupNamed(name)"""
        return self.forceGroupDict.get(name, None)

    def getForceGroupDict(self):
        """getForceGroup()"""
        return self.forceGroupDict

    def saveConfig(self, filename):
        """saveFileData(filename)"""
        f = open(filename.toOsSpecific(), 'wb')
        # Add a blank line
        f.write('\n')

        # Make sure we start with a clean slate
        f.write('self.reset()\n')

        pos = self.getPos()
        hpr = self.getHpr()
        scale = self.getScale()
        f.write('self.setPos(%0.3f, %0.3f, %0.3f)\n' %
                (pos[0], pos[1], pos[2]))
        f.write('self.setHpr(%0.3f, %0.3f, %0.3f)\n' %
                (hpr[0], hpr[1], hpr[2]))
        f.write('self.setScale(%0.3f, %0.3f, %0.3f)\n' %
                (scale[0], scale[1], scale[2]))

        # Save all the particles to file
        num = 0
        for p in self.particlesDict.values():
            target = 'p%d' % num
            num = num + 1
            f.write(target + ' = Particles.Particles(\'%s\')\n' % p.getName())
            p.printParams(f, target)
            f.write('self.addParticles(%s)\n' % target)

        # Save all the forces to file
        num = 0
        for fg in self.forceGroupDict.values():
            target = 'f%d' % num
            num = num + 1
            f.write(target + ' = ForceGroup.ForceGroup(\'%s\')\n' % \
                                                fg.getName())
            fg.printParams(f, target)
            f.write('self.addForceGroup(%s)\n' % target)

        # Close the file
        f.close()

    def loadConfig(self, filename):
        """loadConfig(filename)"""
        if vfs:
            exec vfs.readFile(filename)
        else:
            execfile(filename.toOsSpecific())

<|endoftext|>


<commit_before>"""
    .. module:: OakParkDatasetAdapter
    :platform: Unix
    :synopsis: Contains methods for importing data from Oak Park
    dataset.

    .. moduleauthor:: Phil Ngo <ngo.phil@gmail.com>
    .. moduleauthor:: Miguel Perez <miguel@invalid.com>
    .. moduleauthor:: Stephen Suffian <steve@invalid.com>
    .. moduleauthor:: Sabina Tomkins <sabina.tomkins@gmail.com>

"""

from appliance import ApplianceTrace
from appliance import ApplianceInstance
from appliance import ApplianceSet
from appliance import ApplianceType
import utils
import pymongo
import pandas as pd

def get_db_connection():
    connection = pymongo.MongoClient('ds059938.mongolab.com',59938)
    db = connection['oakparkhistoric']
    db.authenticate('anonymouseenergy', 'KaTard3i3')
    return db

def get_homes(db):
    """
    Returns a dictionary where each key is a home id and each value is a home.
    A home as meta_data, and a list of dates and power values in kwH.
    """
    houses =[]
    ids=[]
    c = db.usage.find()
    for co in c:
        ids.append(str(co['meta']['dataid']))
        houses.append(co)
    return {i:h for i,h in zip (ids,houses)}

def generate_trace_by_dataid(homes,dataid):
    '''
    Returns a trace.
    '''
    house = homes[dataid]
    dates = [x[0] for x in house['interval_readings']]
    values = [x[1]['value'] for x in house['interval_readings']]

    return ApplianceTrace(pd.Series(values, index = dates),house['meta'])

def resample_trace_by_month(trace,month):
    '''
    Returns a set of houses for a given month.
    '''
    return utils.resample_trace(trace,'MS')

def check_complete(home,year,month):
    dt_start = datetime(year,month,1)
    dt_end = datetime(year,month,calendar.monthrange(year,month)[1])
    index_start = home.series.index.searchsorted(dt_start)
    index_end = home.series.index.searchsorted(dt_end)
    return index_end-index_start==((dt_end-dt_start)*48).days

def get_home_series_by_year_month(year,month,home):
    '''
        Returns the home's series sliced by year and month.
        '''
    if check_complete(home,year,month):
        return  home.trace.series[index_start:index_end]

def get_list_of_homes_with_certain_month_year(homes,year,month):
    '''
        Returns a list of homes which have complete trace info for given year and month.
        Assumes that homes is dict where keys are dataids and values are traces.
        '''
    complete_homes = []
    for h in homes.keys():
        if check_complete(homes[h],year,month):
            complete_homes.append(h)
    return complete_homes

<commit_msg>added homes : traces dict<commit_after>"""
    .. module:: OakParkDatasetAdapter
    :platform: Unix
    :synopsis: Contains methods for importing data from Oak Park
    dataset.

    .. moduleauthor:: Phil Ngo <ngo.phil@gmail.com>
    .. moduleauthor:: Miguel Perez <miguel@invalid.com>
    .. moduleauthor:: Stephen Suffian <steve@invalid.com>
    .. moduleauthor:: Sabina Tomkins <sabina.tomkins@gmail.com>

"""

from appliance import ApplianceTrace
from appliance import ApplianceInstance
from appliance import ApplianceSet
from appliance import ApplianceType
import utils
import pymongo
import pandas as pd

def get_db_connection():
    connection = pymongo.MongoClient('ds059938.mongolab.com',59938)
    db = connection['oakparkhistoric']
    db.authenticate('anonymouseenergy', 'KaTard3i3')
    return db

def get_homes(db):
    """
    Returns a dictionary where each key is a home id and each value is a home.
    A home as meta_data, and a list of dates and power values in kwH.
    """
    houses =[]
    ids=[]
    c = db.usage.find()
    for co in c:
        ids.append(str(co['meta']['dataid']))
        houses.append(co)
    return {i:h for i,h in zip (ids,houses)}

def homes_to_traces(homes):
    '''
       Returns a dictionary where the key is a home id and the value is an
       Appliance Trace.
    '''
    homes = {}
    for di in homes.keys():
        homes[di]=generate_trace_by_dataid(homes,di)
    return homes

def generate_trace_by_dataid(homes,dataid):
    '''
    Returns a trace.
    '''
    house = homes[dataid]
    dates = [x[0] for x in house['interval_readings']]
    values = [x[1]['value'] for x in house['interval_readings']]

    return ApplianceTrace(pd.Series(values, index = dates),house['meta'])

def resample_trace_by_month(trace,month):
    '''
    Returns a set of houses for a given month.
    '''
    return utils.resample_trace(trace,'MS')

def check_complete(home,year,month):
    dt_start = datetime(year,month,1)
    dt_end = datetime(year,month,calendar.monthrange(year,month)[1])
    index_start = home.series.index.searchsorted(dt_start)
    index_end = home.series.index.searchsorted(dt_end)
    return index_end-index_start==((dt_end-dt_start)*48).days

def get_home_series_by_year_month(year,month,home):
    '''
        Returns the home's series sliced by year and month.
        '''
    if check_complete(home,year,month):
        return  home.trace.series[index_start:index_end]

def get_list_of_homes_with_certain_month_year(homes,year,month):
    '''
        Returns a list of homes which have complete trace info for given year and month.
        Assumes that homes is dict where keys are dataids and values are traces.
        '''
    complete_homes = []
    for h in homes.keys():
        if check_complete(homes[h],year,month):
            complete_homes.append(h)
    return complete_homes

<|endoftext|>


<commit_before>#!/usr/bin/env python3.5
#
# join_multiplexed_data.py
#
# Script to read pickled output from multiplexed runs generated by
# run_multiplex_models.py, and produce a single output .tab file
# containing predictions, prediction credibility intervals, and the
# observed output for the probes.
#
# (C) The James Hutton Institute 2016
# Author: Leighton Pritchard

"""
join_multiplexed_data.py

A script to join the pickled PyStan output from multiplexed data into a
single .tab file
"""

import logging
import logging.handlers
import os
import pickle
import sys
import time

from argparse import ArgumentParser

import numpy as np
import pandas as pd

class JoinMultiplexException(Exception):
    """Exception raised when script fails"""
    def __init__(self, message):
        super().__init__(message)

# Parse command-line
def parse_cmdline():
    """ Parse command-line arguments
    """
    parser = ArgumentParser(prog=__file__)
    parser.add_argument("-i", "--indir", dest="indirname",
                        action="store", default='multiplexed_data', type=str,
                        help="Parent directory for multiplexed subfolders")
    parser.add_argument("-o", "--outfile", dest="outfilename",
                        action="store", default="multiplexed_predictions.tab",
                        type=str,
                        help="Path to model script")
    parser.add_argument("-l", "--logfile", dest="logfile",
                        action="store", default=None, type=str,
                        help="Logfile location")
    parser.add_argument("-v", "--verbose", dest="verbose",
                        action="store_true",
                        help="Give verbose output")
    return parser.parse_args()

# Set up logger
def logger_setup(logfilename=None, verbose=False):
    """Return logger for script"""
    logger = logging.getLogger(__file__)
    logger.setLevel(logging.DEBUG)

    # Add handler for STDERR
    err_handler = logging.StreamHandler(sys.stderr)
    logger.addHandler(err_handler)
    err_formatter = logging.Formatter('%(levelname)s: %(message)s')
    err_handler.setFormatter(err_formatter)

    # Add logfile handler
    if logfilename:
        try:
            logstream = open(logfilename, 'w')
            err_handler_file = logging.StreamHandler(logstream)
            err_handler_file.setFormatter(err_formatter)
            err_handler_file.setLevel(logging.INFO)
            logger.addHandler(err_handler_file)
        except:
            msg = "Could not open {0} for logging".format(logfilename)
            logger.error(msg)
            raise MultiplexException(msg)

    # Set loglevel
    if verbose:
        err_handler.setLevel(logging.INFO)
    else:
        err_handler.setLevel(logging.WARNING)

    return logger

def get_result_filenames(dirname):
    """Returns a list of path tuples to the test data and pickled output"""
    fnames = []
    for subdir in [dname for dname in os.listdir(dirname) if
                   os.path.isdir(dname)]:
            pickles.append((os.path.join(dirname, subdir, "test.tab"),
                            os.path.join(dirname, subdir, "prediction.pkl")))
    return fnames

def extract_variable_summaries(df, varname):
    """Extracts summary variables for a variable in the passed dataframe

    Calculates mean, std, median, and 5%, 25%, 75% and 95% percentiles
    for the passed variable, returning them as a dataframe.
    """
    # Using Pandas methods
    mean = pd.Series(data[varname][0].mean(0))
    se = pd.Series(data[varname][0].std(0))

    # Need to use numpy functions
    median = pd.Series(np.median(prediction_fit[varname][0], 0))
    perc_5 = pd.Series(np.percentile(prediction_fit[varname][0], 5, 0))
    perc_25 = pd.Series(np.percentile(prediction_fit[varname][0], 25, 0))
    perc_75 = pd.Series(np.percentile(prediction_fit[varname][0], 75, 0))
    perc_95 = pd.Series(np.percentile(prediction_fit[varname][0], 95, 0))

def load_pickle_data(datafilename, picklename, df=None):
    """Returns a Pandas dataframe containing summary of pickled data

    datafilename takes the path to input data corresponding to the pickled
    results
    picklename takes the path to the pickled results file
    If a dataframe is passed in df, data from the pickled file is
    appended to the dataframe.
    """
    # load pickled results and input data
    with open(picklename, 'rb') as ifh:
        data = pickle.load(ifh)
    indata = pd.read_csv(datafilename, sep="\t")

    # Obtain summary of pickled data predictions and join to the input data
    y_pred = extract_variable_summaries(df, 'y_pred')  # Predictions
    outdata = indata.reset_index(drop=True)
    outdata = outdata.join(y_pred, how="outer")

    # Append this dataframe to the passed df
    if df is not None:
        outdata = df.append(outdata)

    return outdata

# Main method for script
def main():
    """Main function for script."""
    args = parse_cmdline()

    # Set up logger
    logger = logger_setup(args.logfile, args.verbose)
    logger.info('# %s logfile', __file__)
    logger.info('# %s', time.asctime())
    logger.info(args)

    # Identify pickle files
    logger.info("Looking for test data/pickle files in %s", args.indirname)
    infiles = get_result_filenames(args.indirname)
    logger.info("Identified input files:\n%s", "\n\t".join(infiles))

    # Load each pickle file, and add results to a growing Pandas dataframe
    logger.info("Loading pickle files into dataframe")
    df = None
    for datafile, results in infiles:
        logger.info("Loading test data from %s, results from %s",
                    datafile, results)
        df = load_pickle_data(datafile, results, df)
    logger.info("Loaded pickled data. Dataframe size: %d x %d", *dfshape)

# SCRIPT
if __name__ == '__main__':
    main()
<commit_msg>add working version of join_multiplexed_data.py<commit_after>#!/usr/bin/env python3.5
#
# join_multiplexed_data.py
#
# Script to read pickled output from multiplexed runs generated by
# run_multiplex_models.py, and produce a single output .tab file
# containing predictions, prediction credibility intervals, and the
# observed output for the probes.
#
# (C) The James Hutton Institute 2016
# Author: Leighton Pritchard

"""
join_multiplexed_data.py

A script to join the pickled PyStan output from multiplexed data into a
single .tab file
"""

import logging
import logging.handlers
import os
import pickle
import sys
import time

from argparse import ArgumentParser

import numpy as np
import pandas as pd

class JoinMultiplexException(Exception):
    """Exception raised when script fails"""
    def __init__(self, message):
        super().__init__(message)

# Parse command-line
def parse_cmdline():
    """ Parse command-line arguments
    """
    parser = ArgumentParser(prog=__file__)
    parser.add_argument("-i", "--indir", dest="indirname",
                        action="store", default='multiplexed_data', type=str,
                        help="Parent directory for multiplexed subfolders")
    parser.add_argument("-o", "--outfile", dest="outfilename",
                        action="store", default="multiplexed_predictions.tab",
                        type=str,
                        help="Path to model script")
    parser.add_argument("-l", "--logfile", dest="logfile",
                        action="store", default=None, type=str,
                        help="Logfile location")
    parser.add_argument("-v", "--verbose", dest="verbose",
                        action="store_true",
                        help="Give verbose output")
    return parser.parse_args()

# Set up logger
def logger_setup(logfilename=None, verbose=False):
    """Return logger for script"""
    logger = logging.getLogger(__file__)
    logger.setLevel(logging.DEBUG)

    # Add handler for STDERR
    err_handler = logging.StreamHandler(sys.stderr)
    logger.addHandler(err_handler)
    err_formatter = logging.Formatter('%(levelname)s: %(message)s')
    err_handler.setFormatter(err_formatter)

    # Add logfile handler
    if logfilename:
        try:
            logstream = open(logfilename, 'w')
            err_handler_file = logging.StreamHandler(logstream)
            err_handler_file.setFormatter(err_formatter)
            err_handler_file.setLevel(logging.INFO)
            logger.addHandler(err_handler_file)
        except:
            msg = "Could not open {0} for logging".format(logfilename)
            logger.error(msg)
            raise MultiplexException(msg)

    # Set loglevel
    if verbose:
        err_handler.setLevel(logging.INFO)
    else:
        err_handler.setLevel(logging.WARNING)

    return logger

def get_result_filenames(dirname):
    """Returns a list of path tuples to the test data and pickled output"""
    fnames = []
    for subdir in [dname for dname in os.listdir(dirname) if
                   os.path.isdir(os.path.join(dirname, dname))]:
            fnames.append((os.path.join(dirname, subdir, "test.tab"),
                           os.path.join(dirname, subdir, "prediction.pkl")))
    return fnames

def extract_variable_summaries(df, varname):
    """Extracts summary variables for a variable in the passed dataframe

    Calculates mean, std, median, and 5%, 25%, 75% and 95% percentiles
    for the passed variable, returning them as a dataframe.
    """
    # Using Pandas methods
    mean = pd.Series(df[varname][0].mean(0))
    se = pd.Series(df[varname][0].std(0))

    # Need to use numpy functions
    median = pd.Series(np.median(df[varname][0], 0))
    perc_5 = pd.Series(np.percentile(df[varname][0], 5, 0))
    perc_25 = pd.Series(np.percentile(df[varname][0], 25, 0))
    perc_75 = pd.Series(np.percentile(df[varname][0], 75, 0))
    perc_95 = pd.Series(np.percentile(df[varname][0], 95, 0))

    return pd.DataFrame({'%s_mean' % varname: mean,
                         '%s_sem' % varname: se,
                         '%s_median' % varname: median,
                         '%s_5pc' % varname: perc_5,
                         '%s_95pc' % varname: perc_95,
                         '%s_25pc' % varname: perc_25,
                         '%s_75pc' % varname: perc_75})

def load_pickle_data(datafilename, picklename, df=None):
    """Returns a Pandas dataframe containing summary of pickled data

    datafilename takes the path to input data corresponding to the pickled
    results
    picklename takes the path to the pickled results file
    If a dataframe is passed in df, data from the pickled file is
    appended to the dataframe.
    """
    # load pickled results and input data
    with open(picklename, 'rb') as ifh:
        results = pickle.load(ifh)
    indata = pd.read_csv(datafilename, sep="\t")

    # Obtain summary of pickled data predictions and join to the input data
    y_pred = extract_variable_summaries(results, 'y_pred')  # Predictions
    outdata = indata.reset_index(drop=True)
    outdata = outdata.join(y_pred, how="outer")

    # Append this dataframe to the passed df
    if df is not None:
        outdata = df.append(outdata)

    return outdata

# Main method for script
def main():
    """Main function for script."""
    args = parse_cmdline()

    # Set up logger
    logger = logger_setup(args.logfile, args.verbose)
    logger.info('# %s logfile', __file__)
    logger.info('# %s', time.asctime())
    logger.info(args)

    # Identify pickle files
    logger.info("Looking for test data/pickle files in %s", args.indirname)
    infiles = get_result_filenames(args.indirname)
    logger.info("Identified input files:\n\t%s",
                "\n\t".join(["%s, %s" % fn for fn in infiles]))

    # Load each pickle file, and add results to a growing Pandas dataframe
    logger.info("Loading pickle files into dataframe")
    df = None
    for datafile, results in infiles:
        logger.info("Loading test data from %s, results from %s",
                    datafile, results)
        df = load_pickle_data(datafile, results, df)
    logger.info("Loaded pickled data.\nDataframe size: %d x %d", *df.shape)
    logger.info("Dataframe columns:\n\t%s",
                '\n\t'.join(sorted([name for name in df.columns])))

    # Write data to file
    logger.info("Writing data to %s", args.outfilename)
    df.sort_values("locus_tag", inplace=True)
    df.to_csv(args.outfilename, sep="\t")
    logger.info("Exiting: %s", time.asctime())

# SCRIPT
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Compute the maximum correlation:
.. math::
    \rho(X:Y) = max_{f, g} E(f(X)g(Y))
"""

import numpy as np

from ..exceptions import ditException
from ..helpers import normalize_rvs

__all__ = (
    'conditional_maximum_correlation_pmf',
    'maximum_correlation',
    'maximum_correlation_pmf',
)

svdvals = lambda m: np.linalg.svd(m, compute_uv=False)

def conditional_maximum_correlation_pmf(pmf):
    """
    Compute the conditional maximum correlation from a 3-dimensional
    pmf. The maximum correlation is computed between the first two dimensions
    given the third.

    Parameters
    ----------
    pmf : np.ndarray
        The probability distribution.

    Returns
    -------
    rho_max : float
        The conditional maximum correlation.
    """
    pXYgZ = pmf / pmf.sum(axis=(0, 1), keepdims=True)
    pXgZ = pXYgZ.sum(axis=1, keepdims=True)
    pYgZ = pXYgZ.sum(axis=0, keepdims=True)
    Q = np.where(pmf, pXYgZ / (np.sqrt(pXgZ) * np.sqrt(pYgZ)), 0)
    Q[np.isnan(Q)] = 0

    rho_max = max(svdvals(np.squeeze(m))[1] for m in np.dsplit(Q, Q.shape[2]))

    return rho_max

def maximum_correlation_pmf(pXY):
    """
    Compute the maximum correlation from a 2-dimensional
    pmf. The maximum correlation is computed between the  two dimensions.

    Parameters
    ----------
    pmf : np.ndarray
        The probability distribution.

    Returns
    -------
    rho_max : float
        The maximum correlation.
    """
    pX = pXY.sum(axis=1, keepdims=True)
    pY = pXY.sum(axis=0, keepdims=True)
    Q = pXY / (np.sqrt(pX) * np.sqrt(pY))
    Q[np.isnan(Q)] = 0

    rho_max = svdvals(Q)[1]

    return rho_max

def maximum_correlation(dist, rvs=None, crvs=None, rv_mode=None):
    """
    Compute the (conditional) maximum or Renyi correlation between two variables:

    .. math::
        rho_max = max_{f, g} rho(f(X,Z), g(Y,Z) | Z)

    Parameters
    ----------
    dist : Distribution
        The distribution for which the maximum correlation is to computed.
    rvs : list, None; len(rvs) == 2
        A list of lists. Each inner list specifies the indexes of the random
        variables for which the maximum correlation is to be computed. If None,
        then all random variables are used, which is equivalent to passing
        `rvs=dist.rvs`.
    crvs : list, None
        A single list of indexes specifying the random variables to
        condition on. If None, then no variables are conditioned on.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If
        equal to 'names', the the elements are interpreted as random
        variable names. If `None`, then the value of `dist._rv_mode` is
        consulted, which defaults to 'indices'.

    Returns
    -------
    rho_max : float; -1 <= rho_max <= 1
        The conditional maximum correlation between `rvs` given `crvs`.
    """
    rvs, crvs, rv_mode = normalize_rvs(dist, rvs, crvs, rv_mode)

    if len(rvs) != 2:
        msg = 'Maximum correlation can only be computed for 2 variables, not {}.'.format(len(rvs))
        raise ditException(msg)

    if crvs:
        dist = dist.copy().coalesce(rvs + [crvs])
    else:
        dist = dist.copy().coalesce(rvs)

    dist.make_dense()
    pmf = dist.pmf.reshape(list(map(len, dist.alphabet)))

    if crvs:
        rho_max = conditional_maximum_correlation_pmf(pmf)
    else:
        rho_max = maximum_correlation_pmf(pmf)

    return rho_max
<commit_msg>fix latex in docstring<commit_after>"""
Compute the maximum correlation:
.. math::
    \rho(X:Y) = \max_{f, g} E(f(X)g(Y))
"""

import numpy as np

from ..exceptions import ditException
from ..helpers import normalize_rvs

__all__ = (
    'conditional_maximum_correlation_pmf',
    'maximum_correlation',
    'maximum_correlation_pmf',
)

svdvals = lambda m: np.linalg.svd(m, compute_uv=False)

def conditional_maximum_correlation_pmf(pmf):
    """
    Compute the conditional maximum correlation from a 3-dimensional
    pmf. The maximum correlation is computed between the first two dimensions
    given the third.

    Parameters
    ----------
    pmf : np.ndarray
        The probability distribution.

    Returns
    -------
    rho_max : float
        The conditional maximum correlation.
    """
    pXYgZ = pmf / pmf.sum(axis=(0, 1), keepdims=True)
    pXgZ = pXYgZ.sum(axis=1, keepdims=True)
    pYgZ = pXYgZ.sum(axis=0, keepdims=True)
    Q = np.where(pmf, pXYgZ / (np.sqrt(pXgZ) * np.sqrt(pYgZ)), 0)
    Q[np.isnan(Q)] = 0

    rho_max = max(svdvals(np.squeeze(m))[1] for m in np.dsplit(Q, Q.shape[2]))

    return rho_max

def maximum_correlation_pmf(pXY):
    """
    Compute the maximum correlation from a 2-dimensional
    pmf. The maximum correlation is computed between the  two dimensions.

    Parameters
    ----------
    pmf : np.ndarray
        The probability distribution.

    Returns
    -------
    rho_max : float
        The maximum correlation.
    """
    pX = pXY.sum(axis=1, keepdims=True)
    pY = pXY.sum(axis=0, keepdims=True)
    Q = pXY / (np.sqrt(pX) * np.sqrt(pY))
    Q[np.isnan(Q)] = 0

    rho_max = svdvals(Q)[1]

    return rho_max

def maximum_correlation(dist, rvs=None, crvs=None, rv_mode=None):
    """
    Compute the (conditional) maximum or Renyi correlation between two variables:

    .. math::
        \rho^{*} = \max_{f, g} \rho(f(X,Z), g(Y,Z) | Z)

    Parameters
    ----------
    dist : Distribution
        The distribution for which the maximum correlation is to computed.
    rvs : list, None; len(rvs) == 2
        A list of lists. Each inner list specifies the indexes of the random
        variables for which the maximum correlation is to be computed. If None,
        then all random variables are used, which is equivalent to passing
        `rvs=dist.rvs`.
    crvs : list, None
        A single list of indexes specifying the random variables to
        condition on. If None, then no variables are conditioned on.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If
        equal to 'names', the the elements are interpreted as random
        variable names. If `None`, then the value of `dist._rv_mode` is
        consulted, which defaults to 'indices'.

    Returns
    -------
    rho_max : float; -1 <= rho_max <= 1
        The conditional maximum correlation between `rvs` given `crvs`.
    """
    rvs, crvs, rv_mode = normalize_rvs(dist, rvs, crvs, rv_mode)

    if len(rvs) != 2:
        msg = 'Maximum correlation can only be computed for 2 variables, not {}.'.format(len(rvs))
        raise ditException(msg)

    if crvs:
        dist = dist.copy().coalesce(rvs + [crvs])
    else:
        dist = dist.copy().coalesce(rvs)

    dist.make_dense()
    pmf = dist.pmf.reshape(list(map(len, dist.alphabet)))

    if crvs:
        rho_max = conditional_maximum_correlation_pmf(pmf)
    else:
        rho_max = maximum_correlation_pmf(pmf)

    return rho_max
<|endoftext|>


<commit_before>"""
Classes allowing "generic" relations through ContentType and object-id fields.
"""

from django.core.exceptions import ObjectDoesNotExist
from django.db import connection
from django.db.models import signals
from django.db import models
from django.db.models.fields.related import RelatedField, Field, ManyToManyRel
from django.db.models.loading import get_model
from django.forms import ModelForm
from django.forms.models import BaseModelFormSet, modelformset_factory, save_instance
from django.contrib.admin.options import InlineModelAdmin, flatten_fieldsets
from django.utils.encoding import smart_unicode

class GenericForeignKey(object):
    """
    Provides a generic relation to any object through content-type/object-id
    fields.
    """

    def __init__(self, ct_field="content_type", fk_field="object_id"):
        self.ct_field = ct_field
        self.fk_field = fk_field

    def contribute_to_class(self, cls, name):
        self.name = name
        self.model = cls
        self.cache_attr = "_%s_cache" % name
        cls._meta.add_virtual_field(self)

        # For some reason I don't totally understand, using weakrefs here doesn't work.
        signals.pre_init.connect(self.instance_pre_init, sender=cls, weak=False)

        # Connect myself as the descriptor for this field
        setattr(cls, name, self)

    def instance_pre_init(self, signal, sender, args, kwargs, **_kwargs):
        """
        Handles initializing an object with the generic FK instaed of
        content-type/object-id fields.
        """
        if self.name in kwargs:
            value = kwargs.pop(self.name)
            kwargs[self.ct_field] = self.get_content_type(obj=value)
            kwargs[self.fk_field] = value._get_pk_val()

    def get_content_type(self, obj=None, id=None, using=None):
        # Convenience function using get_model avoids a circular import when
        # using this model
        ContentType = get_model("contenttypes", "contenttype")
        if obj:
             return ContentType.objects.db_manager(obj._state.db).get_for_model(obj)
        elif id:
             return ContentType.objects.db_manager(using).get_for_id(id)
        else:
            # This should never happen. I love comments like this, don't you?
            raise Exception("Impossible arguments to GFK.get_content_type!")

    def __get__(self, instance, instance_type=None):
        if instance is None:
            return self

        try:
            return getattr(instance, self.cache_attr)
        except AttributeError:
            rel_obj = None

            # Make sure to use ContentType.objects.get_for_id() to ensure that
            # lookups are cached (see ticket #5570). This takes more code than
            # the naive ``getattr(instance, self.ct_field)``, but has better
            # performance when dealing with GFKs in loops and such.
            f = self.model._meta.get_field(self.ct_field)
            ct_id = getattr(instance, f.get_attname(), None)
            if ct_id:
                ct = self.get_content_type(id=ct_id, using=instance._state.db)
                try:
                    rel_obj = ct.get_object_for_this_type(pk=getattr(instance, self.fk_field))
                except ObjectDoesNotExist:
                    pass
            setattr(instance, self.cache_attr, rel_obj)
            return rel_obj

    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError(u"%s must be accessed via instance" % self.related.opts.object_name)

        ct = None
        fk = None
        if value is not None:
            ct = self.get_content_type(obj=value)
            fk = value._get_pk_val()

        setattr(instance, self.ct_field, ct)
        setattr(instance, self.fk_field, fk)
        setattr(instance, self.cache_attr, value)

class GenericRelation(RelatedField, Field):
    """Provides an accessor to generic related objects (e.g. comments)"""

    def __init__(self, to, **kwargs):
        kwargs['verbose_name'] = kwargs.get('verbose_name', None)
        kwargs['rel'] = GenericRel(to,
                            related_name=kwargs.pop('related_name', None),
                            limit_choices_to=kwargs.pop('limit_choices_to', None),
                            symmetrical=kwargs.pop('symmetrical', True))

        # Override content-type/object-id field names on the related class
        self.object_id_field_name = kwargs.pop("object_id_field", "object_id")
        self.content_type_field_name = kwargs.pop("content_type_field", "content_type")

        kwargs['blank'] = True
        kwargs['editable'] = False
        kwargs['serialize'] = False
        Field.__init__(self, **kwargs)

    def get_choices_default(self):
        return Field.get_choices(self, include_blank=False)

    def value_to_string(self, obj):
        qs = getattr(obj, self.name).all()
        return smart_unicode([instance._get_pk_val() for instance in qs])

    def m2m_db_table(self):
        return self.rel.to._meta.db_table

    def m2m_column_name(self):
        return self.object_id_field_name

    def m2m_reverse_name(self):
        return self.rel.to._meta.pk.column

    def contribute_to_class(self, cls, name):
        super(GenericRelation, self).contribute_to_class(cls, name)

        # Save a reference to which model this class is on for future use
        self.model = cls

        # Add the descriptor for the m2m relation
        setattr(cls, self.name, ReverseGenericRelatedObjectsDescriptor(self))

    def contribute_to_related_class(self, cls, related):
        pass

    def set_attributes_from_rel(self):
        pass

    def get_internal_type(self):
        return "ManyToManyField"

    def db_type(self, connection):
        # Since we're simulating a ManyToManyField, in effect, best return the
        # same db_type as well.
        return None

    def extra_filters(self, pieces, pos, negate):
        """
        Return an extra filter to the queryset so that the results are filtered
        on the appropriate content type.
        """
        if negate:
            return []
        ContentType = get_model("contenttypes", "contenttype")
        content_type = ContentType.objects.get_for_model(self.model)
        prefix = "__".join(pieces[:pos + 1])
        return [("%s__%s" % (prefix, self.content_type_field_name),
            content_type)]

class ReverseGenericRelatedObjectsDescriptor(object):
    """
    This class provides the functionality that makes the related-object
    managers available as attributes on a model class, for fields that have
    multiple "remote" values and have a GenericRelation defined in their model
    (rather than having another model pointed *at* them). In the example
    "article.publications", the publications attribute is a
    ReverseGenericRelatedObjectsDescriptor instance.
    """
    def __init__(self, field):
        self.field = field

    def __get__(self, instance, instance_type=None):
        if instance is None:
            return self

        # This import is done here to avoid circular import importing this module
        from django.contrib.contenttypes.models import ContentType

        # Dynamically create a class that subclasses the related model's
        # default manager.
        rel_model = self.field.rel.to
        superclass = rel_model._default_manager.__class__
        RelatedManager = create_generic_related_manager(superclass)

        qn = connection.ops.quote_name

        manager = RelatedManager(
            model = rel_model,
            instance = instance,
            symmetrical = (self.field.rel.symmetrical and instance.__class__ == rel_model),
            join_table = qn(self.field.m2m_db_table()),
            source_col_name = qn(self.field.m2m_column_name()),
            target_col_name = qn(self.field.m2m_reverse_name()),
            content_type = ContentType.objects.db_manager(instance._state.db).get_for_model(instance),
            content_type_field_name = self.field.content_type_field_name,
            object_id_field_name = self.field.object_id_field_name
        )

        return manager

    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError("Manager must be accessed via instance")

        manager = self.__get__(instance)
        manager.clear()
        for obj in value:
            manager.add(obj)

def create_generic_related_manager(superclass):
    """
    Factory function for a manager that subclasses 'superclass' (which is a
    Manager) and adds behavior for generic related objects.
    """

    class GenericRelatedObjectManager(superclass):
        def __init__(self, model=None, core_filters=None, instance=None, symmetrical=None,
                     join_table=None, source_col_name=None, target_col_name=None, content_type=None,
                     content_type_field_name=None, object_id_field_name=None):

            super(GenericRelatedObjectManager, self).__init__()
            self.core_filters = core_filters or {}
            self.model = model
            self.content_type = content_type
            self.symmetrical = symmetrical
            self.instance = instance
            self.join_table = join_table
            self.join_table = model._meta.db_table
            self.source_col_name = source_col_name
            self.target_col_name = target_col_name
            self.content_type_field_name = content_type_field_name
            self.object_id_field_name = object_id_field_name
            self.pk_val = self.instance._get_pk_val()

        def get_query_set(self):
            query = {
                '%s__pk' % self.content_type_field_name : self.content_type.id,
                '%s__exact' % self.object_id_field_name : self.pk_val,
            }
            return superclass.get_query_set(self).using(self.instance._state.db).filter(**query)

        def add(self, *objs):
            for obj in objs:
                if not isinstance(obj, self.model):
                    raise TypeError("'%s' instance expected" % self.model._meta.object_name)
                setattr(obj, self.content_type_field_name, self.content_type)
                setattr(obj, self.object_id_field_name, self.pk_val)
                obj.save()
        add.alters_data = True

        def remove(self, *objs):
            for obj in objs:
                obj.delete(using=self.instance._state.db)
        remove.alters_data = True

        def clear(self):
            for obj in self.all():
                obj.delete(using=self.instance._state.db)
        clear.alters_data = True

        def create(self, **kwargs):
            kwargs[self.content_type_field_name] = self.content_type
            kwargs[self.object_id_field_name] = self.pk_val
            return super(GenericRelatedObjectManager, self).create(**kwargs)
        create.alters_data = True

    return GenericRelatedObjectManager

class GenericRel(ManyToManyRel):
    def __init__(self, to, related_name=None, limit_choices_to=None, symmetrical=True):
        self.to = to
        self.related_name = related_name
        self.limit_choices_to = limit_choices_to or {}
        self.symmetrical = symmetrical
        self.multiple = True
        self.through = None

class BaseGenericInlineFormSet(BaseModelFormSet):
    """
    A formset for generic inline objects to a parent.
    """
    ct_field_name = "content_type"
    ct_fk_field_name = "object_id"

    def __init__(self, data=None, files=None, instance=None, save_as_new=None,
                 prefix=None, queryset=None):
        # Avoid a circular import.
        from django.contrib.contenttypes.models import ContentType
        opts = self.model._meta
        self.instance = instance
        self.rel_name = '-'.join((
            opts.app_label, opts.object_name.lower(),
            self.ct_field.name, self.ct_fk_field.name,
        ))
        if self.instance is None or self.instance.pk is None:
            qs = self.model._default_manager.none()
        else:
            if queryset is None:
                queryset = self.model._default_manager
            qs = queryset.filter(**{
                self.ct_field.name: ContentType.objects.get_for_model(self.instance),
                self.ct_fk_field.name: self.instance.pk,
            })
        super(BaseGenericInlineFormSet, self).__init__(
            queryset=qs, data=data, files=files,
            prefix=prefix
        )

    #@classmethod
    def get_default_prefix(cls):
        opts = cls.model._meta
        return '-'.join((opts.app_label, opts.object_name.lower(),
                        cls.ct_field.name, cls.ct_fk_field.name,
        ))
    get_default_prefix = classmethod(get_default_prefix)

    def save_new(self, form, commit=True):
        # Avoid a circular import.
        from django.contrib.contenttypes.models import ContentType
        kwargs = {
            self.ct_field.get_attname(): ContentType.objects.get_for_model(self.instance).pk,
            self.ct_fk_field.get_attname(): self.instance.pk,
        }
        new_obj = self.model(**kwargs)
        return save_instance(form, new_obj, commit=commit)

def generic_inlineformset_factory(model, form=ModelForm,
                                  formset=BaseGenericInlineFormSet,
                                  ct_field="content_type", fk_field="object_id",
                                  fields=None, exclude=None,
                                  extra=3, can_order=False, can_delete=True,
                                  max_num=0,
                                  formfield_callback=lambda f: f.formfield()):
    """
    Returns an ``GenericInlineFormSet`` for the given kwargs.

    You must provide ``ct_field`` and ``object_id`` if they different from the
    defaults ``content_type`` and ``object_id`` respectively.
    """
    opts = model._meta
    # Avoid a circular import.
    from django.contrib.contenttypes.models import ContentType
    # if there is no field called `ct_field` let the exception propagate
    ct_field = opts.get_field(ct_field)
    if not isinstance(ct_field, models.ForeignKey) or ct_field.rel.to != ContentType:
        raise Exception("fk_name '%s' is not a ForeignKey to ContentType" % ct_field)
    fk_field = opts.get_field(fk_field) # let the exception propagate
    if exclude is not None:
        exclude = list(exclude)
        exclude.extend([ct_field.name, fk_field.name])
    else:
        exclude = [ct_field.name, fk_field.name]
    FormSet = modelformset_factory(model, form=form,
                                   formfield_callback=formfield_callback,
                                   formset=formset,
                                   extra=extra, can_delete=can_delete, can_order=can_order,
                                   fields=fields, exclude=exclude, max_num=max_num)
    FormSet.ct_field = ct_field
    FormSet.ct_fk_field = fk_field
    return FormSet

class GenericInlineModelAdmin(InlineModelAdmin):
    ct_field = "content_type"
    ct_fk_field = "object_id"
    formset = BaseGenericInlineFormSet

    def get_formset(self, request, obj=None):
        if self.declared_fieldsets:
            fields = flatten_fieldsets(self.declared_fieldsets)
        else:
            fields = None
        defaults = {
            "ct_field": self.ct_field,
            "fk_field": self.ct_fk_field,
            "form": self.form,
            "formfield_callback": self.formfield_for_dbfield,
            "formset": self.formset,
            "extra": self.extra,
            "can_delete": True,
            "can_order": False,
            "fields": fields,
            "max_num": self.max_num,
            "exclude": self.exclude
        }
        return generic_inlineformset_factory(self.model, **defaults)

class GenericStackedInline(GenericInlineModelAdmin):
    template = 'admin/edit_inline/stacked.html'

class GenericTabularInline(GenericInlineModelAdmin):
    template = 'admin/edit_inline/tabular.html'
<commit_msg>Fixed #12628 - Don't ignore read-only fields in GenericInlineModelAdmin during save. Thanks, Alex Gaynor.<commit_after>"""
Classes allowing "generic" relations through ContentType and object-id fields.
"""

from django.core.exceptions import ObjectDoesNotExist
from django.db import connection
from django.db.models import signals
from django.db import models
from django.db.models.fields.related import RelatedField, Field, ManyToManyRel
from django.db.models.loading import get_model
from django.forms import ModelForm
from django.forms.models import BaseModelFormSet, modelformset_factory, save_instance
from django.contrib.admin.options import InlineModelAdmin, flatten_fieldsets
from django.utils.encoding import smart_unicode

class GenericForeignKey(object):
    """
    Provides a generic relation to any object through content-type/object-id
    fields.
    """

    def __init__(self, ct_field="content_type", fk_field="object_id"):
        self.ct_field = ct_field
        self.fk_field = fk_field

    def contribute_to_class(self, cls, name):
        self.name = name
        self.model = cls
        self.cache_attr = "_%s_cache" % name
        cls._meta.add_virtual_field(self)

        # For some reason I don't totally understand, using weakrefs here doesn't work.
        signals.pre_init.connect(self.instance_pre_init, sender=cls, weak=False)

        # Connect myself as the descriptor for this field
        setattr(cls, name, self)

    def instance_pre_init(self, signal, sender, args, kwargs, **_kwargs):
        """
        Handles initializing an object with the generic FK instaed of
        content-type/object-id fields.
        """
        if self.name in kwargs:
            value = kwargs.pop(self.name)
            kwargs[self.ct_field] = self.get_content_type(obj=value)
            kwargs[self.fk_field] = value._get_pk_val()

    def get_content_type(self, obj=None, id=None, using=None):
        # Convenience function using get_model avoids a circular import when
        # using this model
        ContentType = get_model("contenttypes", "contenttype")
        if obj:
             return ContentType.objects.db_manager(obj._state.db).get_for_model(obj)
        elif id:
             return ContentType.objects.db_manager(using).get_for_id(id)
        else:
            # This should never happen. I love comments like this, don't you?
            raise Exception("Impossible arguments to GFK.get_content_type!")

    def __get__(self, instance, instance_type=None):
        if instance is None:
            return self

        try:
            return getattr(instance, self.cache_attr)
        except AttributeError:
            rel_obj = None

            # Make sure to use ContentType.objects.get_for_id() to ensure that
            # lookups are cached (see ticket #5570). This takes more code than
            # the naive ``getattr(instance, self.ct_field)``, but has better
            # performance when dealing with GFKs in loops and such.
            f = self.model._meta.get_field(self.ct_field)
            ct_id = getattr(instance, f.get_attname(), None)
            if ct_id:
                ct = self.get_content_type(id=ct_id, using=instance._state.db)
                try:
                    rel_obj = ct.get_object_for_this_type(pk=getattr(instance, self.fk_field))
                except ObjectDoesNotExist:
                    pass
            setattr(instance, self.cache_attr, rel_obj)
            return rel_obj

    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError(u"%s must be accessed via instance" % self.related.opts.object_name)

        ct = None
        fk = None
        if value is not None:
            ct = self.get_content_type(obj=value)
            fk = value._get_pk_val()

        setattr(instance, self.ct_field, ct)
        setattr(instance, self.fk_field, fk)
        setattr(instance, self.cache_attr, value)

class GenericRelation(RelatedField, Field):
    """Provides an accessor to generic related objects (e.g. comments)"""

    def __init__(self, to, **kwargs):
        kwargs['verbose_name'] = kwargs.get('verbose_name', None)
        kwargs['rel'] = GenericRel(to,
                            related_name=kwargs.pop('related_name', None),
                            limit_choices_to=kwargs.pop('limit_choices_to', None),
                            symmetrical=kwargs.pop('symmetrical', True))

        # Override content-type/object-id field names on the related class
        self.object_id_field_name = kwargs.pop("object_id_field", "object_id")
        self.content_type_field_name = kwargs.pop("content_type_field", "content_type")

        kwargs['blank'] = True
        kwargs['editable'] = False
        kwargs['serialize'] = False
        Field.__init__(self, **kwargs)

    def get_choices_default(self):
        return Field.get_choices(self, include_blank=False)

    def value_to_string(self, obj):
        qs = getattr(obj, self.name).all()
        return smart_unicode([instance._get_pk_val() for instance in qs])

    def m2m_db_table(self):
        return self.rel.to._meta.db_table

    def m2m_column_name(self):
        return self.object_id_field_name

    def m2m_reverse_name(self):
        return self.rel.to._meta.pk.column

    def contribute_to_class(self, cls, name):
        super(GenericRelation, self).contribute_to_class(cls, name)

        # Save a reference to which model this class is on for future use
        self.model = cls

        # Add the descriptor for the m2m relation
        setattr(cls, self.name, ReverseGenericRelatedObjectsDescriptor(self))

    def contribute_to_related_class(self, cls, related):
        pass

    def set_attributes_from_rel(self):
        pass

    def get_internal_type(self):
        return "ManyToManyField"

    def db_type(self, connection):
        # Since we're simulating a ManyToManyField, in effect, best return the
        # same db_type as well.
        return None

    def extra_filters(self, pieces, pos, negate):
        """
        Return an extra filter to the queryset so that the results are filtered
        on the appropriate content type.
        """
        if negate:
            return []
        ContentType = get_model("contenttypes", "contenttype")
        content_type = ContentType.objects.get_for_model(self.model)
        prefix = "__".join(pieces[:pos + 1])
        return [("%s__%s" % (prefix, self.content_type_field_name),
            content_type)]

class ReverseGenericRelatedObjectsDescriptor(object):
    """
    This class provides the functionality that makes the related-object
    managers available as attributes on a model class, for fields that have
    multiple "remote" values and have a GenericRelation defined in their model
    (rather than having another model pointed *at* them). In the example
    "article.publications", the publications attribute is a
    ReverseGenericRelatedObjectsDescriptor instance.
    """
    def __init__(self, field):
        self.field = field

    def __get__(self, instance, instance_type=None):
        if instance is None:
            return self

        # This import is done here to avoid circular import importing this module
        from django.contrib.contenttypes.models import ContentType

        # Dynamically create a class that subclasses the related model's
        # default manager.
        rel_model = self.field.rel.to
        superclass = rel_model._default_manager.__class__
        RelatedManager = create_generic_related_manager(superclass)

        qn = connection.ops.quote_name

        manager = RelatedManager(
            model = rel_model,
            instance = instance,
            symmetrical = (self.field.rel.symmetrical and instance.__class__ == rel_model),
            join_table = qn(self.field.m2m_db_table()),
            source_col_name = qn(self.field.m2m_column_name()),
            target_col_name = qn(self.field.m2m_reverse_name()),
            content_type = ContentType.objects.db_manager(instance._state.db).get_for_model(instance),
            content_type_field_name = self.field.content_type_field_name,
            object_id_field_name = self.field.object_id_field_name
        )

        return manager

    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError("Manager must be accessed via instance")

        manager = self.__get__(instance)
        manager.clear()
        for obj in value:
            manager.add(obj)

def create_generic_related_manager(superclass):
    """
    Factory function for a manager that subclasses 'superclass' (which is a
    Manager) and adds behavior for generic related objects.
    """

    class GenericRelatedObjectManager(superclass):
        def __init__(self, model=None, core_filters=None, instance=None, symmetrical=None,
                     join_table=None, source_col_name=None, target_col_name=None, content_type=None,
                     content_type_field_name=None, object_id_field_name=None):

            super(GenericRelatedObjectManager, self).__init__()
            self.core_filters = core_filters or {}
            self.model = model
            self.content_type = content_type
            self.symmetrical = symmetrical
            self.instance = instance
            self.join_table = join_table
            self.join_table = model._meta.db_table
            self.source_col_name = source_col_name
            self.target_col_name = target_col_name
            self.content_type_field_name = content_type_field_name
            self.object_id_field_name = object_id_field_name
            self.pk_val = self.instance._get_pk_val()

        def get_query_set(self):
            query = {
                '%s__pk' % self.content_type_field_name : self.content_type.id,
                '%s__exact' % self.object_id_field_name : self.pk_val,
            }
            return superclass.get_query_set(self).using(self.instance._state.db).filter(**query)

        def add(self, *objs):
            for obj in objs:
                if not isinstance(obj, self.model):
                    raise TypeError("'%s' instance expected" % self.model._meta.object_name)
                setattr(obj, self.content_type_field_name, self.content_type)
                setattr(obj, self.object_id_field_name, self.pk_val)
                obj.save()
        add.alters_data = True

        def remove(self, *objs):
            for obj in objs:
                obj.delete(using=self.instance._state.db)
        remove.alters_data = True

        def clear(self):
            for obj in self.all():
                obj.delete(using=self.instance._state.db)
        clear.alters_data = True

        def create(self, **kwargs):
            kwargs[self.content_type_field_name] = self.content_type
            kwargs[self.object_id_field_name] = self.pk_val
            return super(GenericRelatedObjectManager, self).create(**kwargs)
        create.alters_data = True

    return GenericRelatedObjectManager

class GenericRel(ManyToManyRel):
    def __init__(self, to, related_name=None, limit_choices_to=None, symmetrical=True):
        self.to = to
        self.related_name = related_name
        self.limit_choices_to = limit_choices_to or {}
        self.symmetrical = symmetrical
        self.multiple = True
        self.through = None

class BaseGenericInlineFormSet(BaseModelFormSet):
    """
    A formset for generic inline objects to a parent.
    """
    ct_field_name = "content_type"
    ct_fk_field_name = "object_id"

    def __init__(self, data=None, files=None, instance=None, save_as_new=None,
                 prefix=None, queryset=None):
        # Avoid a circular import.
        from django.contrib.contenttypes.models import ContentType
        opts = self.model._meta
        self.instance = instance
        self.rel_name = '-'.join((
            opts.app_label, opts.object_name.lower(),
            self.ct_field.name, self.ct_fk_field.name,
        ))
        if self.instance is None or self.instance.pk is None:
            qs = self.model._default_manager.none()
        else:
            if queryset is None:
                queryset = self.model._default_manager
            qs = queryset.filter(**{
                self.ct_field.name: ContentType.objects.get_for_model(self.instance),
                self.ct_fk_field.name: self.instance.pk,
            })
        super(BaseGenericInlineFormSet, self).__init__(
            queryset=qs, data=data, files=files,
            prefix=prefix
        )

    #@classmethod
    def get_default_prefix(cls):
        opts = cls.model._meta
        return '-'.join((opts.app_label, opts.object_name.lower(),
                        cls.ct_field.name, cls.ct_fk_field.name,
        ))
    get_default_prefix = classmethod(get_default_prefix)

    def save_new(self, form, commit=True):
        # Avoid a circular import.
        from django.contrib.contenttypes.models import ContentType
        kwargs = {
            self.ct_field.get_attname(): ContentType.objects.get_for_model(self.instance).pk,
            self.ct_fk_field.get_attname(): self.instance.pk,
        }
        new_obj = self.model(**kwargs)
        return save_instance(form, new_obj, commit=commit)

def generic_inlineformset_factory(model, form=ModelForm,
                                  formset=BaseGenericInlineFormSet,
                                  ct_field="content_type", fk_field="object_id",
                                  fields=None, exclude=None,
                                  extra=3, can_order=False, can_delete=True,
                                  max_num=0,
                                  formfield_callback=lambda f: f.formfield()):
    """
    Returns an ``GenericInlineFormSet`` for the given kwargs.

    You must provide ``ct_field`` and ``object_id`` if they different from the
    defaults ``content_type`` and ``object_id`` respectively.
    """
    opts = model._meta
    # Avoid a circular import.
    from django.contrib.contenttypes.models import ContentType
    # if there is no field called `ct_field` let the exception propagate
    ct_field = opts.get_field(ct_field)
    if not isinstance(ct_field, models.ForeignKey) or ct_field.rel.to != ContentType:
        raise Exception("fk_name '%s' is not a ForeignKey to ContentType" % ct_field)
    fk_field = opts.get_field(fk_field) # let the exception propagate
    if exclude is not None:
        exclude = list(exclude)
        exclude.extend([ct_field.name, fk_field.name])
    else:
        exclude = [ct_field.name, fk_field.name]
    FormSet = modelformset_factory(model, form=form,
                                   formfield_callback=formfield_callback,
                                   formset=formset,
                                   extra=extra, can_delete=can_delete, can_order=can_order,
                                   fields=fields, exclude=exclude, max_num=max_num)
    FormSet.ct_field = ct_field
    FormSet.ct_fk_field = fk_field
    return FormSet

class GenericInlineModelAdmin(InlineModelAdmin):
    ct_field = "content_type"
    ct_fk_field = "object_id"
    formset = BaseGenericInlineFormSet

    def get_formset(self, request, obj=None):
        if self.declared_fieldsets:
            fields = flatten_fieldsets(self.declared_fieldsets)
        else:
            fields = None
        if self.exclude is None:
            exclude = []
        else:
            exclude = list(self.exclude)
        exclude.extend(self.get_readonly_fields(request, obj))
        exclude = exclude or None
        defaults = {
            "ct_field": self.ct_field,
            "fk_field": self.ct_fk_field,
            "form": self.form,
            "formfield_callback": self.formfield_for_dbfield,
            "formset": self.formset,
            "extra": self.extra,
            "can_delete": True,
            "can_order": False,
            "fields": fields,
            "max_num": self.max_num,
            "exclude": exclude
        }
        return generic_inlineformset_factory(self.model, **defaults)

class GenericStackedInline(GenericInlineModelAdmin):
    template = 'admin/edit_inline/stacked.html'

class GenericTabularInline(GenericInlineModelAdmin):
    template = 'admin/edit_inline/tabular.html'
<|endoftext|>


<commit_before>"""
Classes allowing "generic" relations through ContentType and object-id fields.
"""

from django import oldforms
from django.core.exceptions import ObjectDoesNotExist
from django.db import connection
from django.db.models import signals
from django.db.models.fields.related import RelatedField, Field, ManyToManyRel
from django.db.models.loading import get_model
from django.dispatch import dispatcher
from django.utils.functional import curry

class GenericForeignKey(object):
    """
    Provides a generic relation to any object through content-type/object-id
    fields.
    """

    def __init__(self, ct_field="content_type", fk_field="object_id"):
        self.ct_field = ct_field
        self.fk_field = fk_field

    def contribute_to_class(self, cls, name):
        # Make sure the fields exist (these raise FieldDoesNotExist,
        # which is a fine error to raise here)
        self.name = name
        self.model = cls
        self.cache_attr = "_%s_cache" % name

        # For some reason I don't totally understand, using weakrefs here doesn't work.
        dispatcher.connect(self.instance_pre_init, signal=signals.pre_init, sender=cls, weak=False)

        # Connect myself as the descriptor for this field
        setattr(cls, name, self)

    def instance_pre_init(self, signal, sender, args, kwargs):
        # Handle initalizing an object with the generic FK instaed of
        # content-type/object-id fields.
        if self.name in kwargs:
            value = kwargs.pop(self.name)
            kwargs[self.ct_field] = self.get_content_type(obj=value)
            kwargs[self.fk_field] = value._get_pk_val()

    def get_content_type(self, obj=None, id=None):
        # Convenience function using get_model avoids a circular import when using this model
        ContentType = get_model("contenttypes", "contenttype")
        if obj:
            return ContentType.objects.get_for_model(obj)
        elif id:
            return ContentType.objects.get_for_id(id)
        else:
            # This should never happen. I love comments like this, don't you?
            raise Exception("Impossible arguments to GFK.get_content_type!")

    def __get__(self, instance, instance_type=None):
        if instance is None:
            raise AttributeError, u"%s must be accessed via instance" % self.name

        try:
            return getattr(instance, self.cache_attr)
        except AttributeError:
            rel_obj = None

            # Make sure to use ContentType.objects.get_for_id() to ensure that
            # lookups are cached (see ticket #5570). This takes more code than
            # the naive ``getattr(instance, self.ct_field)``, but has better
            # performance when dealing with GFKs in loops and such.
            f = self.model._meta.get_field(self.ct_field)
            ct_id = getattr(instance, f.get_attname(), None)
            if ct_id:
                ct = self.get_content_type(id=ct_id)
                try:
                    rel_obj = ct.get_object_for_this_type(pk=getattr(instance, self.fk_field))
                except ObjectDoesNotExist:
                    pass
            setattr(instance, self.cache_attr, rel_obj)
            return rel_obj

    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError, u"%s must be accessed via instance" % self.related.opts.object_name

        ct = None
        fk = None
        if value is not None:
            ct = self.get_content_type(obj=value)
            fk = value._get_pk_val()

        setattr(instance, self.ct_field, ct)
        setattr(instance, self.fk_field, fk)
        setattr(instance, self.cache_attr, value)

class GenericRelation(RelatedField, Field):
    """Provides an accessor to generic related objects (i.e. comments)"""

    def __init__(self, to, **kwargs):
        kwargs['verbose_name'] = kwargs.get('verbose_name', None)
        kwargs['rel'] = GenericRel(to,
                            related_name=kwargs.pop('related_name', None),
                            limit_choices_to=kwargs.pop('limit_choices_to', None),
                            symmetrical=kwargs.pop('symmetrical', True))

        # Override content-type/object-id field names on the related class
        self.object_id_field_name = kwargs.pop("object_id_field", "object_id")
        self.content_type_field_name = kwargs.pop("content_type_field", "content_type")

        kwargs['blank'] = True
        kwargs['editable'] = False
        kwargs['serialize'] = False
        Field.__init__(self, **kwargs)

    def get_manipulator_field_objs(self):
        choices = self.get_choices_default()
        return [curry(oldforms.SelectMultipleField, size=min(max(len(choices), 5), 15), choices=choices)]

    def get_choices_default(self):
        return Field.get_choices(self, include_blank=False)

    def flatten_data(self, follow, obj = None):
        new_data = {}
        if obj:
            instance_ids = [instance._get_pk_val() for instance in getattr(obj, self.name).all()]
            new_data[self.name] = instance_ids
        return new_data

    def m2m_db_table(self):
        return self.rel.to._meta.db_table

    def m2m_column_name(self):
        return self.object_id_field_name

    def m2m_reverse_name(self):
        return self.model._meta.pk.column

    def contribute_to_class(self, cls, name):
        super(GenericRelation, self).contribute_to_class(cls, name)

        # Save a reference to which model this class is on for future use
        self.model = cls

        # Add the descriptor for the m2m relation
        setattr(cls, self.name, ReverseGenericRelatedObjectsDescriptor(self))

    def contribute_to_related_class(self, cls, related):
        pass

    def set_attributes_from_rel(self):
        pass

    def get_internal_type(self):
        return "ManyToManyField"

class ReverseGenericRelatedObjectsDescriptor(object):
    """
    This class provides the functionality that makes the related-object
    managers available as attributes on a model class, for fields that have
    multiple "remote" values and have a GenericRelation defined in their model
    (rather than having another model pointed *at* them). In the example
    "article.publications", the publications attribute is a
    ReverseGenericRelatedObjectsDescriptor instance.
    """
    def __init__(self, field):
        self.field = field

    def __get__(self, instance, instance_type=None):
        if instance is None:
            raise AttributeError, "Manager must be accessed via instance"

        # This import is done here to avoid circular import importing this module
        from django.contrib.contenttypes.models import ContentType

        # Dynamically create a class that subclasses the related model's
        # default manager.
        rel_model = self.field.rel.to
        superclass = rel_model._default_manager.__class__
        RelatedManager = create_generic_related_manager(superclass)

        qn = connection.ops.quote_name

        manager = RelatedManager(
            model = rel_model,
            instance = instance,
            symmetrical = (self.field.rel.symmetrical and instance.__class__ == rel_model),
            join_table = qn(self.field.m2m_db_table()),
            source_col_name = qn(self.field.m2m_column_name()),
            target_col_name = qn(self.field.m2m_reverse_name()),
            content_type = ContentType.objects.get_for_model(self.field.model),
            content_type_field_name = self.field.content_type_field_name,
            object_id_field_name = self.field.object_id_field_name
        )

        return manager

    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError, "Manager must be accessed via instance"

        manager = self.__get__(instance)
        manager.clear()
        for obj in value:
            manager.add(obj)

def create_generic_related_manager(superclass):
    """
    Factory function for a manager that subclasses 'superclass' (which is a
    Manager) and adds behavior for generic related objects.
    """

    class GenericRelatedObjectManager(superclass):
        def __init__(self, model=None, core_filters=None, instance=None, symmetrical=None,
                     join_table=None, source_col_name=None, target_col_name=None, content_type=None,
                     content_type_field_name=None, object_id_field_name=None):

            super(GenericRelatedObjectManager, self).__init__()
            self.core_filters = core_filters or {}
            self.model = model
            self.content_type = content_type
            self.symmetrical = symmetrical
            self.instance = instance
            self.join_table = join_table
            self.join_table = model._meta.db_table
            self.source_col_name = source_col_name
            self.target_col_name = target_col_name
            self.content_type_field_name = content_type_field_name
            self.object_id_field_name = object_id_field_name
            self.pk_val = self.instance._get_pk_val()

        def get_query_set(self):
            query = {
                '%s__pk' % self.content_type_field_name : self.content_type.id,
                '%s__exact' % self.object_id_field_name : self.pk_val,
            }
            return superclass.get_query_set(self).filter(**query)

        def add(self, *objs):
            for obj in objs:
                setattr(obj, self.content_type_field_name, self.content_type)
                setattr(obj, self.object_id_field_name, self.pk_val)
                obj.save()
        add.alters_data = True

        def remove(self, *objs):
            for obj in objs:
                obj.delete()
        remove.alters_data = True

        def clear(self):
            for obj in self.all():
                obj.delete()
        clear.alters_data = True

        def create(self, **kwargs):
            kwargs[self.content_type_field_name] = self.content_type
            kwargs[self.object_id_field_name] = self.pk_val
            obj = self.model(**kwargs)
            obj.save()
            return obj
        create.alters_data = True

    return GenericRelatedObjectManager

class GenericRel(ManyToManyRel):
    def __init__(self, to, related_name=None, limit_choices_to=None, symmetrical=True):
        self.to = to
        self.num_in_admin = 0
        self.related_name = related_name
        self.filter_interface = None
        self.limit_choices_to = limit_choices_to or {}
        self.edit_inline = False
        self.raw_id_admin = False
        self.symmetrical = symmetrical
        self.multiple = True
        assert not (self.raw_id_admin and self.filter_interface), \
            "Generic relations may not use both raw_id_admin and filter_interface"
<commit_msg>Fixed #7017 -- Minor typo fix in a comment. Thanks FranRuiz.<commit_after>"""
Classes allowing "generic" relations through ContentType and object-id fields.
"""

from django import oldforms
from django.core.exceptions import ObjectDoesNotExist
from django.db import connection
from django.db.models import signals
from django.db.models.fields.related import RelatedField, Field, ManyToManyRel
from django.db.models.loading import get_model
from django.dispatch import dispatcher
from django.utils.functional import curry

class GenericForeignKey(object):
    """
    Provides a generic relation to any object through content-type/object-id
    fields.
    """

    def __init__(self, ct_field="content_type", fk_field="object_id"):
        self.ct_field = ct_field
        self.fk_field = fk_field

    def contribute_to_class(self, cls, name):
        # Make sure the fields exist (these raise FieldDoesNotExist,
        # which is a fine error to raise here)
        self.name = name
        self.model = cls
        self.cache_attr = "_%s_cache" % name

        # For some reason I don't totally understand, using weakrefs here doesn't work.
        dispatcher.connect(self.instance_pre_init, signal=signals.pre_init, sender=cls, weak=False)

        # Connect myself as the descriptor for this field
        setattr(cls, name, self)

    def instance_pre_init(self, signal, sender, args, kwargs):
        """
        Handles initializing an object with the generic FK instaed of
        content-type/object-id fields.
        """
        if self.name in kwargs:
            value = kwargs.pop(self.name)
            kwargs[self.ct_field] = self.get_content_type(obj=value)
            kwargs[self.fk_field] = value._get_pk_val()

    def get_content_type(self, obj=None, id=None):
        # Convenience function using get_model avoids a circular import when
        # using this model
        ContentType = get_model("contenttypes", "contenttype")
        if obj:
            return ContentType.objects.get_for_model(obj)
        elif id:
            return ContentType.objects.get_for_id(id)
        else:
            # This should never happen. I love comments like this, don't you?
            raise Exception("Impossible arguments to GFK.get_content_type!")

    def __get__(self, instance, instance_type=None):
        if instance is None:
            raise AttributeError, u"%s must be accessed via instance" % self.name

        try:
            return getattr(instance, self.cache_attr)
        except AttributeError:
            rel_obj = None

            # Make sure to use ContentType.objects.get_for_id() to ensure that
            # lookups are cached (see ticket #5570). This takes more code than
            # the naive ``getattr(instance, self.ct_field)``, but has better
            # performance when dealing with GFKs in loops and such.
            f = self.model._meta.get_field(self.ct_field)
            ct_id = getattr(instance, f.get_attname(), None)
            if ct_id:
                ct = self.get_content_type(id=ct_id)
                try:
                    rel_obj = ct.get_object_for_this_type(pk=getattr(instance, self.fk_field))
                except ObjectDoesNotExist:
                    pass
            setattr(instance, self.cache_attr, rel_obj)
            return rel_obj

    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError, u"%s must be accessed via instance" % self.related.opts.object_name

        ct = None
        fk = None
        if value is not None:
            ct = self.get_content_type(obj=value)
            fk = value._get_pk_val()

        setattr(instance, self.ct_field, ct)
        setattr(instance, self.fk_field, fk)
        setattr(instance, self.cache_attr, value)

class GenericRelation(RelatedField, Field):
    """Provides an accessor to generic related objects (i.e. comments)"""

    def __init__(self, to, **kwargs):
        kwargs['verbose_name'] = kwargs.get('verbose_name', None)
        kwargs['rel'] = GenericRel(to,
                            related_name=kwargs.pop('related_name', None),
                            limit_choices_to=kwargs.pop('limit_choices_to', None),
                            symmetrical=kwargs.pop('symmetrical', True))

        # Override content-type/object-id field names on the related class
        self.object_id_field_name = kwargs.pop("object_id_field", "object_id")
        self.content_type_field_name = kwargs.pop("content_type_field", "content_type")

        kwargs['blank'] = True
        kwargs['editable'] = False
        kwargs['serialize'] = False
        Field.__init__(self, **kwargs)

    def get_manipulator_field_objs(self):
        choices = self.get_choices_default()
        return [curry(oldforms.SelectMultipleField, size=min(max(len(choices), 5), 15), choices=choices)]

    def get_choices_default(self):
        return Field.get_choices(self, include_blank=False)

    def flatten_data(self, follow, obj = None):
        new_data = {}
        if obj:
            instance_ids = [instance._get_pk_val() for instance in getattr(obj, self.name).all()]
            new_data[self.name] = instance_ids
        return new_data

    def m2m_db_table(self):
        return self.rel.to._meta.db_table

    def m2m_column_name(self):
        return self.object_id_field_name

    def m2m_reverse_name(self):
        return self.model._meta.pk.column

    def contribute_to_class(self, cls, name):
        super(GenericRelation, self).contribute_to_class(cls, name)

        # Save a reference to which model this class is on for future use
        self.model = cls

        # Add the descriptor for the m2m relation
        setattr(cls, self.name, ReverseGenericRelatedObjectsDescriptor(self))

    def contribute_to_related_class(self, cls, related):
        pass

    def set_attributes_from_rel(self):
        pass

    def get_internal_type(self):
        return "ManyToManyField"

class ReverseGenericRelatedObjectsDescriptor(object):
    """
    This class provides the functionality that makes the related-object
    managers available as attributes on a model class, for fields that have
    multiple "remote" values and have a GenericRelation defined in their model
    (rather than having another model pointed *at* them). In the example
    "article.publications", the publications attribute is a
    ReverseGenericRelatedObjectsDescriptor instance.
    """
    def __init__(self, field):
        self.field = field

    def __get__(self, instance, instance_type=None):
        if instance is None:
            raise AttributeError, "Manager must be accessed via instance"

        # This import is done here to avoid circular import importing this module
        from django.contrib.contenttypes.models import ContentType

        # Dynamically create a class that subclasses the related model's
        # default manager.
        rel_model = self.field.rel.to
        superclass = rel_model._default_manager.__class__
        RelatedManager = create_generic_related_manager(superclass)

        qn = connection.ops.quote_name

        manager = RelatedManager(
            model = rel_model,
            instance = instance,
            symmetrical = (self.field.rel.symmetrical and instance.__class__ == rel_model),
            join_table = qn(self.field.m2m_db_table()),
            source_col_name = qn(self.field.m2m_column_name()),
            target_col_name = qn(self.field.m2m_reverse_name()),
            content_type = ContentType.objects.get_for_model(self.field.model),
            content_type_field_name = self.field.content_type_field_name,
            object_id_field_name = self.field.object_id_field_name
        )

        return manager

    def __set__(self, instance, value):
        if instance is None:
            raise AttributeError, "Manager must be accessed via instance"

        manager = self.__get__(instance)
        manager.clear()
        for obj in value:
            manager.add(obj)

def create_generic_related_manager(superclass):
    """
    Factory function for a manager that subclasses 'superclass' (which is a
    Manager) and adds behavior for generic related objects.
    """

    class GenericRelatedObjectManager(superclass):
        def __init__(self, model=None, core_filters=None, instance=None, symmetrical=None,
                     join_table=None, source_col_name=None, target_col_name=None, content_type=None,
                     content_type_field_name=None, object_id_field_name=None):

            super(GenericRelatedObjectManager, self).__init__()
            self.core_filters = core_filters or {}
            self.model = model
            self.content_type = content_type
            self.symmetrical = symmetrical
            self.instance = instance
            self.join_table = join_table
            self.join_table = model._meta.db_table
            self.source_col_name = source_col_name
            self.target_col_name = target_col_name
            self.content_type_field_name = content_type_field_name
            self.object_id_field_name = object_id_field_name
            self.pk_val = self.instance._get_pk_val()

        def get_query_set(self):
            query = {
                '%s__pk' % self.content_type_field_name : self.content_type.id,
                '%s__exact' % self.object_id_field_name : self.pk_val,
            }
            return superclass.get_query_set(self).filter(**query)

        def add(self, *objs):
            for obj in objs:
                setattr(obj, self.content_type_field_name, self.content_type)
                setattr(obj, self.object_id_field_name, self.pk_val)
                obj.save()
        add.alters_data = True

        def remove(self, *objs):
            for obj in objs:
                obj.delete()
        remove.alters_data = True

        def clear(self):
            for obj in self.all():
                obj.delete()
        clear.alters_data = True

        def create(self, **kwargs):
            kwargs[self.content_type_field_name] = self.content_type
            kwargs[self.object_id_field_name] = self.pk_val
            obj = self.model(**kwargs)
            obj.save()
            return obj
        create.alters_data = True

    return GenericRelatedObjectManager

class GenericRel(ManyToManyRel):
    def __init__(self, to, related_name=None, limit_choices_to=None, symmetrical=True):
        self.to = to
        self.num_in_admin = 0
        self.related_name = related_name
        self.filter_interface = None
        self.limit_choices_to = limit_choices_to or {}
        self.edit_inline = False
        self.raw_id_admin = False
        self.symmetrical = symmetrical
        self.multiple = True
        assert not (self.raw_id_admin and self.filter_interface), \
            "Generic relations may not use both raw_id_admin and filter_interface"
<|endoftext|>


<commit_before>from django.contrib.redirects.models import Redirect
from django import http
from django.conf import settings

class RedirectFallbackMiddleware(object):
    def process_response(self, request, response):
        if response.status_code != 404:
            return response # No need to check for a redirect for non-404 responses.
        path = request.get_full_path()
        try:
            r = Redirect.objects.get(site__id__exact=settings.SITE_ID, old_path=path)
        except Redirect.DoesNotExist:
            r = None
        if r is None and settings.APPEND_SLASH:
            # Try removing the trailing slash.
            try:
                r = Redirect.objects.get(site__id__exact=settings.SITE_ID,
                    old_path=path[:path.rfind('/')]+path[path.rfind('/')+1:])
            except Redirect.DoesNotExist:
                pass
        if r is not None:
            if r == '':
                return http.HttpResponseGone()
            return http.HttpResponsePermanentRedirect(r.new_path)

        # No redirect was found. Return the response.
        return response
<commit_msg>Fixed #4618 -- Fixed error in django.contrib.redirects new_path handling. Thanks, cephelo@gmail.com<commit_after>from django.contrib.redirects.models import Redirect
from django import http
from django.conf import settings

class RedirectFallbackMiddleware(object):
    def process_response(self, request, response):
        if response.status_code != 404:
            return response # No need to check for a redirect for non-404 responses.
        path = request.get_full_path()
        try:
            r = Redirect.objects.get(site__id__exact=settings.SITE_ID, old_path=path)
        except Redirect.DoesNotExist:
            r = None
        if r is None and settings.APPEND_SLASH:
            # Try removing the trailing slash.
            try:
                r = Redirect.objects.get(site__id__exact=settings.SITE_ID,
                    old_path=path[:path.rfind('/')]+path[path.rfind('/')+1:])
            except Redirect.DoesNotExist:
                pass
        if r is not None:
            if r.new_path == '':
                return http.HttpResponseGone()
            return http.HttpResponsePermanentRedirect(r.new_path)

        # No redirect was found. Return the response.
        return response
<|endoftext|>