<commit_before>'''
Created on 2 Dec 2014

@author: jmht
'''
import clipper
import logging
import os
import shutil
import sys

from iotbx import reflection_file_reader

from simbad.util import simbad_util # Avoid circular dependencies

_logger = logging.getLogger()
_logger.setLevel(logging.DEBUG)

# TODO: Get rid of this function completely
def set_crystal_data(optd):
    """Set crystallographic parameters from mtz file"""

    space_group, resolution, cell_parameters = crystal_data(optd['mtz'])

    optd['space_group'] = space_group
    optd['resolution'] = resolution
    optd['cell_parameters'] = cell_parameters

    return optd

def crystal_data(mtz):
    """Set crystallographic parameters from mtz file

    Parameters
    ----------
    mtz : str
       The path to the mtz file

    Returns
    -------
    space_group : str
       The space group
    resolution : str
       The resolution
    cell_parameters : str
       The cell parameters

    """

    hkl_info = clipper.HKL_info()
    mtz_file = clipper.CCP4MTZfile()
    mtz_file.open_read(mtz)
    mtz_file.import_hkl_info(hkl_info)

    sg, cell = hkl_info.spacegroup(), hkl_info.cell()

    space_group = sg.symbol_hm()
    space_group = space_group.replace(" ", "")

    resolution = "%.2lf" % hkl_info.resolution().limit()
    cell_parameters = "%.2lf %.2lf %.2lf %.2lf %.2lf %.2lf" % (cell.a(),
                                                               cell.b(),
                                                               cell.c(),
                                                               cell.alpha_deg(),
                                                               cell.beta_deg(),
                                                               cell.gamma_deg())

    return space_group, resolution, cell_parameters

def del_column(file_name, column, overwrite=True):
    """Delete a column from an mtz file and return a path to the file"""
    mtzDel = simbad_util.filename_append(file_name, "d{0}".format(column) )
    cmd = [ "mtzutils", "hklin1", file_name, "hklout", mtzDel ]
    stdin = "EXCLUDE 1 {0}".format( column )
    logfile = os.path.join( os.getcwd(), "mtzutils.log" )
    retcode = simbad_util.run_command(cmd, stdin=stdin, logfile=logfile)
    if retcode != 0:
        msg = "Error running mtzutils. Check the logfile: {0}".format(logfile)
        _logger.critical(msg)
        raise RuntimeError, msg

    if overwrite:
        shutil.move(mtzDel,file_name)
        return file_name
    else:
        return mtzDel

def add_rfree(file_name,directory=None,overwrite=True):
    """Run uniqueify on mtz file to generate RFREE data column"""
    mtzUnique = simbad_util.filename_append(file_name, "uniqueify", directory=directory)

    cmd = ['uniqueify', file_name, mtzUnique]
    logfile = os.path.join( os.getcwd(), "uniqueify.log" )
    retcode = simbad_util.run_command(cmd, logfile=logfile)
    if retcode != 0:
        msg = "Error running command: {0}. Check the logfile: {1}".format(" ".join(cmd),logfile)
        _logger.critical(msg)
        raise RuntimeError, msg

    if overwrite:
        shutil.move(mtzUnique,file_name)
        return file_name
    else:
        return mtzUnique

def get_labels(file_name):
    """Return the F, FP and FREE column labels"""

    reflection_file = reflection_file_reader.any_reflection_file(file_name=file_name)
    if not reflection_file.file_type()=="ccp4_mtz":
        msg="File is not of type ccp4_mtz: {0}".format(file_name)
        logging.critical(msg)
        raise RuntimeError,msg

    content=reflection_file.file_content()
    ctypes=content.column_types()
    clabels=content.column_labels()
    ftype='F'
    dtype='D'

    if not ftype in ctypes:
        raise RuntimeError,"Cannot find any structure amplitudes in: {0}".format(file_name)
    F=clabels[ctypes.index(ftype)]

    # FP derived from F
    FP='SIG'+F
    if not FP in clabels:
        raise RuntimeError,"Cannot find label {0} in file: {1}".format(FP,file_name)

    try:
        if not dtype in ctypes:
            raise RuntimeError, "Cannot find any structure amplitudes in: {0}".format(file_name)
        DANO = clabels[ctypes.index(dtype)]

        # SIGDANO derived from DANO
        SIGDANO = 'SIG' + DANO
        if not SIGDANO in clabels:
            raise RuntimeError, "Cannot find label {0} in file: {1}".format(SIGDANO, file_name)
    except RuntimeError:
        DANO, SIGDANO = None, None

    FREE=_get_rfree(content)

    return F,FP,DANO,SIGDANO,FREE

def get_rfree(file_name):
    """Return the Rfree label"""

    reflection_file = reflection_file_reader.any_reflection_file(file_name=file_name)
    if not reflection_file.file_type()=="ccp4_mtz":
        msg="File is not of type ccp4_mtz: {0}".format(file_name)
        logging.critical(msg)
        raise RuntimeError,msg

    # Read the file
    content=reflection_file.file_content()
    return _get_rfree(content)

def _get_rfree(content):
    rfree_label=None
    #print "GOT ",content.column_labels()
    for label in content.column_labels():
        if 'free' in label.lower():
            column = content.get_column(label=label)
            selection_valid = column.selection_valid()
            flags = column.extract_values()
            sel_0 = (flags == 0)
            # extract number of work/test reflections
            n0=( sel_0 & selection_valid).count(True)
            n1=(~sel_0 & selection_valid).count(True)
            #print "Number of 0 (work):",n0
            #print "Number of 1 (test):",n1
            #print float(n0)/float(n1)*100
            if n0>0 and n1>0:
                if rfree_label:
                    _logger.warning("FOUND >1 RFREE label in file!")
                rfree_label=label
    return rfree_label

def to_hkl(mtz_file,hkl_file=None,directory=None,F=None,SIGF=None,FREE=None):

    if directory is None:
        directory=os.getcwd()

    if hkl_file is None:
        name=os.path.splitext(os.path.basename(mtz_file))[0]
        hkl_file=os.path.join(directory,name+".hkl")

    if F is None or SIGF is None or FREE is None:
        F,SIGF,DANO,SIGDANO,FREE=get_labels(mtz_file)

    cmd=['mtz2various','HKLIN',mtz_file,'HKLOUT', hkl_file]
    logfile="mtz2various.log"
    stdin  = """LABIN FP={0} SIGFP={1} FREE={2}
OUTPUT SHELX
FSQUARED
END""".format(F,SIGF,FREE)

    ret = simbad_util.run_command(cmd=cmd, logfile=logfile, directory=None, dolog=False, stdin=stdin)
    if not ret==0:
        raise RuntimeError,"Error converting {0} to HKL format - see log: {1}".format(mtz_file,logfile)
    else:
        os.unlink(logfile)

    return hkl_file

def processReflectionFile(amoptd):
    """Make sure we have a valid mtz file. If necessary convert a given cif file.
       Set the mtz variable in the given amoptd to the reflection file to use
       Return True if it all worked or raise an exception if it failed
    """

    # Now have an mtz so check it's valid
    if not amoptd['mtz'] or not os.path.isfile( amoptd['mtz'] ):
        _logger.critical("Cannot find MTZ file: {0}".format( amoptd['mtz'] ) )
        sys.exit(1)

    # Get column label info
    reflection_file = reflection_file_reader.any_reflection_file(file_name=amoptd['mtz'])
    if not reflection_file.file_type()=="ccp4_mtz":
        _logger.critical("File is not of type ccp4_mtz: {0}".format( amoptd['mtz'] ) )
        sys.exit(1)

    # Read the file
    content=reflection_file.file_content()

    # Check any user-given flags

    for flag in ['F','SIGF','FREE']:
        if amoptd[flag] and amoptd[flag] not in content.column_labels():
            _logger.critical("Cannot find flag {0} label {1} in mtz file {2}".format( flag, amoptd[flag], amoptd['mtz'] ) )
            sys.exit(1)

    # If any of the flags aren't given we set defaults based on what's in the file
    if not amoptd['F']:
        if 'F' not in content.column_types():
            _logger.critical("Cannot find column type F for flag F in mtz file: {0}".format( amoptd['mtz'] ) )
            sys.exit(1)
        amoptd['F']  = content.column_labels()[content.column_types().index('F')]
    if not amoptd['SIGF']:
        l='SIG'+amoptd['F']
        if not l in content.column_labels():
            _logger.critical("Cannot find column type {0} for flag SIGF in mtz file: {0}".format( l, amoptd['mtz'] ) )
            sys.exit(1)
        amoptd['SIGF']  = l

    if amoptd['FREE']:
        # Check is valid
        rfree=_get_rfree(content)
        if not rfree or not rfree==amoptd['FREE']:
            _logger.critical("Given RFREE label {0} is not valid for mtz file: {0}".format( amoptd['FREE'], amoptd['mtz'] ) )
            sys.exit(1)
    else:
        # See if we can find a valid label in the file
        rfree=_get_rfree(content)
        if not rfree:
            # Need to generate RFREE
            _logger.warning("Cannot find a valid FREE flag - running uniquefy to generate column with RFREE data." )
            amoptd['mtz'] = add_rfree( amoptd['mtz'], directory=amoptd['work_dir'],overwrite=False)

            # Check file and get new FREE flag
            rfree=get_rfree(amoptd['mtz'])
            if not rfree:
                _logger.critical("Cannot find valid rfree flag in mtz file {0} after running uniquiefy".format(amoptd['mtz']))
                sys.exit(1)
        amoptd['FREE']  = rfree

    # Return anomalous data labels if the columns exist in the file
    if not amoptd['DANO']:
        if 'D' not in content.column_types():
            _logger.critical("Cannot find column type D for flag DANO in mtz file: {0}".format(amoptd['mtz']))
        else:
            amoptd['DANO'] = content.column_labels()[content.column_types().index('D')]

    if not amoptd['SIGDANO'] and amoptd['DANO']:
        l = 'SIG' + amoptd['DANO']
        if not l in content.column_labels():
            _logger.critical("Cannot find column type {0} for flag SIGDANO in mtz file: {0}".format( l, amoptd['mtz'] ) )
        else:
            amoptd['SIGDANO'] = l

    return True

<commit_msg>removed code smells<commit_after>'''
Created on 2 Dec 2014

@author: jmht
'''
import clipper
import logging
import os
import shutil
import sys

from iotbx import reflection_file_reader

from simbad.util import simbad_util # Avoid circular dependencies

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

# TODO: Get rid of this function completely
def set_crystal_data(optd):
    """Set crystallographic parameters from mtz file"""

    space_group, resolution, cell_parameters = crystal_data(optd['mtz'])

    optd['space_group'] = space_group
    optd['resolution'] = resolution
    optd['cell_parameters'] = cell_parameters

    return optd

def crystal_data(mtz):
    """Set crystallographic parameters from mtz file

    Parameters
    ----------
    mtz : str
       The path to the mtz file

    Returns
    -------
    space_group : str
       The space group
    resolution : str
       The resolution
    cell_parameters : str
       The cell parameters

    """

    hkl_info = clipper.HKL_info()
    mtz_file = clipper.CCP4MTZfile()
    mtz_file.open_read(mtz)
    mtz_file.import_hkl_info(hkl_info)

    sg, cell = hkl_info.spacegroup(), hkl_info.cell()

    space_group = sg.symbol_hm()
    space_group = space_group.replace(" ", "")

    resolution = "%.2lf" % hkl_info.resolution().limit()
    cell_parameters = "%.2lf %.2lf %.2lf %.2lf %.2lf %.2lf" % (cell.a(),
                                                               cell.b(),
                                                               cell.c(),
                                                               cell.alpha_deg(),
                                                               cell.beta_deg(),
                                                               cell.gamma_deg())

    return space_group, resolution, cell_parameters

def del_column(file_name, column, overwrite=True):
    """Delete a column from an mtz file and return a path to the file"""
    mtzDel = simbad_util.filename_append(file_name, "d{0}".format(column) )
    cmd = [ "mtzutils", "hklin1", file_name, "hklout", mtzDel ]
    stdin = "EXCLUDE 1 {0}".format( column )
    logfile = os.path.join( os.getcwd(), "mtzutils.log" )
    retcode = simbad_util.run_command(cmd, stdin=stdin, logfile=logfile)
    if retcode != 0:
        msg = "Error running mtzutils. Check the logfile: {0}".format(logfile)
        logger.critical(msg)
        raise RuntimeError(msg)

    if overwrite:
        shutil.move(mtzDel,file_name)
        return file_name
    else:
        return mtzDel

def add_rfree(file_name,directory=None,overwrite=True):
    """Run uniqueify on mtz file to generate RFREE data column"""
    mtzUnique = simbad_util.filename_append(file_name, "uniqueify", directory=directory)

    cmd = ['uniqueify', file_name, mtzUnique]
    logfile = os.path.join( os.getcwd(), "uniqueify.log" )
    retcode = simbad_util.run_command(cmd, logfile=logfile)
    if retcode != 0:
        msg = "Error running command: {0}. Check the logfile: {1}".format(" ".join(cmd),logfile)
        logger.critical(msg)
        raise RuntimeError(msg)

    if overwrite:
        shutil.move(mtzUnique,file_name)
        return file_name
    else:
        return mtzUnique

def get_labels(file_name):
    """Return the F, FP and FREE column labels"""

    reflection_file = reflection_file_reader.any_reflection_file(file_name=file_name)
    if not reflection_file.file_type()=="ccp4_mtz":
        msg="File is not of type ccp4_mtz: {0}".format(file_name)
        logging.critical(msg)
        raise RuntimeError(msg)

    content=reflection_file.file_content()
    ctypes=content.column_types()
    clabels=content.column_labels()
    ftype='F'
    dtype='D'

    if not ftype in ctypes:
        msg = "Cannot find any structure amplitudes in: {0}".format(file_name)
        raise RuntimeError(msg)
    F=clabels[ctypes.index(ftype)]

    # FP derived from F
    FP='SIG'+F
    if not FP in clabels:
        msg = "Cannot find label {0} in file: {1}".format(FP,file_name)
        raise RuntimeError(msg)

    try:
        if not dtype in ctypes:
            msg = "Cannot find any structure amplitudes in: {0}".format(file_name)
            raise RuntimeError(msg)

        DANO = clabels[ctypes.index(dtype)]

        # SIGDANO derived from DANO
        SIGDANO = 'SIG' + DANO
        if not SIGDANO in clabels:
            msg = "Cannot find label {0} in file: {1}".format(SIGDANO, file_name)
            raise RuntimeError(msg)
    except RuntimeError:
        DANO, SIGDANO = None, None

    FREE=_get_rfree(content)

    return F,FP,DANO,SIGDANO,FREE

def get_rfree(file_name):
    """Return the Rfree label"""

    reflection_file = reflection_file_reader.any_reflection_file(file_name=file_name)
    if not reflection_file.file_type()=="ccp4_mtz":
        msg = "File is not of type ccp4_mtz: {0}".format(file_name)
        logging.critical(msg)
        raise RuntimeError(msg)

    # Read the file
    content=reflection_file.file_content()
    return _get_rfree(content)

def _get_rfree(content):
    rfree_label=None
    #print "GOT ",content.column_labels()
    for label in content.column_labels():
        if 'free' in label.lower():
            column = content.get_column(label=label)
            selection_valid = column.selection_valid()
            flags = column.extract_values()
            sel_0 = (flags == 0)
            # extract number of work/test reflections
            n0=( sel_0 & selection_valid).count(True)
            n1=(~sel_0 & selection_valid).count(True)
            #print "Number of 0 (work):",n0
            #print "Number of 1 (test):",n1
            #print float(n0)/float(n1)*100
            if n0>0 and n1>0:
                if rfree_label:
                    logger.warning("FOUND >1 RFREE label in file!")
                rfree_label=label
    return rfree_label

def to_hkl(mtz_file,hkl_file=None,directory=None,F=None,SIGF=None,FREE=None):

    if directory is None:
        directory=os.getcwd()

    if hkl_file is None:
        name=os.path.splitext(os.path.basename(mtz_file))[0]
        hkl_file=os.path.join(directory,name+".hkl")

    if F is None or SIGF is None or FREE is None:
        F,SIGF,DANO,SIGDANO,FREE=get_labels(mtz_file)

    cmd=['mtz2various','HKLIN',mtz_file,'HKLOUT', hkl_file]
    logfile="mtz2various.log"
    stdin  = """LABIN FP={0} SIGFP={1} FREE={2}
OUTPUT SHELX
FSQUARED
END""".format(F,SIGF,FREE)

    ret = simbad_util.run_command(cmd=cmd, logfile=logfile, directory=None, dolog=False, stdin=stdin)
    if ret == 0:
        os.unlink(logfile)
    else:
        msg = "Error converting {0} to HKL format - see log: {1}".format(mtz_file,logfile)
        raise RuntimeError(msg)

    return hkl_file

def processReflectionFile(amoptd):
    """Make sure we have a valid mtz file. If necessary convert a given cif file.
       Set the mtz variable in the given amoptd to the reflection file to use
       Return True if it all worked or raise an exception if it failed
    """

    # Now have an mtz so check it's valid
    if not amoptd['mtz'] or not os.path.isfile( amoptd['mtz'] ):
        logger.critical("Cannot find MTZ file: %s", amoptd['mtz'])
        sys.exit(1)

    # Get column label info
    reflection_file = reflection_file_reader.any_reflection_file(file_name=amoptd['mtz'])
    if not reflection_file.file_type()=="ccp4_mtz":
        logger.critical("File is not of type ccp4_mtz: %s", amoptd['mtz'])
        sys.exit(1)

    # Read the file
    content=reflection_file.file_content()

    # Check any user-given flags

    for flag in ['F','SIGF','FREE']:
        if amoptd[flag] and amoptd[flag] not in content.column_labels():
            logger.critical("Cannot find flag %s label %s in mtz file %s", flag, amoptd[flag], amoptd['mtz'])
            sys.exit(1)

    # If any of the flags aren't given we set defaults based on what's in the file
    if not amoptd['F']:
        if 'F' not in content.column_types():
            logger.critical("Cannot find column type F for flag F in mtz file: %s", amoptd['mtz'])
            sys.exit(1)
        amoptd['F']  = content.column_labels()[content.column_types().index('F')]
    if not amoptd['SIGF']:
        l='SIG'+amoptd['F']
        if not l in content.column_labels():
            logger.critical("Cannot find column type %s for flag SIGF in mtz file: %s", l, amoptd['mtz'])
            sys.exit(1)
        amoptd['SIGF']  = l

    if amoptd['FREE']:
        # Check is valid
        rfree=_get_rfree(content)
        if not rfree or not rfree==amoptd['FREE']:
            logger.critical("Given RFREE label %s is not valid for mtz file: %s", amoptd['FREE'], amoptd['mtz'])
            sys.exit(1)
    else:
        # See if we can find a valid label in the file
        rfree=_get_rfree(content)
        if not rfree:
            # Need to generate RFREE
            logger.warning("Cannot find a valid FREE flag - running uniquefy to generate column with RFREE data.")
            amoptd['mtz'] = add_rfree( amoptd['mtz'], directory=amoptd['work_dir'],overwrite=False)

            # Check file and get new FREE flag
            rfree=get_rfree(amoptd['mtz'])
            if not rfree:
                logger.critical("Cannot find valid rfree flag in mtz file %s after running uniquiefy", amoptd['mtz'])
                sys.exit(1)
        amoptd['FREE']  = rfree

    # Return anomalous data labels if the columns exist in the file
    if not amoptd['DANO']:
        if 'D' not in content.column_types():
            logger.critical("Cannot find column type D for flag DANO in mtz file: %s", amoptd['mtz'])
        else:
            amoptd['DANO'] = content.column_labels()[content.column_types().index('D')]

    if not amoptd['SIGDANO'] and amoptd['DANO']:
        l = 'SIG' + amoptd['DANO']
        if not l in content.column_labels():
            logger.critical("Cannot find column type %s for flag SIGDANO in mtz file: %s", l, amoptd['mtz'])
        else:
            amoptd['SIGDANO'] = l

    return True

<|endoftext|>


<commit_before>import copy
import datetime
from django.db import models
from django.contrib import admin
from django.contrib.auth.models import User
from django.utils import importlib
from manager import HistoryDescriptor

class HistoricalRecords(object):
    def contribute_to_class(self, cls, name):
        self.manager_name = name
        self.module = cls.__module__
        models.signals.class_prepared.connect(self.finalize, sender=cls)

        def save_without_historical_record(self, *args, **kwargs):
            """Caution! Make sure you know what you're doing before you use this method."""
            self.skip_history_when_saving = True
            ret = self.save(*args, **kwargs)
            del self.skip_history_when_saving
            return ret
        setattr(cls, 'save_without_historical_record', save_without_historical_record)

    def finalize(self, sender, **kwargs):
        history_model = self.create_history_model(sender)
        module = importlib.import_module(self.module)
        setattr(module, history_model.__name__, history_model)

        # The HistoricalRecords object will be discarded,
        # so the signal handlers can't use weak references.
        models.signals.post_save.connect(self.post_save, sender=sender,
                                         weak=False)
        models.signals.post_delete.connect(self.post_delete, sender=sender,
                                           weak=False)

        descriptor = HistoryDescriptor(history_model)
        setattr(sender, self.manager_name, descriptor)

    def create_history_model(self, model):
        """
        Creates a historical model to associate with the model provided.
        """
        attrs = {'__module__': self.module}

        fields = self.copy_fields(model)
        attrs.update(fields)
        attrs.update(self.get_extra_fields(model, fields))
        attrs.update(Meta=type('Meta', (), self.get_meta_options(model)))
        name = 'Historical%s' % model._meta.object_name
        return type(name, (models.Model,), attrs)

    def copy_fields(self, model):
        """
        Creates copies of the model's original fields, returning
        a dictionary mapping field name to copied field object.
        """
        fields = {}

        for field in model._meta.fields:
            field = copy.copy(field)
            fk = None

            if isinstance(field, models.AutoField):
                # The historical model gets its own AutoField, so any
                # existing one must be replaced with an IntegerField.
                field.__class__ = models.IntegerField

            if isinstance(field, models.ForeignKey):
                field.__class__ = models.IntegerField
                #ughhhh. open to suggestions here
                try:
                    field.rel = None
                except:
                    pass
                try:
                    field.related = None
                except:
                    pass
                try:
                    field.related_query_name = None
                except:
                    pass
                field.null = True
                field.blank = True
                fk = True
            else:
                fk = False

            # The historical instance should not change creation/modification timestamps.
            field.auto_now = False
            field.auto_now_add = False

            if field.primary_key or field.unique:
                # Unique fields can no longer be guaranteed unique,
                # but they should still be indexed for faster lookups.
                field.primary_key = False
                field._unique = False
                field.db_index = True
            if fk:
                fields[field.name+"_id"] = field
            else:
                fields[field.name] = field

        return fields

    def get_extra_fields(self, model, fields):
        """
        Returns a dictionary of fields that will be added to the historical
        record model, in addition to the ones returned by copy_fields below.
        """
        @models.permalink
        def revert_url(self):
            opts = model._meta
            return ('%s:%s_%s_simple_history' %
                    (admin.site.name, opts.app_label, opts.module_name),
                    [getattr(self, opts.pk.attname), self.history_id])
        def get_instance(self):
            return model(**dict([(k, getattr(self, k)) for k in fields]))

        rel_nm = '_%s_history' % model._meta.object_name.lower()
        return {
            'history_id': models.AutoField(primary_key=True),
            'history_date': models.DateTimeField(default=datetime.datetime.now),
            'history_type': models.CharField(max_length=1, choices=(
                ('+', 'Created'),
                ('~', 'Changed'),
                ('-', 'Deleted'),
            )),
            'history_object': HistoricalObjectDescriptor(model),
            'changed_by': models.ForeignKey(User, null=True),
            'instance': property(get_instance),
            'revert_url': revert_url,
            '__unicode__': lambda self: u'%s as of %s' % (self.history_object,
                                                          self.history_date)
        }

    def get_meta_options(self, model):
        """
        Returns a dictionary of fields that will be added to
        the Meta inner class of the historical record model.
        """
        return {
            'ordering': ('-history_date', '-history_id'),
        }

    def post_save(self, instance, created, **kwargs):
        if not created and hasattr(instance, 'skip_history_when_saving'):
            return
        self.create_historical_record(instance, created and '+' or '~')

    def post_delete(self, instance, **kwargs):
        self.create_historical_record(instance, '-')

    def create_historical_record(self, instance, type):
        changed_by = getattr(instance, '_changed_by_user', None)
        manager = getattr(instance, self.manager_name)
        attrs = {}
        for field in instance._meta.fields:
            attrs[field.attname] = getattr(instance, field.attname)
        manager.create(history_type=type, changed_by=changed_by, **attrs)

class HistoricalObjectDescriptor(object):
    def __init__(self, model):
        self.model = model

    def __get__(self, instance, owner):
        values = (getattr(instance, f.attname) for f in self.model._meta.fields)
        return self.model(*values)
<commit_msg>Fixing for compatibility with django 1.4.<commit_after>import copy
import datetime
from django.db import models
from django.contrib import admin
from django.contrib.auth.models import User
from django.utils import importlib
from manager import HistoryDescriptor

class HistoricalRecords(object):
    def contribute_to_class(self, cls, name):
        self.manager_name = name
        self.module = cls.__module__
        models.signals.class_prepared.connect(self.finalize, sender=cls)

        def save_without_historical_record(self, *args, **kwargs):
            """Caution! Make sure you know what you're doing before you use this method."""
            self.skip_history_when_saving = True
            ret = self.save(*args, **kwargs)
            del self.skip_history_when_saving
            return ret
        setattr(cls, 'save_without_historical_record', save_without_historical_record)

    def finalize(self, sender, **kwargs):
        history_model = self.create_history_model(sender)
        module = importlib.import_module(self.module)
        setattr(module, history_model.__name__, history_model)

        # The HistoricalRecords object will be discarded,
        # so the signal handlers can't use weak references.
        models.signals.post_save.connect(self.post_save, sender=sender,
                                         weak=False)
        models.signals.post_delete.connect(self.post_delete, sender=sender,
                                           weak=False)

        descriptor = HistoryDescriptor(history_model)
        setattr(sender, self.manager_name, descriptor)

    def create_history_model(self, model):
        """
        Creates a historical model to associate with the model provided.
        """
        attrs = {'__module__': self.module}

        fields = self.copy_fields(model)
        attrs.update(fields)
        attrs.update(self.get_extra_fields(model, fields))
        attrs.update(Meta=type('Meta', (), self.get_meta_options(model)))
        name = 'Historical%s' % model._meta.object_name
        return type(name, (models.Model,), attrs)

    def copy_fields(self, model):
        """
        Creates copies of the model's original fields, returning
        a dictionary mapping field name to copied field object.
        """
        fields = {}

        for field in model._meta.fields:
            field = copy.copy(field)
            fk = None

            if isinstance(field, models.AutoField):
                # The historical model gets its own AutoField, so any
                # existing one must be replaced with an IntegerField.
                field.__class__ = models.IntegerField

            if isinstance(field, models.ForeignKey):
                field.__class__ = models.IntegerField
                #ughhhh. open to suggestions here
                try:
                    field.rel = None
                except:
                    pass
                try:
                    field.related = None
                except:
                    pass
                try:
                    field.related_query_name = None
                except:
                    pass
                field.null = True
                field.blank = True
                fk = True
            else:
                fk = False

            # The historical instance should not change creation/modification timestamps.
            field.auto_now = False
            field.auto_now_add = False

            if field.primary_key or field.unique:
                # Unique fields can no longer be guaranteed unique,
                # but they should still be indexed for faster lookups.
                field.primary_key = False
                field._unique = False
                field.db_index = True
            if fk:
                field.name = field.name + "_id"
            fields[field.name] = field

        return fields

    def get_extra_fields(self, model, fields):
        """
        Returns a dictionary of fields that will be added to the historical
        record model, in addition to the ones returned by copy_fields below.
        """
        @models.permalink
        def revert_url(self):
            opts = model._meta
            return ('%s:%s_%s_simple_history' %
                    (admin.site.name, opts.app_label, opts.module_name),
                    [getattr(self, opts.pk.attname), self.history_id])
        def get_instance(self):
            return model(**dict([(k, getattr(self, k)) for k in fields]))

        rel_nm = '_%s_history' % model._meta.object_name.lower()
        return {
            'history_id': models.AutoField(primary_key=True),
            'history_date': models.DateTimeField(default=datetime.datetime.now),
            'history_type': models.CharField(max_length=1, choices=(
                ('+', 'Created'),
                ('~', 'Changed'),
                ('-', 'Deleted'),
            )),
            'history_object': HistoricalObjectDescriptor(model),
            'changed_by': models.ForeignKey(User, null=True),
            'instance': property(get_instance),
            'revert_url': revert_url,
            '__unicode__': lambda self: u'%s as of %s' % (self.history_object,
                                                          self.history_date)
        }

    def get_meta_options(self, model):
        """
        Returns a dictionary of fields that will be added to
        the Meta inner class of the historical record model.
        """
        return {
            'ordering': ('-history_date', '-history_id'),
        }

    def post_save(self, instance, created, **kwargs):
        if not created and hasattr(instance, 'skip_history_when_saving'):
            return
        self.create_historical_record(instance, created and '+' or '~')

    def post_delete(self, instance, **kwargs):
        self.create_historical_record(instance, '-')

    def create_historical_record(self, instance, type):
        changed_by = getattr(instance, '_changed_by_user', None)
        manager = getattr(instance, self.manager_name)
        attrs = {}
        for field in instance._meta.fields:
            attrs[field.attname] = getattr(instance, field.attname)
        manager.create(history_type=type, changed_by=changed_by, **attrs)

class HistoricalObjectDescriptor(object):
    def __init__(self, model):
        self.model = model

    def __get__(self, instance, owner):
        values = (getattr(instance, f.attname) for f in self.model._meta.fields)
        return self.model(*values)
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time
import wizard
import ir
import pooler
from osv.osv import except_osv
import netsvc
from tools.translate import _

invoice_form = """<?xml version="1.0"?>
<form string="Create invoices">
    <separator colspan="4" string="Create invoices" />
    <field name="journal_id"/>
    <newline/>
    <field name="group"/>
    <newline/>
    <field name="type"/>
</form>
"""

invoice_fields = {
    'journal_id': {
        'string': 'Destination Journal',
        'type': 'many2one',
        'relation': 'account.journal',
        'required': True
    },
    'group': {
        'string': 'Group by partner',
        'type':'boolean'
    },
    'type': {
        'string': 'Type',
        'type': 'selection',
        'selection': [
            ('out_invoice', 'Customer Invoice'),
            ('in_invoice', 'Supplier Invoice'),
            ('out_refund', 'Customer Refund'),
            ('in_refund', 'Supplier Refund'),
            ],
        'required': True
    },
}

def _get_type(obj, cr, uid, data, context):
    picking_obj=pooler.get_pool(cr.dbname).get('stock.picking')
    usage = 'customer'
    pick = picking_obj.browse(cr, uid, data['id'], context)
    if pick.invoice_state=='invoiced':
        raise wizard.except_wizard(_('UserError'),_('Invoice is already created.'))
    if pick.invoice_state=='none':
        raise wizard.except_wizard(_('UserError'),_('Invoice cannot be created from Packing.'))

    if pick.move_lines:
        usage=pick.move_lines[0].location_id.usage

    if pick.type == 'out' and usage == 'supplier':
        type = 'in_refund'
    elif pick.type == 'out' and usage == 'customer':
        type = 'out_invoice'
    elif pick.type == 'in' and usage == 'supplier':
        type = 'in_invoice'
    elif pick.type == 'in' and usage == 'customer':
        type = 'out_refund'
    else:
        type = 'out_invoice'
    return {'type': type}

def _create_invoice(obj, cr, uid, data, context):
    if data['form'].get('new_picking',False):
        data['id'] = data['form']['new_picking']
        data['ids'] = [data['form']['new_picking']]
    pool = pooler.get_pool(cr.dbname)
    picking_obj = pooler.get_pool(cr.dbname).get('stock.picking')
    mod_obj = pool.get('ir.model.data')
    act_obj = pool.get('ir.actions.act_window')

    type = data['form']['type']

    res = picking_obj.action_invoice_create(cr, uid, data['ids'],
            journal_id=data['form']['journal_id'],group=data['form']['group'],
            type=type, context= context)

    invoice_ids = res.values()
    if not invoice_ids:
        raise  wizard.except_wizard(_('Error'),_('Invoice is not created'))

    if type == 'out_invoice':
        xml_id = 'action_invoice_tree5'
    elif type == 'in_invoice':
        xml_id = 'action_invoice_tree8'
    elif type == 'out_refund':
        xml_id = 'action_invoice_tree10'
    else:
        xml_id = 'action_invoice_tree12'

    result = mod_obj._get_id(cr, uid, 'account', xml_id)
    id = mod_obj.read(cr, uid, result, ['res_id'])
    result = act_obj.read(cr, uid, id['res_id'])
    result['res_id'] = invoice_ids
    return result

class make_invoice_onshipping(wizard.interface):
    states = {
        'init': {
            'actions': [_get_type],
            'result': {
                'type': 'form',
                'arch': invoice_form,
                'fields': invoice_fields,
                'state': [
                    ('end', 'Cancel'),
                    ('create_invoice', 'Create invoice')
                ]
            }
        },
        'create_invoice': {
            'actions': [],
            'result': {
                'type': 'action',
                'action': _create_invoice,
                'state': 'end'
            }
        },
    }

make_invoice_onshipping("stock.invoice_onshipping")

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<commit_msg>[IMP] stock: wizard_invoice_onshipping.py: pep8<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import wizard
import pooler
from tools.translate import _

invoice_form = """<?xml version="1.0"?>
<form string="Create invoices">
    <separator colspan="4" string="Create invoices" />
    <field name="journal_id"/>
    <newline/>
    <field name="group"/>
    <newline/>
    <field name="type"/>
</form>
"""

invoice_fields = {
    'journal_id': {
        'string': 'Destination Journal',
        'type': 'many2one',
        'relation': 'account.journal',
        'required': True
    },
    'group': {
        'string': 'Group by partner',
        'type': 'boolean'
    },
    'type': {
        'string': 'Type',
        'type': 'selection',
        'selection': [
            ('out_invoice', 'Customer Invoice'),
            ('in_invoice', 'Supplier Invoice'),
            ('out_refund', 'Customer Refund'),
            ('in_refund', 'Supplier Refund'),
            ],
        'required': True
    },
}

def _get_type(obj, cr, uid, data, context):
    picking_obj = pooler.get_pool(cr.dbname).get('stock.picking')
    usage = 'customer'
    pick = picking_obj.browse(cr, uid, data['id'], context)
    if pick.invoice_state == 'invoiced':
        raise wizard.except_wizard(_('UserError'), _('Invoice is already created.'))
    if pick.invoice_state == 'none':
        raise wizard.except_wizard(_('UserError'), _('Invoice cannot be created from Packing.'))

    if pick.move_lines:
        usage = pick.move_lines[0].location_id.usage

    if pick.type == 'out' and usage == 'supplier':
        type = 'in_refund'
    elif pick.type == 'out' and usage == 'customer':
        type = 'out_invoice'
    elif pick.type == 'in' and usage == 'supplier':
        type = 'in_invoice'
    elif pick.type == 'in' and usage == 'customer':
        type = 'out_refund'
    else:
        type = 'out_invoice'
    return {'type': type}

def _create_invoice(obj, cr, uid, data, context):
    if data['form'].get('new_picking', False):
        data['id'] = data['form']['new_picking']
        data['ids'] = [data['form']['new_picking']]
    pool = pooler.get_pool(cr.dbname)
    picking_obj = pooler.get_pool(cr.dbname).get('stock.picking')
    mod_obj = pool.get('ir.model.data')
    act_obj = pool.get('ir.actions.act_window')

    type = data['form']['type']

    res = picking_obj.action_invoice_create(cr, uid, data['ids'],
            journal_id = data['form']['journal_id'], group=data['form']['group'],
            type=type, context= context)

    invoice_ids = res.values()
    if not invoice_ids:
        raise  wizard.except_wizard(_('Error'), _('Invoice is not created'))

    if type == 'out_invoice':
        xml_id = 'action_invoice_tree5'
    elif type == 'in_invoice':
        xml_id = 'action_invoice_tree8'
    elif type == 'out_refund':
        xml_id = 'action_invoice_tree10'
    else:
        xml_id = 'action_invoice_tree12'

    result = mod_obj._get_id(cr, uid, 'account', xml_id)
    id = mod_obj.read(cr, uid, result, ['res_id'])
    result = act_obj.read(cr, uid, id['res_id'])
    result['res_id'] = invoice_ids
    return result

class make_invoice_onshipping(wizard.interface):
    states = {
        'init': {
            'actions': [_get_type],
            'result': {
                'type': 'form',
                'arch': invoice_form,
                'fields': invoice_fields,
                'state': [
                    ('end', 'Cancel'),
                    ('create_invoice', 'Create invoice')
                ]
            }
        },
        'create_invoice': {
            'actions': [],
            'result': {
                'type': 'action',
                'action': _create_invoice,
                'state': 'end'
            }
        },
    }

make_invoice_onshipping("stock.invoice_onshipping")

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<|endoftext|>


<commit_before>from .utils.common_utils import json_file_to_dict, valid_timeout

class CONFIG:
    config = json_file_to_dict('./config.json')
    HTTP_TIMEOUT = config['HTTP_TIMEOUT'] if isinstance(config, dict) and 'HTTP_TIMEOUT' in config and valid_timeout(config['HTTP_TIMEOUT']) else 60
    TCP_TIMEOUT = config['TCP_TIMEOUT'] if isinstance(config, dict) and 'TCP_TIMEOUT' in config and valid_timeout(config['TCP_TIMEOUT']) else 60
    HTTP_KEEP_ALIVE_TIMEOUT = config['HTTP_KEEP_ALIVE_TIMEOUT'] if isinstance(config, dict) and 'HTTP_KEEP_ALIVE_TIMEOUT' in config and valid_timeout(config['HTTP_KEEP_ALIVE_TIMEOUT']) else 15
    INTERNAL_HTTP_PREFIX = config['INTERNAL_HTTP_PREFIX'] if isinstance(config, dict) and 'INTERNAL_HTTP_PREFIX' in config  else ''<commit_msg>default internal point<commit_after>from .utils.common_utils import json_file_to_dict, valid_timeout

class CONFIG:
    config = json_file_to_dict('./config.json')
    HTTP_TIMEOUT = config['HTTP_TIMEOUT'] if isinstance(config, dict) and 'HTTP_TIMEOUT' in config and valid_timeout(config['HTTP_TIMEOUT']) else 60
    TCP_TIMEOUT = config['TCP_TIMEOUT'] if isinstance(config, dict) and 'TCP_TIMEOUT' in config and valid_timeout(config['TCP_TIMEOUT']) else 60
    HTTP_KEEP_ALIVE_TIMEOUT = config['HTTP_KEEP_ALIVE_TIMEOUT'] if isinstance(config, dict) and 'HTTP_KEEP_ALIVE_TIMEOUT' in config and valid_timeout(config['HTTP_KEEP_ALIVE_TIMEOUT']) else 15
    INTERNAL_HTTP_PREFIX = config['INTERNAL_HTTP_PREFIX'] if isinstance(config, dict) and 'INTERNAL_HTTP_PREFIX' in config  else '/__is_internal__'<|endoftext|>


<commit_before># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import, unicode_literals

import re
import warnings

import pathvalidate as pv
import typepy
from six.moves import map

from ._func import validate_table_name
from .error import SqlSyntaxError

class SqlQuery(object):
    """
    Support class for making SQLite query.
    """

    __RE_SANITIZE = re.compile(
        "[{:s}]".format(re.escape("%/()[]<>.:;'\"!\# -+=\n\r")))
    __RE_SANITIZE_ATTR = re.compile("[{:s}]".format(re.escape("'\"")))
    __RE_TABLE_STR = re.compile("[{:s}]".format(re.escape("%()-+/.")))
    __RE_TO_ATTR_QUOTE = re.compile("[{:s}]".format(re.escape("[_]")))
    __RE_TO_ATTR_BRACKET = re.compile(
        "[{:s}0-9\s#]".format(re.escape("%()-+/.'\"")))
    __RE_SPACE = re.compile("[\s]+")

    __VALID_WHERE_OPERATION_LIST = [
        "=", "==", "!=", "<>", ">", ">=", "<", "<=",
    ]

    @classmethod
    def sanitize(cls, query_item):
        """
        Replace invalid characters in a SQLite query with empty characters.

        :param str query_item: String to be sanitized.
        :return:
            String that exclude invalid chars.
            Invalid operators are as follows:
            ``"%"``, ``"/"``, ``"("``, ``")"``, ``"["``, ``"]"``,
            ``"<"``, ``">"``, ``"."``, ``":"``, ``";"``, ``"'"``,
            ``"!"``, ``"\"``, ``"#"``, ``"-"``, ``"+"``, ``"="``,
            ``"\\n"``, ``"\\r"``
        :rtype: str

        :Example:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.sanitize("k<e:y")
            'key'
        """

        return cls.__RE_SANITIZE.sub("", query_item)

    @classmethod
    def sanitize_attr(cls, query_item):
        return cls.__RE_SANITIZE_ATTR.sub("_", query_item)

    @classmethod
    def to_table_str(cls, name):
        warnings.warn(
            "to_table_str will be deleted in the future, "
            "use to_table_name instead.",
            DeprecationWarning)

        return cls.to_table_name(name)

    @classmethod
    def to_table_name(cls, name):
        """
        :param str name: Table name.
        :return: String that suitable for table name of a SQLite query.
        :rtype: str

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.to_table_name("length")
            'length'
            >>> SqlQuery.to_table_name("length(cm)")
            '[length(cm)]'
            >>> SqlQuery.to_table_name("string length")
            "'string length'"
        """

        if cls.__RE_TABLE_STR.search(name):
            return "[{:s}]".format(name)

        if cls.__RE_SPACE.search(name):
            return "'{:s}'".format(name)

        return name

    @classmethod
    def to_attr_str(cls, name, operation_query=""):
        """
        :param str name: Attribute name.
        :param str operation_query:
            Used as a SQLite function if the value is not empty.
        :return: String that suitable for attribute name of a SQLite query.
        :rtype: str

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.to_attr_str("key")
            'key'
            >>> SqlQuery.to_attr_str("a+b")
            '[a+b]'
            >>> SqlQuery.to_attr_str("key", operation_query="SUM")
            'SUM(key)'
        """

        name = cls.sanitize_attr(name)

        is_quote = cls.__RE_TO_ATTR_QUOTE.search(name) is not None

        try:
            pv.validate_sqlite_attr_name(name)
        except pv.InvalidReservedNameError:
            is_quote = True
        except (pv.ValidReservedNameError, pv.NullNameError, pv.InvalidCharError):
            pass

        if is_quote:
            sql_name = '"{:s}"'.format(name)
        elif cls.__RE_TO_ATTR_BRACKET.search(name):
            sql_name = "[{:s}]".format(name)
        elif name == "join":
            sql_name = "[{:s}]".format(name)
        else:
            sql_name = name

        if typepy.is_not_null_string(operation_query):
            sql_name = "{:s}({:s})".format(operation_query, sql_name)

        return sql_name

    @classmethod
    def to_attr_str_list(cls, name_list, operation_query=""):
        """
        :param list/tuple name_list: List of attribute names.
        :param str operation_query:
            Used as a SQLite function if the value is not empty.
        :return:
            List of strings that suitable for
            attribute names of a SQLite query.
        :rtype: list/itertools.imap

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> list(SqlQuery.to_attr_str_list(["key", "a+b"]))
            ['key', '[a+b]']
            >>> SqlQuery.to_attr_str_list(["key", "a+b"], operation_query="AVG")
            ['AVG(key)', 'AVG([a+b])']

        .. seealso::
            :py:meth:`.to_attr_str`
        """

        if typepy.is_null_string(operation_query):
            return list(map(cls.to_attr_str, name_list))

        return [
            "{:s}({:s})".format(operation_query, cls.to_attr_str(name))
            for name in name_list
        ]

    @classmethod
    def to_value_str(cls, value):
        """
        :param str value: Value associated with a key.
        :return:
            String that suitable for a value of a key.
            Return ``"NULL"`` if the value is |None|.
        :rtype: str

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.to_value_str(1.2)
            '1.2'
            >>> SqlQuery.to_value_str("value")
            "'value'"
            >>> SqlQuery.to_value_str(None)
            'NULL'
        """

        if value is None:
            return "NULL"

        if any([
                typepy.type.Integer(value).is_type(),
                typepy.type.RealNumber(value).is_type()
        ]):
            return str(value)

        try:
            if value.find("'") != -1:
                return '"{}"'.format(value)
        except (TypeError, AttributeError):
            pass

        return "'{}'".format(value)

    @classmethod
    def to_value_str_list(cls, value_list):
        """
        :param list value_list: List of values associated with a key.
        :return:
            List of values that executed ``to_value_str`` method for each item.
        :rtype: itertools.imap

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> list(SqlQuery.to_value_str_list([1, "a", None]))
            ['1', "'a'", 'NULL']

        .. seealso::
            :py:meth:`.to_value_str`
        """

        return map(cls.to_value_str, value_list)

    @classmethod
    def make_select(cls, select, table, where=None, extra=None):
        """
        Make SELECT query.

        :param str select: Attribute for SELECT query.
        :param str table: Table name of executing the query.
        :param str where:
            Add a WHERE clause to execute query,
            if the value is not |None|.
        :param extra extra:
            Add additional clause to execute query,
            if the value is not |None|.
        :return: Query of SQLite.
        :rtype: str
        :raises ValueError: ``select`` is empty string.
        :raises simplesqlite.InvalidTableNameError:
            |raises_validate_table_name|

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.make_select(select="value", table="example")
            'SELECT value FROM example'
            >>> SqlQuery.make_select(select="value", table="example", where=SqlQuery.make_where("key", 1))
            'SELECT value FROM example WHERE key = 1'
            >>> SqlQuery.make_select(select="value", table="example", where=SqlQuery.make_where("key", 1), extra="ORDER BY value")
            'SELECT value FROM example WHERE key = 1 ORDER BY value'
        """

        validate_table_name(table)
        if typepy.is_null_string(select):
            raise ValueError("SELECT query is null")

        query_list = [
            "SELECT {:s}".format(select),
            "FROM {:s}".format(cls.to_table_name(table)),
        ]
        if typepy.is_not_null_string(where):
            query_list.append("WHERE {:s}".format(where))
        if typepy.is_not_null_string(extra):
            query_list.append(extra)

        return " ".join(query_list)

    @classmethod
    def make_insert(cls, table, insert_tuple):
        """
        Make INSERT query.

        :param str table: Table name of executing the query.
        :param list/tuple insert_tuple: Insertion data.
        :return: Query of SQLite.
        :rtype: str
        :raises ValueError: If ``insert_tuple`` is empty |list|/|tuple|.
        :raises simplesqlite.InvalidTableNameError:
            |raises_validate_table_name|
        """

        validate_table_name(table)

        table = cls.to_table_name(table)

        if typepy.is_empty_sequence(insert_tuple):
            raise ValueError("empty insert list/tuple")

        return "INSERT INTO {:s} VALUES ({:s})".format(
            table, ",".join(['?' for _i in insert_tuple]))

    @classmethod
    def make_update(cls, table, set_query, where=None):
        """
        Make UPDATE query.

        :param str table: Table name of executing the query.
        :param str set_query: SET part of the UPDATE query.
        :param str where:
            Add a WHERE clause to execute query,
            if the value is not |None|.
        :return: Query of SQLite.
        :rtype: str
        :raises ValueError: If ``set_query`` is empty string.
        :raises simplesqlite.InvalidTableNameError:
            |raises_validate_table_name|
        """

        validate_table_name(table)
        if typepy.is_null_string(set_query):
            raise ValueError("SET query is null")

        query_list = [
            "UPDATE {:s}".format(cls.to_table_name(table)),
            "SET {:s}".format(set_query),
        ]
        if typepy.is_not_null_string(where):
            query_list.append("WHERE {:s}".format(where))

        return " ".join(query_list)

    @classmethod
    def make_where(cls, key, value, operation="="):
        """
        Make part of WHERE query.

        :param str key: Attribute name of the key.
        :param str value: Value of the right hand side associated with the key.
        :param str operation: Operator of WHERE query.
        :return: Part of WHERE query of SQLite.
        :rtype: str
        :raises simplesqlite.SqlSyntaxError:
            If **a)** ``operation`` is invalid operator.
            Valid operators are as follows:
            ``"="``, ``"=="``, ``"!="``, ``"<>"``,
            ``">"``, ``">="``, ``"<"``, ``"<="``.
            **b)** the ``value`` is |None| and
            the ``operation`` is not ``"="``/``"!="``.

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.make_where("key", "hoge")
            "key = 'hoge'"
            >>> SqlQuery.make_where("value", 1, operation=">")
            'value > 1'
        """

        if operation not in cls.__VALID_WHERE_OPERATION_LIST:
            raise SqlSyntaxError("operation not supported: " + str(operation))

        if value is not None:
            return "{:s} {:s} {:s}".format(
                cls.to_attr_str(key), operation, cls.to_value_str(value))

        if operation == "=":
            return "{:s} IS NULL".format(cls.to_attr_str(key))
        elif operation == "!=":
            return "{:s} IS NOT NULL".format(cls.to_attr_str(key))

        raise SqlSyntaxError(
            "Invalid operation ({:s}) with None right-hand side".format(
                operation))

    @classmethod
    def make_where_in(cls, key, value_list):
        """
        Make part of WHERE IN query.

        :param str key: Attribute name of the key.
        :param str value_list:
            List of values that the right hand side associated with the key.
        :return: Part of WHERE query of SQLite.
        :rtype: str

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.make_where_in("key", ["hoge", "foo", "bar"])
            "key IN ('hoge', 'foo', 'bar')"
        """

        return "{:s} IN ({:s})".format(
            cls.to_attr_str(key), ", ".join(cls.to_value_str_list(value_list)))

    @classmethod
    def make_where_not_in(cls, key, value_list):
        """
        Make part of WHERE NOT IN query.

        :param str key: Attribute name of the key.
        :param str value_list:
            List of values that the right hand side associated with the key.
        :return: Part of WHERE query of SQLite.
        :rtype: str

        :Example:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.make_where_not_in("key", ["hoge", "foo", "bar"])
            "key NOT IN ('hoge', 'foo', 'bar')"
        """

        return "{:s} NOT IN ({:s})".format(
            cls.to_attr_str(key), ", ".join(cls.to_value_str_list(value_list)))
<commit_msg>Simplify condition<commit_after># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import, unicode_literals

import re
import warnings

import pathvalidate as pv
import typepy
from six.moves import map

from ._func import validate_table_name
from .error import SqlSyntaxError

class SqlQuery(object):
    """
    Support class for making SQLite query.
    """

    __RE_SANITIZE = re.compile(
        "[{:s}]".format(re.escape("%/()[]<>.:;'\"!\# -+=\n\r")))
    __RE_SANITIZE_ATTR = re.compile("[{:s}]".format(re.escape("'\"")))
    __RE_TABLE_STR = re.compile("[{:s}]".format(re.escape("%()-+/.")))
    __RE_TO_ATTR_QUOTE = re.compile("[{:s}]".format(re.escape("[_]")))
    __RE_TO_ATTR_BRACKET = re.compile(
        "[{:s}0-9\s#]".format(re.escape("%()-+/.'\"")))
    __RE_SPACE = re.compile("[\s]+")

    __VALID_WHERE_OPERATION_LIST = [
        "=", "==", "!=", "<>", ">", ">=", "<", "<=",
    ]

    @classmethod
    def sanitize(cls, query_item):
        """
        Replace invalid characters in a SQLite query with empty characters.

        :param str query_item: String to be sanitized.
        :return:
            String that exclude invalid chars.
            Invalid operators are as follows:
            ``"%"``, ``"/"``, ``"("``, ``")"``, ``"["``, ``"]"``,
            ``"<"``, ``">"``, ``"."``, ``":"``, ``";"``, ``"'"``,
            ``"!"``, ``"\"``, ``"#"``, ``"-"``, ``"+"``, ``"="``,
            ``"\\n"``, ``"\\r"``
        :rtype: str

        :Example:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.sanitize("k<e:y")
            'key'
        """

        return cls.__RE_SANITIZE.sub("", query_item)

    @classmethod
    def sanitize_attr(cls, query_item):
        return cls.__RE_SANITIZE_ATTR.sub("_", query_item)

    @classmethod
    def to_table_str(cls, name):
        warnings.warn(
            "to_table_str will be deleted in the future, "
            "use to_table_name instead.",
            DeprecationWarning)

        return cls.to_table_name(name)

    @classmethod
    def to_table_name(cls, name):
        """
        :param str name: Table name.
        :return: String that suitable for table name of a SQLite query.
        :rtype: str

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.to_table_name("length")
            'length'
            >>> SqlQuery.to_table_name("length(cm)")
            '[length(cm)]'
            >>> SqlQuery.to_table_name("string length")
            "'string length'"
        """

        if cls.__RE_TABLE_STR.search(name):
            return "[{:s}]".format(name)

        if cls.__RE_SPACE.search(name):
            return "'{:s}'".format(name)

        return name

    @classmethod
    def to_attr_str(cls, name, operation_query=""):
        """
        :param str name: Attribute name.
        :param str operation_query:
            Used as a SQLite function if the value is not empty.
        :return: String that suitable for attribute name of a SQLite query.
        :rtype: str

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.to_attr_str("key")
            'key'
            >>> SqlQuery.to_attr_str("a+b")
            '[a+b]'
            >>> SqlQuery.to_attr_str("key", operation_query="SUM")
            'SUM(key)'
        """

        name = cls.sanitize_attr(name)

        is_quote = cls.__RE_TO_ATTR_QUOTE.search(name) is not None

        try:
            pv.validate_sqlite_attr_name(name)
        except pv.InvalidReservedNameError:
            is_quote = True
        except (pv.ValidReservedNameError, pv.NullNameError, pv.InvalidCharError):
            pass

        if is_quote:
            sql_name = '"{:s}"'.format(name)
        elif cls.__RE_TO_ATTR_BRACKET.search(name):
            sql_name = "[{:s}]".format(name)
        elif name == "join":
            sql_name = "[{:s}]".format(name)
        else:
            sql_name = name

        if typepy.is_not_null_string(operation_query):
            sql_name = "{:s}({:s})".format(operation_query, sql_name)

        return sql_name

    @classmethod
    def to_attr_str_list(cls, name_list, operation_query=""):
        """
        :param list/tuple name_list: List of attribute names.
        :param str operation_query:
            Used as a SQLite function if the value is not empty.
        :return:
            List of strings that suitable for
            attribute names of a SQLite query.
        :rtype: list/itertools.imap

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> list(SqlQuery.to_attr_str_list(["key", "a+b"]))
            ['key', '[a+b]']
            >>> SqlQuery.to_attr_str_list(["key", "a+b"], operation_query="AVG")
            ['AVG(key)', 'AVG([a+b])']

        .. seealso::
            :py:meth:`.to_attr_str`
        """

        if typepy.is_null_string(operation_query):
            return list(map(cls.to_attr_str, name_list))

        return [
            "{:s}({:s})".format(operation_query, cls.to_attr_str(name))
            for name in name_list
        ]

    @classmethod
    def to_value_str(cls, value):
        """
        :param str value: Value associated with a key.
        :return:
            String that suitable for a value of a key.
            Return ``"NULL"`` if the value is |None|.
        :rtype: str

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.to_value_str(1.2)
            '1.2'
            >>> SqlQuery.to_value_str("value")
            "'value'"
            >>> SqlQuery.to_value_str(None)
            'NULL'
        """

        if value is None:
            return "NULL"

        if typepy.Integer(value).is_type() or typepy.RealNumber(value).is_type():
            return str(value)

        try:
            if value.find("'") != -1:
                return '"{}"'.format(value)
        except (TypeError, AttributeError):
            pass

        return "'{}'".format(value)

    @classmethod
    def to_value_str_list(cls, value_list):
        """
        :param list value_list: List of values associated with a key.
        :return:
            List of values that executed ``to_value_str`` method for each item.
        :rtype: itertools.imap

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> list(SqlQuery.to_value_str_list([1, "a", None]))
            ['1', "'a'", 'NULL']

        .. seealso::
            :py:meth:`.to_value_str`
        """

        return map(cls.to_value_str, value_list)

    @classmethod
    def make_select(cls, select, table, where=None, extra=None):
        """
        Make SELECT query.

        :param str select: Attribute for SELECT query.
        :param str table: Table name of executing the query.
        :param str where:
            Add a WHERE clause to execute query,
            if the value is not |None|.
        :param extra extra:
            Add additional clause to execute query,
            if the value is not |None|.
        :return: Query of SQLite.
        :rtype: str
        :raises ValueError: ``select`` is empty string.
        :raises simplesqlite.InvalidTableNameError:
            |raises_validate_table_name|

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.make_select(select="value", table="example")
            'SELECT value FROM example'
            >>> SqlQuery.make_select(select="value", table="example", where=SqlQuery.make_where("key", 1))
            'SELECT value FROM example WHERE key = 1'
            >>> SqlQuery.make_select(select="value", table="example", where=SqlQuery.make_where("key", 1), extra="ORDER BY value")
            'SELECT value FROM example WHERE key = 1 ORDER BY value'
        """

        validate_table_name(table)
        if typepy.is_null_string(select):
            raise ValueError("SELECT query is null")

        query_list = [
            "SELECT {:s}".format(select),
            "FROM {:s}".format(cls.to_table_name(table)),
        ]
        if typepy.is_not_null_string(where):
            query_list.append("WHERE {:s}".format(where))
        if typepy.is_not_null_string(extra):
            query_list.append(extra)

        return " ".join(query_list)

    @classmethod
    def make_insert(cls, table, insert_tuple):
        """
        Make INSERT query.

        :param str table: Table name of executing the query.
        :param list/tuple insert_tuple: Insertion data.
        :return: Query of SQLite.
        :rtype: str
        :raises ValueError: If ``insert_tuple`` is empty |list|/|tuple|.
        :raises simplesqlite.InvalidTableNameError:
            |raises_validate_table_name|
        """

        validate_table_name(table)

        table = cls.to_table_name(table)

        if typepy.is_empty_sequence(insert_tuple):
            raise ValueError("empty insert list/tuple")

        return "INSERT INTO {:s} VALUES ({:s})".format(
            table, ",".join(['?' for _i in insert_tuple]))

    @classmethod
    def make_update(cls, table, set_query, where=None):
        """
        Make UPDATE query.

        :param str table: Table name of executing the query.
        :param str set_query: SET part of the UPDATE query.
        :param str where:
            Add a WHERE clause to execute query,
            if the value is not |None|.
        :return: Query of SQLite.
        :rtype: str
        :raises ValueError: If ``set_query`` is empty string.
        :raises simplesqlite.InvalidTableNameError:
            |raises_validate_table_name|
        """

        validate_table_name(table)
        if typepy.is_null_string(set_query):
            raise ValueError("SET query is null")

        query_list = [
            "UPDATE {:s}".format(cls.to_table_name(table)),
            "SET {:s}".format(set_query),
        ]
        if typepy.is_not_null_string(where):
            query_list.append("WHERE {:s}".format(where))

        return " ".join(query_list)

    @classmethod
    def make_where(cls, key, value, operation="="):
        """
        Make part of WHERE query.

        :param str key: Attribute name of the key.
        :param str value: Value of the right hand side associated with the key.
        :param str operation: Operator of WHERE query.
        :return: Part of WHERE query of SQLite.
        :rtype: str
        :raises simplesqlite.SqlSyntaxError:
            If **a)** ``operation`` is invalid operator.
            Valid operators are as follows:
            ``"="``, ``"=="``, ``"!="``, ``"<>"``,
            ``">"``, ``">="``, ``"<"``, ``"<="``.
            **b)** the ``value`` is |None| and
            the ``operation`` is not ``"="``/``"!="``.

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.make_where("key", "hoge")
            "key = 'hoge'"
            >>> SqlQuery.make_where("value", 1, operation=">")
            'value > 1'
        """

        if operation not in cls.__VALID_WHERE_OPERATION_LIST:
            raise SqlSyntaxError("operation not supported: " + str(operation))

        if value is not None:
            return "{:s} {:s} {:s}".format(
                cls.to_attr_str(key), operation, cls.to_value_str(value))

        if operation == "=":
            return "{:s} IS NULL".format(cls.to_attr_str(key))
        elif operation == "!=":
            return "{:s} IS NOT NULL".format(cls.to_attr_str(key))

        raise SqlSyntaxError(
            "Invalid operation ({:s}) with None right-hand side".format(
                operation))

    @classmethod
    def make_where_in(cls, key, value_list):
        """
        Make part of WHERE IN query.

        :param str key: Attribute name of the key.
        :param str value_list:
            List of values that the right hand side associated with the key.
        :return: Part of WHERE query of SQLite.
        :rtype: str

        :Examples:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.make_where_in("key", ["hoge", "foo", "bar"])
            "key IN ('hoge', 'foo', 'bar')"
        """

        return "{:s} IN ({:s})".format(
            cls.to_attr_str(key), ", ".join(cls.to_value_str_list(value_list)))

    @classmethod
    def make_where_not_in(cls, key, value_list):
        """
        Make part of WHERE NOT IN query.

        :param str key: Attribute name of the key.
        :param str value_list:
            List of values that the right hand side associated with the key.
        :return: Part of WHERE query of SQLite.
        :rtype: str

        :Example:
            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.make_where_not_in("key", ["hoge", "foo", "bar"])
            "key NOT IN ('hoge', 'foo', 'bar')"
        """

        return "{:s} NOT IN ({:s})".format(
            cls.to_attr_str(key), ", ".join(cls.to_value_str_list(value_list)))
<|endoftext|>


<commit_before>import json
import requests
from flask import Flask, request, jsonify, url_for, g
from functools import wraps
from threading import Thread

app = Flask( __name__ )
app.hosts = []

def decoder( js ):
	return js

def encoder( v ):
	print v
	return json.dumps( { "value": v } )

app.encoder = encoder
app.decoder = decoder

data = {}

def async( f ):
	@wraps( f )
	def wrapper( *args, **kwargs ):
		thread = Thread( target=f, args=args, kwargs=kwargs )
		thread.start()

	return wrapper

def get( h, host_str, headers={} ):

	try:
		return requests.get( host_str, headers=headers, timeout=5 )
	except requests.exceptions.ConnectionError:
		app.logger.debug( "Removing %s from the hosts list, because it appears to be down." % ( h ) )
		index = app.hosts.index( h )
		del app.hosts[index]
		raise

@async
def distribute_set( args, origin, host ):
	set_str = "&".join( [ "%s=%s" % ( k, v ) for k, v in args.iteritems() ] )
	headers = { "Propagate": True, "OriginHost": host }
	hosts = app.hosts[:]
	for h in hosts:
		if h != origin:
			host_str = "http://%s/set?%s" % ( h, set_str )

			app.logger.debug( "Propagating set to %s" % host_str )

			get( h, host_str, headers=headers )

@async
def distribute_del( key, origin, host ):
	headers = { "Propagate": True, "OriginHost": host }
	hosts = app.hosts[:]
	for h in hosts:
		host_str = "http://%s/del/%s" % ( h, key )

		app.logger.debug( "Propagating delete to %s" % host_str )

		get( h, host_str, headers=headers )
@async
def distribute_get( key, origin, host ):

	headers = { "Propagate": True, "OriginHost": host }

	hosts = app.hosts[:]
	for h in hosts:
		if h != origin:
			host_str = "http://%s/get/%s" % ( h, key )

			result = get( h, host_str, headers=headers )

			if not result.text == "":
				data[key] = result.text
				break

@async
def send_pings( me ):
	hosts = app.hosts[:]
	for h in hosts:
		host_str = "http://%s/ping?me=%s" % ( h, me )
		result = get( h, host_str )

		if not result.text in app.hosts:
			app.logger.debug( "New host: %s" % result.text )
			app.hosts.append( result.text )

#@app.before_first_request
def send_the_pings():
	send_pings( request.host )

@app.route( "/ping" )
def ping():
	host = request.args["me"]

	if not host in app.hosts:
		app.hosts.append( host )

	return request.host

@app.route( "/set" )
def set_key():

	for k, v in request.args.iteritems():
		app.logger.debug( "Setting %s => %s" % ( k, v ) )
		data[k] = v

	app.logger.debug( "Should we propagate the set value?" )
	if not "Propagate" in request.headers:
		app.logger.debug( "No `Propagate` in headers so let's start the propagation." )
		distribute_set( request.args, "", request.host )
	else:
		app.logger.debug( "There was a `Propagate` in the headers so there must be a `OriginHost` as well so lets grab it and then start propagating." )
		origin = request.headers.get( "OriginHost" )
		distribute_set( request.args, origin, request.host )

	return ""

@app.route( "/get/<key>" )
def get_key( key ):

	app.logger.debug( "Trying to get the value for %s" % key )
	if key in data:
		return app.encoder( data[key] )
	else:
		app.logger.debug( "The key did not exist so lets try and grab it from the other nodes." )
		app.logger.debug( "Should we propagate the set value?" )
		if not "Propagate" in request.headers:
			app.logger.debug( "No `Propagate` in headers so let's start the propagation." )
			distribute_get( key, "", request.host )
		else:
			app.logger.debug( "There was a `Propagate` in the headers so there must be a `OriginHost` as well so lets grab it and then start propagating." )
			origin = request.headers.get( "OriginHost" )
			distribute_get( key, origin, request.host )

	return ""

@app.route( "/del/<key>" )
def del_key( key ):
	del data[key]

	app.logger.debug( "Should we propagate the del value?" )
	if not "Propagate" in request.headers:
		app.logger.debug( "No `Propagate` in headers so let's start the propagation." )
		distribute_del( key, "", request.host )
	else:
		app.logger.debug( "There was a `Propagate` in the headers so there must be a `OriginHost` as well so lets grab it and then start propagating." )
		origin = request.headers.get( "OriginHost" )
		distribute_del( key, origin, request.host )

	return ""

<commit_msg>Fixed a del issue not checking if the key existed first.<commit_after>import json
import requests
from flask import Flask, request, jsonify, url_for, g
from functools import wraps
from threading import Thread

app = Flask( __name__ )
app.hosts = []

def decoder( js ):
	return js

def encoder( v ):
	print v
	return json.dumps( { "value": v } )

app.encoder = encoder
app.decoder = decoder

data = {}

def async( f ):
	@wraps( f )
	def wrapper( *args, **kwargs ):
		thread = Thread( target=f, args=args, kwargs=kwargs )
		thread.start()

	return wrapper

def get( h, host_str, headers={} ):

	try:
		return requests.get( host_str, headers=headers, timeout=5 )
	except requests.exceptions.ConnectionError:
		app.logger.debug( "Removing %s from the hosts list, because it appears to be down." % ( h ) )
		index = app.hosts.index( h )
		del app.hosts[index]
		raise

@async
def distribute_set( args, origin, host ):
	set_str = "&".join( [ "%s=%s" % ( k, v ) for k, v in args.iteritems() ] )
	headers = { "Propagate": True, "OriginHost": host }
	hosts = app.hosts[:]
	for h in hosts:
		if h != origin:
			host_str = "http://%s/set?%s" % ( h, set_str )

			app.logger.debug( "Propagating set to %s" % host_str )

			get( h, host_str, headers=headers )

@async
def distribute_del( key, origin, host ):
	headers = { "Propagate": True, "OriginHost": host }
	hosts = app.hosts[:]
	for h in hosts:
		host_str = "http://%s/del/%s" % ( h, key )

		app.logger.debug( "Propagating delete to %s" % host_str )

		get( h, host_str, headers=headers )
@async
def distribute_get( key, origin, host ):

	headers = { "Propagate": True, "OriginHost": host }

	hosts = app.hosts[:]
	for h in hosts:
		if h != origin:
			host_str = "http://%s/get/%s" % ( h, key )

			result = get( h, host_str, headers=headers )

			if not result.text == "":
				data[key] = result.text
				break

@async
def send_pings( me ):
	hosts = app.hosts[:]
	for h in hosts:
		host_str = "http://%s/ping?me=%s" % ( h, me )
		result = get( h, host_str )

		if not result.text in app.hosts:
			app.logger.debug( "New host: %s" % result.text )
			app.hosts.append( result.text )

#@app.before_first_request
def send_the_pings():
	send_pings( request.host )

@app.route( "/ping" )
def ping():
	host = request.args["me"]

	if not host in app.hosts:
		app.hosts.append( host )

	return request.host

@app.route( "/set" )
def set_key():

	for k, v in request.args.iteritems():
		app.logger.debug( "Setting %s => %s" % ( k, v ) )
		data[k] = v

	app.logger.debug( "Should we propagate the set value?" )
	if not "Propagate" in request.headers:
		app.logger.debug( "No `Propagate` in headers so let's start the propagation." )
		distribute_set( request.args, "", request.host )
	else:
		app.logger.debug( "There was a `Propagate` in the headers so there must be a `OriginHost` as well so lets grab it and then start propagating." )
		origin = request.headers.get( "OriginHost" )
		distribute_set( request.args, origin, request.host )

	return ""

@app.route( "/get/<key>" )
def get_key( key ):

	app.logger.debug( "Trying to get the value for %s" % key )
	if key in data:
		return app.encoder( data[key] )
	else:
		app.logger.debug( "The key did not exist so lets try and grab it from the other nodes." )
		app.logger.debug( "Should we propagate the set value?" )
		if not "Propagate" in request.headers:
			app.logger.debug( "No `Propagate` in headers so let's start the propagation." )
			distribute_get( key, "", request.host )
		else:
			app.logger.debug( "There was a `Propagate` in the headers so there must be a `OriginHost` as well so lets grab it and then start propagating." )
			origin = request.headers.get( "OriginHost" )
			distribute_get( key, origin, request.host )

	return ""

@app.route( "/del/<key>" )
def del_key( key ):
	if key in data:
		del data[key]

		app.logger.debug( "Should we propagate the del value?" )
		if not "Propagate" in request.headers:
			app.logger.debug( "No `Propagate` in headers so let's start the propagation." )
			distribute_del( key, "", request.host )
		else:
			app.logger.debug( "There was a `Propagate` in the headers so there must be a `OriginHost` as well so lets grab it and then start propagating." )
			origin = request.headers.get( "OriginHost" )
			distribute_del( key, origin, request.host )

	return ""

<|endoftext|>


<commit_before>from HTMLParser import HTMLParser
from os import path

import requests
import xbmc
import xbmcaddon
import xbmcgui
from resources.lib.menu import Menu

class NFLC(object):
    _short = str(None)
    _categories = dict()
    _website_url = str(None)
    _parameters = list()

    def go(self):
        if "id" in self._parameters:
            self.play_video()
        elif "category" in self._parameters:
            self.list_videos()
        else:
            self.list_categories()

    def play_video(self):
        parameters = {"id": self._parameters["id"]}
        response = requests.get("{0}/media/nflc-content.json".format(self._website_url), params=parameters)
        data = response.json()

        title = data["headline"]
        thumbnail = data["imagePaths"]["l"]
        remotehost = data["cdnData"]["streamingRemoteHost"]
        video_path = self._get_path_to_video(data["cdnData"]["bitrateInfo"])

        if not video_path.startswith("http://"):
            video_path = "{0}{1}?r=&fp=&v=&g=".format(remotehost, video_path)

        listitem = xbmcgui.ListItem(title, thumbnailImage=thumbnail)
        listitem.setProperty("PlayPath", video_path)
        xbmc.Player().play(video_path, listitem)

    @classmethod
    def _get_path_to_video(cls, videos):
        max_bitrate = int(xbmcaddon.Addon("plugin.video.nfl-teams").getSetting("max_bitrate")) * 1000000 or 5000000
        bitrate = -1
        lowest_bitrate = None

        for video in videos:
            if video["rate"] > bitrate and video["rate"] <= max_bitrate:
                best_video = video["path"]
                bitrate = video["rate"]
            if not lowest_bitrate or video["rate"] < lowest_bitrate:
                lowest_video = video["path"]
                lowest_bitrate = video["rate"]

        return best_video or lowest_video

    def list_videos(self):
        parameters = {"type": "VIDEO", "channelKey": self._parameters["category"]}

        response = requests.get("{0}/media/nflc-playlist-video.json".format(self._website_url))
        data = response.json()
        html_parser = HTMLParser()

        with Menu(["none"]) as menu:
            for video in data[str(self._parameters["category"])]["data"]:
                menu.add_item({
                    "url_params": {"team": self._short, "id": video["video_id"]},
                    "name": html_parser.unescape(video["title"]),
                    "folder": False,
                    "thumbnail": "{0}{1}".format(self._website_url, video["thumbnail"]),
                })

    def list_categories(self):
        with Menu(["none"]) as menu:
            for category_id, category_name in self._categories:
                menu.add_item({
                    "url_params": {"team": self._short, "category": category_id},
                    "name": category_name,
                    "folder": True,
                    "thumbnail": path.join("resources", "images", "{0}.png".format(self._short)),
                })
<commit_msg>Add "All Videos" menu item to NFLC teams too<commit_after>from HTMLParser import HTMLParser
from os import path

import requests
import xbmc
import xbmcaddon
import xbmcgui
from resources.lib.menu import Menu

class NFLC(object):
    _short = str(None)
    _categories = dict()
    _website_url = str(None)
    _parameters = list()

    def go(self):
        if "id" in self._parameters:
            self.play_video()
        elif "category" in self._parameters:
            self.list_videos()
        else:
            self.list_categories()

    def play_video(self):
        parameters = {"id": self._parameters["id"]}
        response = requests.get("{0}/media/nflc-content.json".format(self._website_url), params=parameters)
        data = response.json()

        title = data["headline"]
        thumbnail = data["imagePaths"]["l"]
        remotehost = data["cdnData"]["streamingRemoteHost"]
        video_path = self._get_path_to_video(data["cdnData"]["bitrateInfo"])

        if not video_path.startswith("http://"):
            video_path = "{0}{1}?r=&fp=&v=&g=".format(remotehost, video_path)

        listitem = xbmcgui.ListItem(title, thumbnailImage=thumbnail)
        listitem.setProperty("PlayPath", video_path)
        xbmc.Player().play(video_path, listitem)

    @classmethod
    def _get_path_to_video(cls, videos):
        max_bitrate = int(xbmcaddon.Addon("plugin.video.nfl-teams").getSetting("max_bitrate")) * 1000000 or 5000000
        bitrate = -1
        lowest_bitrate = None

        for video in videos:
            if video["rate"] > bitrate and video["rate"] <= max_bitrate:
                best_video = video["path"]
                bitrate = video["rate"]
            if not lowest_bitrate or video["rate"] < lowest_bitrate:
                lowest_video = video["path"]
                lowest_bitrate = video["rate"]

        return best_video or lowest_video

    def list_videos(self):
        parameters = {"type": "VIDEO", "channelKey": self._parameters["category"]}

        response = requests.get("{0}/media/nflc-playlist-video.json".format(self._website_url))
        data = response.json()
        html_parser = HTMLParser()

        videos = list()
        if self._parameters["category"] == "all":
            for category in data.itervalues():
                videos.extend(category["data"])
        else:
            videos = data[str(self._parameters["category"])]["data"]

        with Menu(["none"]) as menu:
            for video in videos:
                menu.add_item({
                    "url_params": {"team": self._short, "id": video["video_id"]},
                    "name": html_parser.unescape(video["title"]),
                    "folder": False,
                    "thumbnail": "{0}{1}".format(self._website_url, video["thumbnail"]),
                })

    def list_categories(self):
        with Menu(["none"]) as menu:
            menu.add_item({
                "url_params": {"team": self._short, "category": "all"},
                "name": "[B]All Videos[/B]",
                "folder": True,
                "thumbnail": path.join("resources", "images", "{0}.png".format(self._short))
            })
            for category_id, category_name in self._categories:
                menu.add_item({
                    "url_params": {"team": self._short, "category": category_id},
                    "name": category_name,
                    "folder": True,
                    "thumbnail": path.join("resources", "images", "{0}.png".format(self._short)),
                })
<|endoftext|>


<commit_before>#!/usr/bin/env python

NAME = 'DOSarrest'

def is_waf(self):
    return self.matchheader(('X-DIS-Request-ID', '.+')) or self.matchheader(('server', 'DOSarrest'))<commit_msg>Update dosarrest.py<commit_after>#!/usr/bin/env python

NAME = 'DOSarrest'

def is_waf(self):
    return self.matchheader(('X-DIS-Request-ID', '.+')) or self.matchheader(('server', 'DOSarrest'))
<|endoftext|>


<commit_before>import claripy
from .plugin import SimStatePlugin

class SimRegNameView(SimStatePlugin):
    def __init__(self):
        super(SimRegNameView, self).__init__()

    def __getattr__(self, k):
        try:
            return self.state.reg_expr(self.state.arch.registers[k][0], self.state.arch.registers[k][1])
        except KeyError:
            return getattr(super(SimRegNameView, self), k)

    def __setattr__(self, k, v):
        if k == 'state':
            return object.__setattr__(self, k, v)

        v = _raw_ast(v)
        if not isinstance(v, claripy.Bits):
            v = self.state.se.BVV(v, self.state.arch.registers[k][1]*8)

        try:
            return self.state.store_reg(self.state.arch.registers[k][0], v)
        except KeyError:
            raise AttributeError(k)

    def __dir__(self):
        return self.state.arch.registers.keys()

    def copy(self):
        return SimRegNameView()

    def merge(self, others, merge_flag, flag_values):
        return False, [ ]

    def widen(self, others, merge_flag, flag_values):
        return False, [ ]

class SimMemView(SimStatePlugin):
    def __init__(self, ty=None, addr=None, state=None):
        super(SimMemView, self).__init__()
        self._type = ty
        self._addr = addr
        if state is not None:
            self.set_state(state)

    def set_state(self, state):
        super(SimMemView, self).set_state(state)

    def _deeper(self, **kwargs):
        if 'ty' not in kwargs:
            kwargs['ty'] = self._type
        if 'addr' not in kwargs:
            kwargs['addr'] = self._addr
        if 'state' not in kwargs:
            kwargs['state'] = self.state
        return SimMemView(**kwargs)

    def __getitem__(self, k):
        if isinstance(k, slice):
            if k.step is not None:
                raise ValueError("Slices with strides are not supported")
            elif k.start is None:
                raise ValueError("Must specify start index")
            elif k.stop is not None:
                raise ValueError("Slices with stop index are not supported")
            else:
                return self._deeper(addr=k.start)
        else:
            raise KeyError(k)

    def __setitem__(self, k, v):
        self.__getitem__(k).store(v)

    types = {}
    state = None

    def __repr__(self):
        value = '<unresolvable>' if not self.resolvable else self.resolved
        if isinstance(self._addr, claripy.BV) and not self._addr.symbolic:
            if hasattr(self._addr.model, 'value'):
                addr = format(self._addr.model.value, '#x')
            else:
                addr = repr(self._addr.model)
        else:
            addr = repr(self._addr)
        return '<{} {} at {}>'.format(self._type.name,
                                      value,
                                      addr)

    def __dir__(self):
        return self._type._refine_dir() if self._type else SimMemView.types.keys()

    def __getattr__(self, k):
        if k in ('resolvable', 'resolved'):
            return object.__getattribute__(self, k)
        return self._type._refine(self, k) if self._type else self._deeper(ty=SimMemView.types[k](self.state.arch))

    def __setattr__(self, k, v):
        if k in ('state', '_addr', '_type'):
            return object.__setattr__(self, k, v)
        self.__getattr__(k).store(v)

    def copy(self):
        return SimMemView()

    def merge(self, others, merge_flag, flag_values):
        return False, [ ]

    def widen(self, others, merge_flag, flag_values):
        return False, [ ]

    @property
    def resolvable(self):
        return self._type is not None and self._addr is not None

    @property
    def resolved(self):
        if not self.resolvable:
            raise ValueError("Trying to resolve value without type and addr defined")
        return self._type.extract(self.state, self._addr)

    def store(self, value):
        if self._addr is None:
            raise ValueError("Trying to store to location without specifying address")

        if isinstance(value, claripy.BV):
            return self.state.store_mem(self._addr, value)

        if self._type is None:
            raise ValueError("Trying to store to location without specifying type")

        return self._type.store(self.state, self._addr, value)

from ..s_type import ALL_TYPES
SimMemView.types = ALL_TYPES # identity purposefully here

from ..s_action_object import _raw_ast
<commit_msg>support `state.mem[addr].int` syntax<commit_after>import claripy
from .plugin import SimStatePlugin

class SimRegNameView(SimStatePlugin):
    def __init__(self):
        super(SimRegNameView, self).__init__()

    def __getattr__(self, k):
        try:
            return self.state.reg_expr(self.state.arch.registers[k][0], self.state.arch.registers[k][1])
        except KeyError:
            return getattr(super(SimRegNameView, self), k)

    def __setattr__(self, k, v):
        if k == 'state':
            return object.__setattr__(self, k, v)

        v = _raw_ast(v)
        if not isinstance(v, claripy.Bits):
            v = self.state.se.BVV(v, self.state.arch.registers[k][1]*8)

        try:
            return self.state.store_reg(self.state.arch.registers[k][0], v)
        except KeyError:
            raise AttributeError(k)

    def __dir__(self):
        return self.state.arch.registers.keys()

    def copy(self):
        return SimRegNameView()

    def merge(self, others, merge_flag, flag_values):
        return False, [ ]

    def widen(self, others, merge_flag, flag_values):
        return False, [ ]

class SimMemView(SimStatePlugin):
    def __init__(self, ty=None, addr=None, state=None):
        super(SimMemView, self).__init__()
        self._type = ty
        self._addr = addr
        if state is not None:
            self.set_state(state)

    def set_state(self, state):
        super(SimMemView, self).set_state(state)

    def _deeper(self, **kwargs):
        if 'ty' not in kwargs:
            kwargs['ty'] = self._type
        if 'addr' not in kwargs:
            kwargs['addr'] = self._addr
        if 'state' not in kwargs:
            kwargs['state'] = self.state
        return SimMemView(**kwargs)

    def __getitem__(self, k):
        if isinstance(k, slice):
            if k.step is not None:
                raise ValueError("Slices with strides are not supported")
            elif k.start is None:
                raise ValueError("Must specify start index")
            elif k.stop is not None:
                raise ValueError("Slices with stop index are not supported")
            else:
                addr = k.start
        else:
            addr = k
        return self._deeper(addr=addr)

    def __setitem__(self, k, v):
        self.__getitem__(k).store(v)

    types = {}
    state = None

    def __repr__(self):
        value = '<unresolvable>' if not self.resolvable else self.resolved
        if isinstance(self._addr, claripy.BV) and not self._addr.symbolic:
            if hasattr(self._addr.model, 'value'):
                addr = format(self._addr.model.value, '#x')
            else:
                addr = repr(self._addr.model)
        else:
            addr = repr(self._addr)
        return '<{} {} at {}>'.format(self._type.name,
                                      value,
                                      addr)

    def __dir__(self):
        return self._type._refine_dir() if self._type else SimMemView.types.keys()

    def __getattr__(self, k):
        if k in ('resolvable', 'resolved'):
            return object.__getattribute__(self, k)
        return self._type._refine(self, k) if self._type else self._deeper(ty=SimMemView.types[k](self.state.arch))

    def __setattr__(self, k, v):
        if k in ('state', '_addr', '_type'):
            return object.__setattr__(self, k, v)
        self.__getattr__(k).store(v)

    def copy(self):
        return SimMemView()

    def merge(self, others, merge_flag, flag_values):
        return False, [ ]

    def widen(self, others, merge_flag, flag_values):
        return False, [ ]

    @property
    def resolvable(self):
        return self._type is not None and self._addr is not None

    @property
    def resolved(self):
        if not self.resolvable:
            raise ValueError("Trying to resolve value without type and addr defined")
        return self._type.extract(self.state, self._addr)

    def store(self, value):
        if self._addr is None:
            raise ValueError("Trying to store to location without specifying address")

        if isinstance(value, claripy.BV):
            return self.state.store_mem(self._addr, value)

        if self._type is None:
            raise ValueError("Trying to store to location without specifying type")

        return self._type.store(self.state, self._addr, value)

from ..s_type import ALL_TYPES
SimMemView.types = ALL_TYPES # identity purposefully here

from ..s_action_object import _raw_ast
<|endoftext|>