<commit_before>import os
import sys
import shutil
import logging
import time

try:
    # Python 3.x
    from urllib.parse import quote
    from urllib.parse import urlencode
    from urllib.request import urlopen
    from urllib.error import URLError
    from urllib.request import HTTPSHandler
    from urllib.request import HTTPPasswordMgrWithDefaultRealm
    from urllib.request import HTTPBasicAuthHandler
    from urllib.request import build_opener
    from urllib.request import install_opener
except ImportError:
    # Python 2.x
    from urllib import urlencode
    from urllib2 import quote
    from urllib2 import urlopen
    from urllib2 import URLError
    from urllib2 import HTTPSHandler
    from urllib2 import HTTPPasswordMgrWithDefaultRealm
    from urllib2 import HTTPBasicAuthHandler
    from urllib2 import build_opener
    from urllib2 import install_opener

import tarfile

import fhem

class FhemSelfTester:
    def __init__(self):
        self.log = logging.getLogger('SelfTester')

    def download(self, filename, urlpath):
        if os.path.exists(filename):
            return True
        try:
            dat = urlopen(urlpath).read()
        except Exception as e:
            self.log.error("Failed to download {}, {}".format(urlpath, e))
            return False
        try:
            with open(filename, 'wb') as f:
                f.write(dat)
        except Exception as e:
            self.log.error("Failed to write {}, {}".format(filename, e))
        return True

    def install(self, archivename, destination):
        if not archivename.endswith("tar.gz"):
            self.log.error(
                "Archive needs to be of type *.tar.gz: {}".format(archivename))
            return False
        if not os.path.exists(archivename):
            self.log.error("Archive {} not found.".format(archivename))
            return False
        if "fhem" not in destination:
            self.log.error(
                "Dangerous or inconsistent fhem install-path: {}, need destination with 'fhem' in name.".format(destination))
            return False
        if os.path.exists(destination):
            try:
                shutil.rmtree(destination)
            except Exception as e:
                self.log.error(
                    "Failed to remove existing installation at {}".format(destination))
                return False
        try:
            tar = tarfile.open(archivename, "r:gz")
            tar.extractall(destination)
            tar.close()
        except Exception as e:
            self.log.error("Failed to extract {}, {}".format(archivename, e))
        return True

    def is_running(self, fhem_url='localhost', protocol='http', port=8083):
        fh = fhem.Fhem(fhem_url, protocol=protocol, port=port)
        ver = fh.send_cmd('version')
        if ver is not None:
            fh.close()
            return ver
        return None

    def shutdown(self, fhem_url='localhost', protocol='http', port=8083):
        fh = fhem.Fhem(fhem_url, protocol=protocol, port=port)
        fh.log.level = logging.CRITICAL
        try:
            self.log.warning("Shutting down fhem at {}".format(fhem_url))
            fh.send_cmd("shutdown")
        except:
            pass
        self.log.warning("Fhem shutdown complete.")

if __name__ == '__main__':
    st = FhemSelfTester()
    config = {
        'archivename': "./fhem-5.9.tar.gz",
        'urlpath': "https://fhem.de/fhem-5.9.tar.gz",
        'destination': "./fhem",
        'exec': "cd fhem/fhem-5.9/ && perl fhem.pl fhem.cfg",
        'testhost': 'localhost',
        'protocol': 'http',
        'port': 8083
    }

    if st.is_running(fhem_url=config['testhost'], protocol='http', port=8083) is not None:
        print("Fhem is already running!")
        st.shutdown(fhem_url=config['testhost'], protocol='http', port=8083)
        time.sleep(1)
        if st.is_running(fhem_url=config['testhost'], protocol='http', port=8083) is not None:
            print("Shutdown failed!")
            sys.exit(-3)
        print("--------------------")
        print("Reinstalling FHEM...")

    if not st.download(config['archivename'], config['urlpath']):
        print("Download failed.")
        sys.exit(-1)

    if not st.install(config['archivename'], config['destination']):
        print("Install failed")
        sys.exit(-2)

    os.system(config['exec'])
    time.sleep(1)

    if st.is_running(fhem_url=config['testhost'], protocol='http', port=8083) is None:
        print("Fhem is NOT running after install and start!")
        sys.exit(-4)

    print("Install should be ok, Fhem running.")

    sys.exit(0)
<commit_msg>create dummy device on automatic fhem installation and test reading of reading<commit_after>import os
import sys
import shutil
import logging
import time

try:
    # Python 3.x
    from urllib.parse import quote
    from urllib.parse import urlencode
    from urllib.request import urlopen
    from urllib.error import URLError
    from urllib.request import HTTPSHandler
    from urllib.request import HTTPPasswordMgrWithDefaultRealm
    from urllib.request import HTTPBasicAuthHandler
    from urllib.request import build_opener
    from urllib.request import install_opener
except ImportError:
    # Python 2.x
    from urllib import urlencode
    from urllib2 import quote
    from urllib2 import urlopen
    from urllib2 import URLError
    from urllib2 import HTTPSHandler
    from urllib2 import HTTPPasswordMgrWithDefaultRealm
    from urllib2 import HTTPBasicAuthHandler
    from urllib2 import build_opener
    from urllib2 import install_opener

import tarfile

import fhem

class FhemSelfTester:
    def __init__(self):
        self.log = logging.getLogger('SelfTester')

    def download(self, filename, urlpath):
        if os.path.exists(filename):
            return True
        try:
            dat = urlopen(urlpath).read()
        except Exception as e:
            self.log.error("Failed to download {}, {}".format(urlpath, e))
            return False
        try:
            with open(filename, 'wb') as f:
                f.write(dat)
        except Exception as e:
            self.log.error("Failed to write {}, {}".format(filename, e))
        return True

    def install(self, archivename, destination, sanity_check_file):
        if not archivename.endswith("tar.gz"):
            self.log.error(
                "Archive needs to be of type *.tar.gz: {}".format(archivename))
            return False
        if not os.path.exists(archivename):
            self.log.error("Archive {} not found.".format(archivename))
            return False
        if "fhem" not in destination or not os.path.exists(sanity_check_file):
            self.log.error(
                "Dangerous or inconsistent fhem install-path: {}, need destination with 'fhem' in name.".format(destination))
            return False
        if os.path.exists(destination):
            try:
                shutil.rmtree(destination)
            except Exception as e:
                self.log.error(
                    "Failed to remove existing installation at {}".format(destination))
                return False
        try:
            tar = tarfile.open(archivename, "r:gz")
            tar.extractall(destination)
            tar.close()
        except Exception as e:
            self.log.error("Failed to extract {}, {}".format(archivename, e))
        return True

    def is_running(self, fhem_url='localhost', protocol='http', port=8083):
        fh = fhem.Fhem(fhem_url, protocol=protocol, port=port)
        ver = fh.send_cmd('version')
        if ver is not None:
            fh.close()
            return ver
        return None

    def shutdown(self, fhem_url='localhost', protocol='http', port=8083):
        fh = fhem.Fhem(fhem_url, protocol=protocol, port=port)
        fh.log.level = logging.CRITICAL
        try:
            self.log.warning("Shutting down fhem at {}".format(fhem_url))
            fh.send_cmd("shutdown")
        except:
            pass
        self.log.warning("Fhem shutdown complete.")

if __name__ == '__main__':
    st = FhemSelfTester()
    config = {
        'archivename': "./fhem-5.9.tar.gz",
        'urlpath': "https://fhem.de/fhem-5.9.tar.gz",
        'destination': "./fhem",
        'fhem_file': "./fhem/fhem-5.9/fhem.pl",
        'exec': "cd fhem/fhem-5.9/ && perl fhem.pl fhem.cfg",
        'testhost': 'localhost',
        'protocol': 'http',
        'port': 8083
    }

    if st.is_running(fhem_url=config['testhost'], protocol='http', port=8083) is not None:
        print("Fhem is already running!")
        st.shutdown(fhem_url=config['testhost'], protocol='http', port=8083)
        time.sleep(1)
        if st.is_running(fhem_url=config['testhost'], protocol='http', port=8083) is not None:
            print("Shutdown failed!")
            sys.exit(-3)
        print("--------------------")
        print("Reinstalling FHEM...")

    if not st.download(config['archivename'], config['urlpath']):
        print("Download failed.")
        sys.exit(-1)

    if not st.install(config['archivename'], config['destination'], config['fhem_file']):
        print("Install failed")
        sys.exit(-2)

    os.system(config['exec'])
    time.sleep(1)

    if st.is_running(fhem_url=config['testhost'], protocol='http', port=8083) is None:
        print("Fhem is NOT running after install and start!")
        sys.exit(-4)

    print("Install should be ok, Fhem running.")

    fh = fhem.Fhem(config['testhost'], protocol='http', port=8083)
    fh.send_cmd("define grotzel dummy")
    fh.send_cmd("attr grotzel setList state:on,off")
    fh.send_cmd("set grotzel on")
    fh.send_cmd("attr grotzel readingList temperature humidity")
    fh.send_cmd("setreading grotzel temperature 10.2")
    fh.send_cmd("setreading grotzel humidity 88.9")

    temp = fh.get_dev_reading("grotzel", "temperature")
    if temp == 10.2:
        print("Reading-test ok.")
    else:
        print("Failed to set and read dummy reading!")
        sys.exit(-5)

    fh.close()

    sys.exit(0)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Program to align rnaseq transcriptome reads to the genome using STAR aligner

Requirement:
    STAR - https://github.com/alexdobin/STAR/releases
    pysam - http://code.google.com/p/pysam/
"""

import os
import sys
import subprocess
from collections import defaultdict
from gfftools import helper, GFFParser

def run_star_alignment(org_db, read_type='PE', max_mates_gap_length=100000, num_cpus=1):
    """
    wrapper for running STAR program

    @args org_db: a python dictionary with all details about a single organism
    @type org_db: defaultdict
    @args read_type: library type - paired-end or single-end (default: PE)
    @type read_type: str
    @args max_mates_gap_length: maximum insert size from the sample (default: 10000)
    @type max_mates_gap_length: int
    @args num_cpus: number of threads to use for the run (default: 1)
    @type num_cpus: int
    """

    ## genome indices and annotation file
    genome_dir = org_db['genome_index_dir']
    gtf_db = org_db['gtf']

    ## check for the file type
    gff_hand = helper.open_file(gtf_db)
    for rec in gff_hand:
        rec = rec.strip('\n\r')
        # skip empty line fasta identifier and commented line
        if not rec or rec[0] in  ['#', '>']:
            continue
        # skip the genome sequence
        if not re.search('\t', rec):
            continue
        parts = rec.split('\t')
        assert len(parts) >= 8, rec
        ftype, tags = GFFParser.attribute_tags(parts[-1])
        break
    gff_hand.close()

    ## library type
    if read_type == 'PE':
        read_file = "%s/%s %s/%s" % (org_db['fastq_path'], org_db['fastq'][0], org_db['fastq_path'], org_db['fastq'][1])
    else:
        read_file = "%s/%s" % (org_db['fastq_path'], org_db['fastq'][0])

    ## getting the command to uncompress the read file
    zip_type = {".gz" : "gzip -c", ".bz2" : "bzip2 -c"}
    file_prefx, ext = os.path.splitext(org_db['fastq'][0])

    ## genomic feature information
    max_lenth_intron = org_db['max_intron_len']

    out_prefix = '%s/%s_' % (org_db['read_map_dir'], org_db['short_name'])

    ## according to the file type
    if ftype:
        make_star_run = "STAR \
        --genomeDir %s \
        --readFilesIn %s \
        --readFilesCommand %s \
        --outFileNamePrefix %s \
        --runThreadN %d \
        --outFilterMultimapScoreRange 2 \
        --outFilterMultimapNmax 30 \
        --outFilterMismatchNmax 5 \
        --alignIntronMax %d \
        --sjdbGTFfile %s \
        --sjdbGTFtagExonParentTranscript Parent \
        --sjdbScore 1 \
        --sjdbOverhang 5 \
        --alignMatesGapMax %d\
        --outSAMstrandField intronMotif \
        --outFilterIntronMotifs RemoveNoncanonical \
        --outSAMtype BAM SortedByCoordinate \
        --genomeLoad LoadAndRemove" % (genome_dir, read_file,
            zip_type[ext], out_prefix, num_cpus, max_lenth_intron, gtf_db, max_mates_gap_length)
    else:
        make_star_run = "STAR \
        --genomeDir %s \
        --readFilesIn %s \
        --readFilesCommand %s \
        --outFileNamePrefix %s \
        --runThreadN %d \
        --outFilterMultimapScoreRange 2 \
        --outFilterMultimapNmax 30 \
        --outFilterMismatchNmax 4 \
        --alignIntronMax %d \
        --sjdbGTFfile %s \
        --sjdbGTFfeatureExon exon \
        --sjdbScore 1 \
        --sjdbOverhang 5 \
        --alignMatesGapMax %d\
        --outSAMstrandField intronMotif \
        --outFilterIntronMotifs RemoveNoncanonical \
        --outSAMtype BAM SortedByCoordinate \
        --genomeLoad LoadAndRemove" % (genome_dir, read_file,
            zip_type[ext], out_prefix, num_cpus, max_lenth_intron, gtf_db, max_mates_gap_length)

    sys.stdout.write('\trunning STAR program as: %s \n' % make_star_run)
    process = subprocess.Popen(make_star_run, shell=True)
    process.wait()

def uniq_mapped_reads(bam_file, multi_map=1):
    """
    Filter a BAM file to extract only the uniquely mapped reads

    @args bam_file: binary file for storing the sequencing reads.
    @type bam_file: str
    @args multi_map: number of hits of a read (default 1)
    @type multi_map: integer
    """

    import pysam

    ## indexing the in bam file
    if not os.path.exists(bam_file + ".bai"):
        pysam.index(bam_file)

    #import time
    #t_start = time.time()

    sam_file = pysam.Samfile(bam_file, "rb")

    ## getting the abs path of in bam file
    bam_file_path = os.path.abspath(bam_file)
    bam_file_dir = os.path.dirname(bam_file_path)

    ## defining the out bam file
    uniq_map_bam_file = '%s/unique_map_reads.bam' % bam_file_dir
    out_bam_fh = pysam.Samfile(uniq_map_bam_file, "wb", template=sam_file)

    ## filtering the alignment based on the read hits
    for read in sam_file.fetch():
        for tag in read.tags:
            if tag[0] == 'NH':
                if int(tag[1]) <= multi_map:
                    out_bam_fh.write(read)

    sam_file.close()
    out_bam_fh.close()

    ## indexing the out bam file
    pysam.index(uniq_map_bam_file)

    #time_taken = time.time() - t_start
    #print time_taken

def read_directions_count(bam_file):
    """
    get the reads directions count from a bam file

    @args bam_file: binary file formt for storing sequencing reads information
    @type bam_file: str
    """

    import pysam

    ## indexing the in bam file
    if not os.path.exists(bam_file + ".bai"):
        pysam.index(bam_file)

    reverse_cnt = 0
    forward_cnt = 0

    bam_fh = pysam.Samfile(bam_file, "rb")

    for read in bam_fh.fetch():
         if read.is_proper_pair and read.is_read1:
            if read.is_reverse:
                reverse_cnt += 1
            else:
                forward_cnt += 1

    bam_fh.close()
    return {'forward_reads_count': forward_cnt, 'reverse_reads_count': reverse_cnt}

def calculate_insert_size_from_bam(bam_file):
    """
    estimate the insert size from uniquely mapped reads in a BAM file
    """

    import pysam

def calculate_insert_size(org_db):
    """
    calculate the insert-size from raw read sequence file
    """

    import re
    from glob import glob

    #FIXME
    """
    vipin@gpu-3-9: /cbio/grlab/nobackup/SignalPrediction/SRA-rnaseq/S_enterica/source_data$ python ~/tmp/7281991/estimate-insert-sizes /cbio/grlab/share/databases/genomes/S_enterica/ensembl_release-21/Salmonella_enterica_subsp_enterica_serovar_typhimurium_str_lt2.GCA_000006945.1.21.dna.toplevel.fa SRR863221_1.fastq.bz2 SRR863221_2.fastq.bz2
    Processing:
     SRR863221_1.fastq.bz2
      SRR863221_2.fastq.bz2
      [M::mem_pestat] # candidate unique pairs for (FF, FR, RF, RR): (0, 0, 0, 0)
      [M::mem_pestat] skip orientation FF as there are not enough pairs
      [M::mem_pestat] skip orientation FR as there are not enough pairs
      [M::mem_pestat] skip orientation RF as there are not enough pairs
      [M::mem_pestat] skip orientation RR as there are not enough pairs
      Traceback (most recent call last):
        File "/cbio/grlab/home/vipin/tmp/7281991/estimate-insert-sizes", line 100, in <module>
            mean = most_likely[1]['mean']
            KeyError: 'mean'

    """
<commit_msg>alignment run finished successfully test completed<commit_after>#!/usr/bin/env python
"""
Program to align rnaseq transcriptome reads to the genome using STAR aligner

Requirement:
    STAR - https://github.com/alexdobin/STAR/releases
    pysam - http://code.google.com/p/pysam/
"""

import os
import re
import sys
import subprocess
from collections import defaultdict
from gfftools import helper, GFFParser

def run_star_alignment(org_db, read_type='PE', max_mates_gap_length=100000, num_cpus=1):
    """
    wrapper for running STAR program

    @args org_db: a python dictionary with all details about a single organism
    @type org_db: defaultdict
    @args read_type: library type - paired-end or single-end (default: PE)
    @type read_type: str
    @args max_mates_gap_length: maximum insert size from the sample (default: 10000)
    @type max_mates_gap_length: int
    @args num_cpus: number of threads to use for the run (default: 1)
    @type num_cpus: int
    """

    ## genome indices and annotation file
    genome_dir = org_db['genome_index_dir']
    gtf_db = org_db['gtf']

    ## check for the file type
    gff_hand = helper.open_file(gtf_db)
    for rec in gff_hand:
        rec = rec.strip('\n\r')
        # skip empty line fasta identifier and commented line
        if not rec or rec[0] in  ['#', '>']:
            continue
        # skip the genome sequence
        if not re.search('\t', rec):
            continue
        parts = rec.split('\t')
        assert len(parts) >= 8, rec
        ftype, tags = GFFParser.attribute_tags(parts[-1])
        break
    gff_hand.close()

    ## library type
    if read_type == 'PE':
        read_file = "%s/%s %s/%s" % (org_db['fastq_path'], org_db['fastq'][0], org_db['fastq_path'], org_db['fastq'][1])
    else:
        read_file = "%s/%s" % (org_db['fastq_path'], org_db['fastq'][0])

    ## getting the command to uncompress the read file
    zip_type = {".gz" : "zcat", ".bz2" : "bzcat"}
    file_prefx, ext = os.path.splitext(org_db['fastq'][0])

    ## genomic feature information
    max_lenth_intron = org_db['max_intron_len']

    out_prefix = '%s/%s_' % (org_db['read_map_dir'], org_db['short_name'])

    ## according to the file type
    if ftype:
        make_star_run = "STAR \
        --genomeDir %s \
        --readFilesIn %s \
        --readFilesCommand %s \
        --outFileNamePrefix %s \
        --runThreadN %d \
        --outFilterMultimapScoreRange 2 \
        --outFilterMultimapNmax 30 \
        --outFilterMismatchNmax 4 \
        --alignIntronMax %d \
        --sjdbGTFfile %s \
        --sjdbGTFtagExonParentTranscript Parent \
        --sjdbScore 1 \
        --sjdbOverhang 5 \
        --alignMatesGapMax %d\
        --outSAMstrandField intronMotif \
        --outFilterIntronMotifs RemoveNoncanonical \
        --outSAMtype BAM SortedByCoordinate \
        --limitBAMsortRAM 8000000000 \
        --genomeLoad LoadAndRemove" % (genome_dir, read_file,
            zip_type[ext], out_prefix, num_cpus, max_lenth_intron, gtf_db, max_mates_gap_length)
    else:
        make_star_run = "STAR \
        --genomeDir %s \
        --readFilesIn %s \
        --readFilesCommand %s \
        --outFileNamePrefix %s \
        --runThreadN %d \
        --outFilterMultimapScoreRange 2 \
        --outFilterMultimapNmax 30 \
        --outFilterMismatchNmax 4 \
        --alignIntronMax %d \
        --sjdbGTFfile %s \
        --sjdbGTFfeatureExon exon \
        --sjdbScore 1 \
        --sjdbOverhang 5 \
        --alignMatesGapMax %d\
        --outSAMstrandField intronMotif \
        --outFilterIntronMotifs RemoveNoncanonical \
        --outSAMtype BAM SortedByCoordinate \
        --limitBAMsortRAM 8000000000 \
        --genomeLoad LoadAndRemove" % (genome_dir, read_file,
            zip_type[ext], out_prefix, num_cpus, max_lenth_intron, gtf_db, max_mates_gap_length)

    sys.stdout.write('\trunning STAR program as: %s \n' % make_star_run)
    process = subprocess.Popen(make_star_run, shell=True)
    process.wait()

def uniq_mapped_reads(bam_file, multi_map=1):
    """
    Filter a BAM file to extract only the uniquely mapped reads

    @args bam_file: binary file for storing the sequencing reads.
    @type bam_file: str
    @args multi_map: number of hits of a read (default 1)
    @type multi_map: integer
    """

    import pysam

    ## indexing the in bam file
    if not os.path.exists(bam_file + ".bai"):
        pysam.index(bam_file)

    #import time
    #t_start = time.time()

    sam_file = pysam.Samfile(bam_file, "rb")

    ## getting the abs path of in bam file
    bam_file_path = os.path.abspath(bam_file)
    bam_file_dir = os.path.dirname(bam_file_path)

    ## defining the out bam file
    uniq_map_bam_file = '%s/unique_map_reads.bam' % bam_file_dir
    out_bam_fh = pysam.Samfile(uniq_map_bam_file, "wb", template=sam_file)

    ## filtering the alignment based on the read hits
    for read in sam_file.fetch():
        for tag in read.tags:
            if tag[0] == 'NH':
                if int(tag[1]) <= multi_map:
                    out_bam_fh.write(read)

    sam_file.close()
    out_bam_fh.close()

    ## indexing the out bam file
    pysam.index(uniq_map_bam_file)

    #time_taken = time.time() - t_start
    #print time_taken

def read_directions_count(bam_file):
    """
    get the reads directions count from a bam file

    @args bam_file: binary file formt for storing sequencing reads information
    @type bam_file: str
    """

    import pysam

    ## indexing the in bam file
    if not os.path.exists(bam_file + ".bai"):
        pysam.index(bam_file)

    reverse_cnt = 0
    forward_cnt = 0

    bam_fh = pysam.Samfile(bam_file, "rb")

    for read in bam_fh.fetch():
         if read.is_proper_pair and read.is_read1:
            if read.is_reverse:
                reverse_cnt += 1
            else:
                forward_cnt += 1

    bam_fh.close()
    return {'forward_reads_count': forward_cnt, 'reverse_reads_count': reverse_cnt}

def calculate_insert_size_from_bam(bam_file):
    """
    estimate the insert size from uniquely mapped reads in a BAM file
    """

    import pysam

def calculate_insert_size(org_db):
    """
    calculate the insert-size from raw read sequence file
    """

    import re
    from glob import glob

    #FIXME
    """
    vipin@gpu-3-9: /cbio/grlab/nobackup/SignalPrediction/SRA-rnaseq/S_enterica/source_data$ python ~/tmp/7281991/estimate-insert-sizes /cbio/grlab/share/databases/genomes/S_enterica/ensembl_release-21/Salmonella_enterica_subsp_enterica_serovar_typhimurium_str_lt2.GCA_000006945.1.21.dna.toplevel.fa SRR863221_1.fastq.bz2 SRR863221_2.fastq.bz2
    Processing:
     SRR863221_1.fastq.bz2
      SRR863221_2.fastq.bz2
      [M::mem_pestat] # candidate unique pairs for (FF, FR, RF, RR): (0, 0, 0, 0)
      [M::mem_pestat] skip orientation FF as there are not enough pairs
      [M::mem_pestat] skip orientation FR as there are not enough pairs
      [M::mem_pestat] skip orientation RF as there are not enough pairs
      [M::mem_pestat] skip orientation RR as there are not enough pairs
      Traceback (most recent call last):
        File "/cbio/grlab/home/vipin/tmp/7281991/estimate-insert-sizes", line 100, in <module>
            mean = most_likely[1]['mean']
            KeyError: 'mean'

    """
<|endoftext|>


<commit_before>"""
Utility functions to mimic the template support functions for vtkVariant
"""

import vtk
import vtkConstants

_variant_type_map = {
    'char' : vtkConstants.VTK_CHAR,
    'unsigned char' : vtkConstants.VTK_UNSIGNED_CHAR,
    'signed char' : vtkConstants.VTK_SIGNED_CHAR,
    'short' : vtkConstants.VTK_SHORT,
    'unsigned short' : vtkConstants.VTK_UNSIGNED_SHORT,
    'int' : vtkConstants.VTK_INT,
    'unsigned int' : vtkConstants.VTK_UNSIGNED_INT,
    'long' : vtkConstants.VTK_LONG,
    'unsigned long' : vtkConstants.VTK_UNSIGNED_LONG,
    'long long' : vtkConstants.VTK_LONG_LONG,
    'unsigned long long' : vtkConstants.VTK_UNSIGNED_LONG_LONG,
    '__int64' : vtkConstants.VTK___INT64,
    'unsigned __int64' : vtkConstants.VTK_UNSIGNED___INT64,
    'float' : vtkConstants.VTK_FLOAT,
    'double' : vtkConstants.VTK_DOUBLE,
    'string' : vtkConstants.VTK_STRING,
    'unicode string' : vtkConstants.VTK_UNICODE_STRING,
}

_variant_method_map = {
    'char' : 'ToChar',
    'unsigned char' : 'ToUnsignedChar',
    'signed char' : 'ToSignedChar',
    'short' : 'ToShort',
    'unsigned short' : 'ToUnsignedShort',
    'int' : 'ToInt',
    'unsigned int' : 'ToUnsignedInt',
    'long' : 'ToLong',
    'unsigned long' : 'ToUnsignedLong',
    'long long' : 'ToLongLong',
    'unsigned long long' : 'ToUnsignedLongLong',
    '__int64' : 'To__Int64',
    'unsigned __int64' : 'ToUnsigned__Int64',
    'float' : 'ToFloat',
    'double' : 'ToDouble',
    'string' : 'ToString',
    'unicode string' : 'ToUnicodeString',
}

_variant_check_map = {
    'char' : 'IsChar',
    'unsigned char' : 'IsUnsignedChar',
    'signed char' : 'IsSignedChar',
    'short' : 'IsShort',
    'unsigned short' : 'IsUnsignedShort',
    'int' : 'IsInt',
    'unsigned int' : 'IsUnsignedInt',
    'long' : 'IsLong',
    'unsigned long' : 'IsUnsignedLong',
    'long long' : 'IsLongLong',
    'unsigned long long' : 'IsUnsignedLongLong',
    '__int64' : 'Is__Int64',
    'unsigned __int64' : 'IsUnsigned__Int64',
    'float' : 'IsFloat',
    'double' : 'IsDouble',
    'string' : 'IsString',
    'unicode string' : 'IsUnicodeString',
}

def vtkVariantCreate(v, t):
    """
    Create a vtkVariant of the specified type, where the type is in the
    following format: 'int', 'unsigned int', etc. for numeric types,
    'string', and 'unicode string'.
    """
    try:
        i = _variant_type_map[t]
    except KeyError:
        raise ValueError, "second parameter must be 'int', 'string', etc."
    return vtk.vtkVariant(v, i)

def vtkVariantExtract(v, t=None):
    """
    Extract the specified value type from the vtkVariant, where the type is
    int the following format:    'int', 'unsigned int', etc. for numerics,
    'string', and 'unicode string'.
    """
    if t == None:
        if v.IsVTKObject():
             raise ValueError, "use ToVTKObject() instead"
        t = v.GetTypeAsString()
    try:
        i = _variant_type_map[t]
    except KeyError:
        raise ValueError, "second parameter must be 'int', 'string', etc."
    if getattr(v, _variant_check_map[t])():
        return getattr(v, _variant_method_map[t])()
    else:
        return None

def vtkVariantCast(v, t):
    """
    Cast the vtkVariant to the specified value type, where the type is
    int the following format:    'int', 'unsigned int', etc. for numerics,
    'string', and 'unicode string'.
    """
    try:
        i = _variant_type_map[t]
    except KeyError:
        raise ValueError, "second parameter must be 'int', 'string', etc."
    u = vtk.vtkVariant(v, i)
    if u.IsValid():
        return getattr(u, _variant_method_map[t])()
    else:
        return None
<commit_msg>ENH: vtkVariantStrictEquality(), vtkVariantStrictWeakOrder() for python<commit_after>"""
Utility functions to mimic the template support functions for vtkVariant
"""

import vtk
import vtkConstants

_variant_type_map = {
    'char' : vtkConstants.VTK_CHAR,
    'unsigned char' : vtkConstants.VTK_UNSIGNED_CHAR,
    'signed char' : vtkConstants.VTK_SIGNED_CHAR,
    'short' : vtkConstants.VTK_SHORT,
    'unsigned short' : vtkConstants.VTK_UNSIGNED_SHORT,
    'int' : vtkConstants.VTK_INT,
    'unsigned int' : vtkConstants.VTK_UNSIGNED_INT,
    'long' : vtkConstants.VTK_LONG,
    'unsigned long' : vtkConstants.VTK_UNSIGNED_LONG,
    'long long' : vtkConstants.VTK_LONG_LONG,
    'unsigned long long' : vtkConstants.VTK_UNSIGNED_LONG_LONG,
    '__int64' : vtkConstants.VTK___INT64,
    'unsigned __int64' : vtkConstants.VTK_UNSIGNED___INT64,
    'float' : vtkConstants.VTK_FLOAT,
    'double' : vtkConstants.VTK_DOUBLE,
    'string' : vtkConstants.VTK_STRING,
    'unicode string' : vtkConstants.VTK_UNICODE_STRING,
    'vtkObjectBase' : vtkConstants.VTK_OBJECT,
    'vtkObject' : vtkConstants.VTK_OBJECT,
}

_variant_method_map = {
    vtk.VTK_CHAR : 'ToChar',
    vtk.VTK_UNSIGNED_CHAR : 'ToUnsignedChar',
    vtk.VTK_SIGNED_CHAR : 'ToSignedChar',
    vtk.VTK_SHORT : 'ToShort',
    vtk.VTK_UNSIGNED_SHORT : 'ToUnsignedShort',
    vtk.VTK_INT : 'ToInt',
    vtk.VTK_UNSIGNED_INT : 'ToUnsignedInt',
    vtk.VTK_LONG : 'ToLong',
    vtk.VTK_UNSIGNED_LONG : 'ToUnsignedLong',
    vtk.VTK_LONG_LONG : 'ToLongLong',
    vtk.VTK_UNSIGNED_LONG_LONG : 'ToUnsignedLongLong',
    vtk.VTK___INT64 : 'To__Int64',
    vtk.VTK_UNSIGNED___INT64 : 'ToUnsigned__Int64',
    vtk.VTK_FLOAT : 'ToFloat',
    vtk.VTK_DOUBLE : 'ToDouble',
    vtk.VTK_STRING : 'ToString',
    vtk.VTK_UNICODE_STRING : 'ToUnicodeString',
    vtk.VTK_OBJECT : 'ToVTKObject',
}

_variant_check_map = {
    vtk.VTK_CHAR : 'IsChar',
    vtk.VTK_UNSIGNED_CHAR : 'IsUnsignedChar',
    vtk.VTK_SIGNED_CHAR : 'IsSignedChar',
    vtk.VTK_SHORT : 'IsShort',
    vtk.VTK_UNSIGNED_SHORT : 'IsUnsignedShort',
    vtk.VTK_INT : 'IsInt',
    vtk.VTK_UNSIGNED_INT : 'IsUnsignedInt',
    vtk.VTK_LONG : 'IsLong',
    vtk.VTK_UNSIGNED_LONG : 'IsUnsignedLong',
    vtk.VTK_LONG_LONG : 'IsLongLong',
    vtk.VTK_UNSIGNED_LONG_LONG : 'IsUnsignedLongLong',
    vtk.VTK___INT64 : 'Is__Int64',
    vtk.VTK_UNSIGNED___INT64 : 'IsUnsigned__Int64',
    vtk.VTK_FLOAT : 'IsFloat',
    vtk.VTK_DOUBLE : 'IsDouble',
    vtk.VTK_STRING : 'IsString',
    vtk.VTK_UNICODE_STRING : 'IsUnicodeString',
    vtk.VTK_OBJECT : 'IsVTKObject',
}

def vtkVariantCreate(v, t):
    """
    Create a vtkVariant of the specified type, where the type is in the
    following format: 'int', 'unsigned int', etc. for numeric types,
    and 'string' or 'unicode string' for strings.  You can also use an
    integer VTK type constant for the type.
    """
    if not issubclass(type(t), int):
        i = _variant_type_map[t]

    return vtk.vtkVariant(v, i)

def vtkVariantExtract(v, t=None):
    """
    Extract the specified value type from the vtkVariant, where the type is
    in the following format: 'int', 'unsigned int', etc. for numeric types,
    and 'string' or 'unicode string' for strings.  You can also use an
    integer VTK type constant for the type.  Set the type to 'None" to
    extract the value in its native type.
    """
    if t == None:
        t = v.GetType()
    elif not issubclass(type(t), int):
        t = _variant_type_map[t]

    if getattr(v, _variant_check_map[t])():
        return getattr(v, _variant_method_map[t])()
    else:
        return None

def vtkVariantCast(v, t):
    """
    Cast the vtkVariant to the specified value type, where the type is
    in the following format: 'int', 'unsigned int', etc. for numeric types,
    and 'string' or 'unicode string' for strings.  You can also use an
    integer VTK type constant for the type.
    """
    if not issubclass(type(t), int):
        t = _variant_type_map[t]

    v = vtk.vtkVariant(v, i)

    if v.IsValid():
        return getattr(v, _variant_method_map[t])()
    else:
        return None

def vtkVariantStrictWeakOrder(s1, s2):
    """
    Compare variants by type first, and then by value.  The return values
    are -1, 0, 1 like the python cmp() method, for compatibility with the
    python list sort() method.  This is in contrast with the C++ version,
    which returns true or false.
    """
    t1 = s1.GetType()
    t2 = s2.GetType()

    # check based on type
    if t1 != t2:
        return cmp(t1,t2)

    v1 = s1.IsValid()
    v2 = s2.IsValid()

    # check based on validity
    if (not v1) and (not v2):
        return 0
    elif v1 != v2:
        return cmp(v1,v2)

    # extract and compare the values
    r1 = getattr(s1, _variant_method_map[t1])()
    r2 = getattr(s2, _variant_method_map[t2])()

    # compare vtk objects by classname
    if t1 == vtk.VTK_OBJECT:
        return cmp(r1.GetClassName(), r2.GetClassName())

    return cmp(r1, r2)

def vtkVariantStrictEquality(s1, s2):
    """
    Check two variants for strict equality of type and value.
    """
    t1 = s1.GetType()
    t2 = s2.GetType()

    # check based on type
    if t1 != t2:
        return False

    v1 = s1.IsValid()
    v2 = s2.IsValid()

    # check based on validity
    if (not v1) and (not v2):
        return True
    elif v1 != v2:
        return False

    # extract and compare the values
    r1 = getattr(s1, _variant_method_map[t1])()
    r2 = getattr(s2, _variant_method_map[t2])()

    return (r1 == r2)

def vtkVariantLessThan(s1, s2):
    """
    Return true if s1 < s2.  This isn't very useful in Python.
    """
    return (s1 < s2)

def vtkVariantEqual(s1, s2):
    """
    Return true if s1 == s2.  This isn't very useful in Python.
    """
    return (s1 == s2)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Implementation of an OpenFlow flow table

@author: Colin Scott (cs@cs.berkeley.edu)

"""
from collections import namedtuple
from libopenflow_01 import *
from pox.lib.revent import *

import time

# FlowTable Entries:
#   match - ofp_match (13-tuple)
#   counters - hash from name -> count. May be stale
#   actions - ordered list of ofp_action_*s to apply for matching packets
class TableEntry (object):
  """
  Models a flow table entry, with a match, actions, and options/flags/counters.
  Note: the current time can either be specified explicitely with the optional 'now' parameter or is taken from time.time()
  """

  def __init__(self,priority=OFP_DEFAULT_PRIORITY, cookie = 0, idle_timeout=0, hard_timeout=0, match=ofp_match(), actions=[], buffer_id=None, now=None):
    # overriding __new__ instead of init to make fields optional. There's probably a better way to do this.
    if now==None: now = time.time()
    self.counters = {
        'created': now,
        'last_touched': now,
        'bytes': 0,
        'packets': 0
    }
    self.priority = priority
    self.cookie = cookie
    self.idle_timeout = idle_timeout
    self.hard_timeout = hard_timeout
    self.match = match
    self.actions = actions
    self.buffer_id = buffer_id

  @staticmethod
  def from_flow_mod(flow_mod):
    priority = flow_mod.priority
    cookie = flow_mod.cookie
    match = flow_mod.match
    actions = flow_mod.actions
    buffer_id = flow_mod.buffer_id

    return TableEntry(priority, cookie, flow_mod.idle_timeout, flow_mod.hard_timeout, match, actions, buffer_id)

  def to_flow_mod(self, **kw):
    return ofp_flow_mod(priority = self.priority, cookie = self.cookie, match = self.match,
                        idle_timeout = self.idle_timeout, hard_timeout = self.hard_timeout,
                          actions = self.actions, buffer_id = self.buffer_id, **kw)

  def is_matched_by(self, match, priority = None, strict = False):
    """ return whether /this/ entry is matched by some other entry (e.g., for FLOW_MOD updates) """
    if(strict):
      return (self.match == match and self.priority == priority)
    else:
      return match.matches_with_wildcards(self.match)

  def touch_packet(self, byte_count, now=None):
    """ update the counters and expiry timer of this entry for a packet with a given byte count"""
    if now==None: now = time.time()
    self.counters["bytes"] += byte_count
    self.counters["packets"] += 1
    self.counters["last_touched"] = now

  def is_expired(self, now=None):
    """" return whether this flow entry is expired due to its idle timeout or hard timeout"""
    if now==None: now = time.time()
    return (self.hard_timeout > 0 and now - self.counters["created"] > self.hard_timeout) or (self.idle_timeout > 0 and now - self.counters["last_touched"] > self.idle_timeout)

  def __str__ (self):
    return self.__class__.__name__ + "\n  " + self.show()

  def __repr__(self):
    return "TableEntry("+self.show() + ")"

  def show(self):
       return "priority=%s, cookie=%x, idle_timeoout=%d, hard_timeout=%d, match=%s, actions=%s buffer_id=%d" % (
          self.priority, self.cookie, self.idle_timeout, self.hard_timeout, self.match, repr(self.actions), self.buffer_id)

class FlowTableModification (Event):
  def __init__(self, added=[], removed=[]):
    Event.__init__(self)
    self.added = added
    self.removed = removed

class FlowTable (EventMixin):
  _eventMixin_events = set([FlowTableModification])

  """
  General model of a flow table. Maintains an ordered list of flow entries, and finds
  matching entries for packets and other entries. Supports expiration of flows.
  """
  def __init__(self):
    EventMixin.__init__(self)
    # For now we represent the table as a multidimensional array.
    #
    # [ (cookie, match, counters, actions),
    #   (cookie, match, counters, actions),
    #    ...                        ]
    #
    # Implies O(N) lookup for now. TODO: fix
    self._table = []

  @property
  def entries(self):
    return self._table

  def __len__(self):
    return len(self._table)

  def add_entry(self, entry):
    if not isinstance(entry, TableEntry):
      raise "Not an Entry type"
    self._table.append(entry)

    # keep table sorted by descending priority, with exact matches always going first
    # note: python sort is stable
    self._table.sort(key=lambda(e): (e.priority if e.match.is_wildcarded else (1<<16) + 1), reverse=True)

    self.raiseEvent(FlowTableModification(added=[entry]))

  def remove_entry(self, entry):
    if not isinstance(entry, TableEntry):
      raise "Not an Entry type"
    self._table.remove(entry)
    self.raiseEvent(FlowTableModification(removed=[entry]))

  def entries_for_port(self, port_no):
    entries = []
    for entry in self._table:
      actions = entry.actions
      if len(actions) > 0:
        last_action = actions[-1]
        if type(last_action) == ofp_action_output:
          outgoing_port = last_action.port.port_no
          if outgoing_port == port_no:
            entries.apend(entry)
    return entries

  def matching_entries(self, match, priority=0, strict=False):
    return [ entry for entry in self._table if entry.is_matched_by(match, priority, strict) ]

  def expired_entries(self, now=None):
    return [ entry for entry in self._table if entry.is_expired(now) ]

  def remove_expired_entries(self, now=None):
    remove_flows = self.expired_entries(now)
    for entry in remove_flows:
        self._table.remove(entry)
    self.raiseEvent(FlowTableModification(removed=remove_flows))
    return remove_flows

  def remove_matching_entries(self, match, priority=0, strict=False):
    remove_flows = self.matching_entries(match, priority, strict)
    for entry in remove_flows:
        self._table.remove(entry)
    self.raiseEvent(FlowTableModification(removed=remove_flows))
    return remove_flows

  def entry_for_packet(self, packet, in_port=None):
    """ return the highest priority flow table entry that matches the given packet
    on the given in_port, or None if no matching entry is found. """
    packet_match = ofp_match.from_packet(packet, in_port)
    for entry in self._table:
      if entry.match.matches_with_wildcards(packet_match):
        return entry
    else:
      return None

class SwitchFlowTable(FlowTable):
  """
  Model a flow table for our switch implementation. Handles the behavior in response
  to the OF messages send to the switch
  """

  def process_flow_mod(self, flow_mod):
    """ Process a flow mod sent to the switch
    @return a tuple (added|modified|removed, [list of affected entries])
    """
    if(flow_mod.flags & OFPFF_CHECK_OVERLAP): raise("OFPFF_CHECK_OVERLAP checking not implemented")
    if(flow_mod.out_port != OFPP_NONE): raise("flow_mod outport checking not implemented")

    if flow_mod.command == OFPFC_ADD:
      # exactly matching entries have to be removed
      self.remove_matching_entries(flow_mod.match,flow_mod.priority, strict=True)
      return ("added", self.add_entry(TableEntry.from_flow_mod(flow_mod)))
    elif flow_mod.command == OFPFC_MODIFY or flow_mod.command == OFPFC_MODIFY_STRICT:
      is_strict = (flow_mod.command == OFPFC_MODIFY_STRICT)
      modified = []
      for entry in self._table:
        # update the actions field in the matching flows
        if(entry.is_matched_by(flow_mod.match, priority=flow_mod.priority, strict=is_strict)):
          entry.actions = flow_mod.actions
          modified.append(entry)
      if(len(modified) == 0):
        # if no matching entry is found, modify acts as add
        return ("added", self.add_entry(TableEntry.from_flow_mod(flow_mod)))
      else:
        return ("modified", modified)

    elif flow_mod.command == OFPFC_DELETE or flow_mod.command == OFPFC_DELETE_STRICT:
      is_strict = (flow_mod.command == OFPFC_DELETE_STRICT)
      return ("removed", self.remove_matching_entries(flow_mod.match, flow_mod.priority, strict=True))
    else:
      raise AttributeError("Command not yet implemented: %s" % flow_mod.command)

class NOMFlowTable(EventMixin):
  _eventMixin_events = set([FlowTableModification])
  """
  Model a flow table for use in our NOM model. Keep in sync with a switch through a
  connection.
  """
  ADD = OFPFC_ADD
  REMOVE = OFPFC_DELETE
  REMOVE_STRICT = OFPFC_DELETE_STRICT
  TIME_OUT = 2

  def __init__(self, switch):
    EventMixin.__init__(self)
    self.flow_table = FlowTable()
    self.switch = switch

    # a list of pending flow table entries : tuples (ADD|REMOVE, entry)
    self.pending = []

    # a map of pending barriers barrier_xid-> ([entry1,entry2])
    self.pending_barrier_to_ops = {}
    # a map of pending barriers per request entry -> (barrier_xid, time)
    self.pending_op_to_barrier = {}

    self.listenTo(switch)

  def install(self, entries=[]):
    """ asynchronously install entries in the flow table. will raise a FlowTableModification event when
        the change has been processed by the switch """
    self._mod(entries, NOMFlowTable.ADD)

  def remove_with_wildcards(self, entries=[]):
    """ asynchronously remove entries in the flow table. will raise a FlowTableModification event when
        the change has been processed by the switch """
    self._mod(entries, NOMFlowTable.REMOVE)

  def remove_strict(self, entries=[]):
    """ asynchronously remove entries in the flow table. will raise a FlowTableModification event when
        the change has been processed by the switch """
    self._mod(entries, NOMFlowTable.REMOVE_STRICT)

  @property
  def entries(self):
    return self.flow_table.entries

  @property
  def num_pending(self):
    return len(self.pending)

  def __len__(self):
    return len(self.flow_table)

  def _mod(self, entries, command):
    if isinstance(entries, TableEntry):
      entries = [ entries ]

    for entry in entries:
      if(command == NOMFlowTable.REMOVE):
        self.pending = filter(lambda(command, pentry): not (command == NOMFlowTable.ADD and entry.matches_with_wildcards(pentry)), self.pending)
      elif(command == NOMFlowTable.REMOVE_STRICT):
        self.pending = filter(lambda(command, pentry): not (command == NOMFlowTable.ADD and entry == pentry), self.pending)

      self.pending.append( (command, entry) )

    self._sync_pending()

  def _sync_pending(self, clear=False):
    if not self.switch.connected:
      return False

    # resync the switch
    if clear:
      self.pending_barrier_to_ops = {}
      self.pending_op_to_barrier = {}
      self.pending = filter(lambda(op): op[0] == NOMFlowTable.ADD, self.pending)

      self.switch.send(ofp_flow_mod(command=OFPFC_DELETE, match=ofp_match()))
      self.switch.send(ofp_barrier_request())

      todo = map(lambda(e): (NOMFlowTable.ADD, e), self.flow_table.entries) + self.pending
    else:
      todo = [ op for op in self.pending
          if op not in self.pending_op_to_barrier or (self.pending_op_to_barrier[op][1] + NOMFlowTable.TIME_OUT) < time.time ]

    for op in todo:
      fmod_xid = self.switch.xid_generator.next()
      flow_mod = op[1].to_flow_mod(xid=fmod_xid, command=op[0])
      self.switch.send(flow_mod)

    barrier_xid = self.switch.xid_generator.next()
    self.switch.send(ofp_barrier_request(xid=barrier_xid))
    now = time.time()
    self.pending_barrier_to_ops[barrier_xid] = todo

    for op in todo:
      self.pending_op_to_barrier[op] = (barrier_xid, now)

  def _handle_SwitchConnectionUp(self, event):
    # sync all_flows
    self._sync_pending(clear=True)

  def _handle_SwitchConnectionDown(self, event):
    # connection down. too bad for our unconfirmed entries
    self.pending_barrier_to_ops = {}
    self.pending_op_to_barrier = {}

  def _handle_BarrierIn(self, barrier):
    # yeah. barrier in. time to sync some of these flows
    if barrier.xid in self.pending_barrier_to_ops:
      added = []
      removed = []
      print "barrier in: pending for barrier: %d: %s" % (barrier.xid, self.pending_barrier_to_ops[barrier.xid])
      for op in self.pending_barrier_to_ops[barrier.xid]:
        (command, entry) = op
        if(command == NOMFlowTable.ADD):
          self.flow_table.add_entry(entry)
          added.append(entry)
        else:
          removed.extend(self.flow_table.remove_matching_entries(entry.match, entry.priority, strict=command == NOMFlowTable.REMOVE_STRICT))
        print "op: %s, pending: %s" % (op, self.pending)
        self.pending.remove(op)
      del self.pending_barrier_to_ops[barrier.xid]
      self.raiseEvent(FlowTableModification(added = added, removed=removed))
      return EventHalt
    else:
      return EventContinue

  def _handle_FlowRemoved(self, event):
    """ process a flow removed event -- remove the matching flow from the table. """
    flow_removed = event.ofp
    for entry in self.flow_table.entries:
      if(flow_removed.match == entry.match and flow_removed.priority == entry.priority):
        self.flow_table.remove_entry(entry)
        self.raiseEvent(FlowTableModification(removed=[entry]))
        return EventHalt
    return EventContinue
<commit_msg>openflow.flowtable: fix minor issue in show()<commit_after>#!/usr/bin/env python
"""
Implementation of an OpenFlow flow table

@author: Colin Scott (cs@cs.berkeley.edu)

"""
from collections import namedtuple
from libopenflow_01 import *
from pox.lib.revent import *

import time

# FlowTable Entries:
#   match - ofp_match (13-tuple)
#   counters - hash from name -> count. May be stale
#   actions - ordered list of ofp_action_*s to apply for matching packets
class TableEntry (object):
  """
  Models a flow table entry, with a match, actions, and options/flags/counters.
  Note: the current time can either be specified explicitely with the optional 'now' parameter or is taken from time.time()
  """

  def __init__(self,priority=OFP_DEFAULT_PRIORITY, cookie = 0, idle_timeout=0, hard_timeout=0, match=ofp_match(), actions=[], buffer_id=None, now=None):
    # overriding __new__ instead of init to make fields optional. There's probably a better way to do this.
    if now==None: now = time.time()
    self.counters = {
        'created': now,
        'last_touched': now,
        'bytes': 0,
        'packets': 0
    }
    self.priority = priority
    self.cookie = cookie
    self.idle_timeout = idle_timeout
    self.hard_timeout = hard_timeout
    self.match = match
    self.actions = actions
    self.buffer_id = buffer_id

  @staticmethod
  def from_flow_mod(flow_mod):
    priority = flow_mod.priority
    cookie = flow_mod.cookie
    match = flow_mod.match
    actions = flow_mod.actions
    buffer_id = flow_mod.buffer_id

    return TableEntry(priority, cookie, flow_mod.idle_timeout, flow_mod.hard_timeout, match, actions, buffer_id)

  def to_flow_mod(self, **kw):
    return ofp_flow_mod(priority = self.priority, cookie = self.cookie, match = self.match,
                        idle_timeout = self.idle_timeout, hard_timeout = self.hard_timeout,
                          actions = self.actions, buffer_id = self.buffer_id, **kw)

  def is_matched_by(self, match, priority = None, strict = False):
    """ return whether /this/ entry is matched by some other entry (e.g., for FLOW_MOD updates) """
    if(strict):
      return (self.match == match and self.priority == priority)
    else:
      return match.matches_with_wildcards(self.match)

  def touch_packet(self, byte_count, now=None):
    """ update the counters and expiry timer of this entry for a packet with a given byte count"""
    if now==None: now = time.time()
    self.counters["bytes"] += byte_count
    self.counters["packets"] += 1
    self.counters["last_touched"] = now

  def is_expired(self, now=None):
    """" return whether this flow entry is expired due to its idle timeout or hard timeout"""
    if now==None: now = time.time()
    return (self.hard_timeout > 0 and now - self.counters["created"] > self.hard_timeout) or (self.idle_timeout > 0 and now - self.counters["last_touched"] > self.idle_timeout)

  def __str__ (self):
    return self.__class__.__name__ + "\n  " + self.show()

  def __repr__(self):
    return "TableEntry("+self.show() + ")"

  def show(self):
       return "priority=%s, cookie=%x, idle_timeoout=%d, hard_timeout=%d, match=%s, actions=%s buffer_id=%s" % (
          self.priority, self.cookie, self.idle_timeout, self.hard_timeout, self.match, repr(self.actions), str(self.buffer_id))

class FlowTableModification (Event):
  def __init__(self, added=[], removed=[]):
    Event.__init__(self)
    self.added = added
    self.removed = removed

class FlowTable (EventMixin):
  _eventMixin_events = set([FlowTableModification])

  """
  General model of a flow table. Maintains an ordered list of flow entries, and finds
  matching entries for packets and other entries. Supports expiration of flows.
  """
  def __init__(self):
    EventMixin.__init__(self)
    # For now we represent the table as a multidimensional array.
    #
    # [ (cookie, match, counters, actions),
    #   (cookie, match, counters, actions),
    #    ...                        ]
    #
    # Implies O(N) lookup for now. TODO: fix
    self._table = []

  @property
  def entries(self):
    return self._table

  def __len__(self):
    return len(self._table)

  def add_entry(self, entry):
    if not isinstance(entry, TableEntry):
      raise "Not an Entry type"
    self._table.append(entry)

    # keep table sorted by descending priority, with exact matches always going first
    # note: python sort is stable
    self._table.sort(key=lambda(e): (e.priority if e.match.is_wildcarded else (1<<16) + 1), reverse=True)

    self.raiseEvent(FlowTableModification(added=[entry]))

  def remove_entry(self, entry):
    if not isinstance(entry, TableEntry):
      raise "Not an Entry type"
    self._table.remove(entry)
    self.raiseEvent(FlowTableModification(removed=[entry]))

  def entries_for_port(self, port_no):
    entries = []
    for entry in self._table:
      actions = entry.actions
      if len(actions) > 0:
        last_action = actions[-1]
        if type(last_action) == ofp_action_output:
          outgoing_port = last_action.port.port_no
          if outgoing_port == port_no:
            entries.apend(entry)
    return entries

  def matching_entries(self, match, priority=0, strict=False):
    return [ entry for entry in self._table if entry.is_matched_by(match, priority, strict) ]

  def expired_entries(self, now=None):
    return [ entry for entry in self._table if entry.is_expired(now) ]

  def remove_expired_entries(self, now=None):
    remove_flows = self.expired_entries(now)
    for entry in remove_flows:
        self._table.remove(entry)
    self.raiseEvent(FlowTableModification(removed=remove_flows))
    return remove_flows

  def remove_matching_entries(self, match, priority=0, strict=False):
    remove_flows = self.matching_entries(match, priority, strict)
    for entry in remove_flows:
        self._table.remove(entry)
    self.raiseEvent(FlowTableModification(removed=remove_flows))
    return remove_flows

  def entry_for_packet(self, packet, in_port=None):
    """ return the highest priority flow table entry that matches the given packet
    on the given in_port, or None if no matching entry is found. """
    packet_match = ofp_match.from_packet(packet, in_port)
    for entry in self._table:
      if entry.match.matches_with_wildcards(packet_match):
        return entry
    else:
      return None

class SwitchFlowTable(FlowTable):
  """
  Model a flow table for our switch implementation. Handles the behavior in response
  to the OF messages send to the switch
  """

  def process_flow_mod(self, flow_mod):
    """ Process a flow mod sent to the switch
    @return a tuple (added|modified|removed, [list of affected entries])
    """
    if(flow_mod.flags & OFPFF_CHECK_OVERLAP): raise("OFPFF_CHECK_OVERLAP checking not implemented")
    if(flow_mod.out_port != OFPP_NONE): raise("flow_mod outport checking not implemented")

    if flow_mod.command == OFPFC_ADD:
      # exactly matching entries have to be removed
      self.remove_matching_entries(flow_mod.match,flow_mod.priority, strict=True)
      return ("added", self.add_entry(TableEntry.from_flow_mod(flow_mod)))
    elif flow_mod.command == OFPFC_MODIFY or flow_mod.command == OFPFC_MODIFY_STRICT:
      is_strict = (flow_mod.command == OFPFC_MODIFY_STRICT)
      modified = []
      for entry in self._table:
        # update the actions field in the matching flows
        if(entry.is_matched_by(flow_mod.match, priority=flow_mod.priority, strict=is_strict)):
          entry.actions = flow_mod.actions
          modified.append(entry)
      if(len(modified) == 0):
        # if no matching entry is found, modify acts as add
        return ("added", self.add_entry(TableEntry.from_flow_mod(flow_mod)))
      else:
        return ("modified", modified)

    elif flow_mod.command == OFPFC_DELETE or flow_mod.command == OFPFC_DELETE_STRICT:
      is_strict = (flow_mod.command == OFPFC_DELETE_STRICT)
      return ("removed", self.remove_matching_entries(flow_mod.match, flow_mod.priority, strict=True))
    else:
      raise AttributeError("Command not yet implemented: %s" % flow_mod.command)

class NOMFlowTable(EventMixin):
  _eventMixin_events = set([FlowTableModification])
  """
  Model a flow table for use in our NOM model. Keep in sync with a switch through a
  connection.
  """
  ADD = OFPFC_ADD
  REMOVE = OFPFC_DELETE
  REMOVE_STRICT = OFPFC_DELETE_STRICT
  TIME_OUT = 2

  def __init__(self, switch):
    EventMixin.__init__(self)
    self.flow_table = FlowTable()
    self.switch = switch

    # a list of pending flow table entries : tuples (ADD|REMOVE, entry)
    self.pending = []

    # a map of pending barriers barrier_xid-> ([entry1,entry2])
    self.pending_barrier_to_ops = {}
    # a map of pending barriers per request entry -> (barrier_xid, time)
    self.pending_op_to_barrier = {}

    self.listenTo(switch)

  def install(self, entries=[]):
    """ asynchronously install entries in the flow table. will raise a FlowTableModification event when
        the change has been processed by the switch """
    self._mod(entries, NOMFlowTable.ADD)

  def remove_with_wildcards(self, entries=[]):
    """ asynchronously remove entries in the flow table. will raise a FlowTableModification event when
        the change has been processed by the switch """
    self._mod(entries, NOMFlowTable.REMOVE)

  def remove_strict(self, entries=[]):
    """ asynchronously remove entries in the flow table. will raise a FlowTableModification event when
        the change has been processed by the switch """
    self._mod(entries, NOMFlowTable.REMOVE_STRICT)

  @property
  def entries(self):
    return self.flow_table.entries

  @property
  def num_pending(self):
    return len(self.pending)

  def __len__(self):
    return len(self.flow_table)

  def _mod(self, entries, command):
    if isinstance(entries, TableEntry):
      entries = [ entries ]

    for entry in entries:
      if(command == NOMFlowTable.REMOVE):
        self.pending = filter(lambda(command, pentry): not (command == NOMFlowTable.ADD and entry.matches_with_wildcards(pentry)), self.pending)
      elif(command == NOMFlowTable.REMOVE_STRICT):
        self.pending = filter(lambda(command, pentry): not (command == NOMFlowTable.ADD and entry == pentry), self.pending)

      self.pending.append( (command, entry) )

    self._sync_pending()

  def _sync_pending(self, clear=False):
    if not self.switch.connected:
      return False

    # resync the switch
    if clear:
      self.pending_barrier_to_ops = {}
      self.pending_op_to_barrier = {}
      self.pending = filter(lambda(op): op[0] == NOMFlowTable.ADD, self.pending)

      self.switch.send(ofp_flow_mod(command=OFPFC_DELETE, match=ofp_match()))
      self.switch.send(ofp_barrier_request())

      todo = map(lambda(e): (NOMFlowTable.ADD, e), self.flow_table.entries) + self.pending
    else:
      todo = [ op for op in self.pending
          if op not in self.pending_op_to_barrier or (self.pending_op_to_barrier[op][1] + NOMFlowTable.TIME_OUT) < time.time ]

    for op in todo:
      fmod_xid = self.switch.xid_generator.next()
      flow_mod = op[1].to_flow_mod(xid=fmod_xid, command=op[0])
      self.switch.send(flow_mod)

    barrier_xid = self.switch.xid_generator.next()
    self.switch.send(ofp_barrier_request(xid=barrier_xid))
    now = time.time()
    self.pending_barrier_to_ops[barrier_xid] = todo

    for op in todo:
      self.pending_op_to_barrier[op] = (barrier_xid, now)

  def _handle_SwitchConnectionUp(self, event):
    # sync all_flows
    self._sync_pending(clear=True)

  def _handle_SwitchConnectionDown(self, event):
    # connection down. too bad for our unconfirmed entries
    self.pending_barrier_to_ops = {}
    self.pending_op_to_barrier = {}

  def _handle_BarrierIn(self, barrier):
    # yeah. barrier in. time to sync some of these flows
    if barrier.xid in self.pending_barrier_to_ops:
      added = []
      removed = []
      print "barrier in: pending for barrier: %d: %s" % (barrier.xid, self.pending_barrier_to_ops[barrier.xid])
      for op in self.pending_barrier_to_ops[barrier.xid]:
        (command, entry) = op
        if(command == NOMFlowTable.ADD):
          self.flow_table.add_entry(entry)
          added.append(entry)
        else:
          removed.extend(self.flow_table.remove_matching_entries(entry.match, entry.priority, strict=command == NOMFlowTable.REMOVE_STRICT))
        print "op: %s, pending: %s" % (op, self.pending)
        self.pending.remove(op)
      del self.pending_barrier_to_ops[barrier.xid]
      self.raiseEvent(FlowTableModification(added = added, removed=removed))
      return EventHalt
    else:
      return EventContinue

  def _handle_FlowRemoved(self, event):
    """ process a flow removed event -- remove the matching flow from the table. """
    flow_removed = event.ofp
    for entry in self.flow_table.entries:
      if(flow_removed.match == entry.match and flow_removed.priority == entry.priority):
        self.flow_table.remove_entry(entry)
        self.raiseEvent(FlowTableModification(removed=[entry]))
        return EventHalt
    return EventContinue
<|endoftext|>


<commit_before>from urllib.parse import quote_plus

import sublime
import sublime_plugin

from .base import BaseBlame
from .templates import blame_phantom_css, blame_phantom_html_template

# @todo Make a command to open the latest diff ("CommitDescription") for the current line in a single keystroke.
# @body Currently it takes a keystroke and then a mouse click on "Show"

class Blame(BaseBlame, sublime_plugin.TextCommand):

    # Overrides --------------------------------------------------

    def __init__(self, view):
        super().__init__(view)
        self.phantom_set = sublime.PhantomSet(view, "git-blame")

    def run(self, edit, prevving=False, fixed_row_num=None, sha_skip_list=[]):
        if not self.has_suitable_view():
            self.tell_user_to_save()
            return

        phantoms = []

        if prevving:
            # We'll be getting blame information for the line whose existing phantom's
            # [Prev] button was clicked, regardless of where the text cursor(s)
            # currently are.
            relevant_regions = [sublime.Region(self.view.text_point(fixed_row_num, 0))]
        else:
            # We'll be getting blame information for the lines where text cursor(s)
            # currently are.
            relevant_regions = self.view.sel()

        for region in relevant_regions:
            line_region = self.view.line(region)

            # When this Command is ran for a line with a phantom already visible, we
            # erase the phantom (i.e. toggle it). But if the reason this Command is
            # being ran is because the user is clicking the [Prev] button, just erasing
            # the existing phantom is not sufficient, because we need to then display
            # another phantom with updated content.
            if self.phantom_exists_for_region(line_region) and not prevving:
                continue

            row_num, _ = self.view.rowcol(region.begin())
            line_num = row_num + 1

            full_path = self.view.file_name()

            try:
                blame_output = self.get_blame_text(
                    full_path, line_num=line_num, sha_skip_list=sha_skip_list
                )
            except Exception as e:
                self.communicate_error(e)
                return

            blame = self.parse_line(blame_output)
            if not blame:
                self.communicate_error(
                    "Failed to parse anything for {0}. Has git's output format changed?".format(
                        self.__class__.__name__
                    )
                )
                return
            sha = blame["sha"]
            author = blame["author"]
            date = blame["date"]
            time = blame["time"]

            # The SHA output by `git blame` may have a leading caret to indicate that it
            # is a "boundary commit". That needs to be stripped before passing the SHA
            # back to git CLI commands for other purposes.
            sha_normalised = sha.strip("^")

            if sha_skip_list:
                recently_skipped_sha = sha_skip_list[-1]
                if sha_normalised == recently_skipped_sha:
                    sublime.message_dialog(
                        "No earlier commits affected line {0}".format(line_num)
                    )
                    return

            phantoms.append(
                sublime.Phantom(
                    line_region,
                    blame_phantom_html_template.format(
                        css=blame_phantom_css,
                        sha=sha,
                        sha_not_latest_indicator=" *" if sha_skip_list else "",
                        author=author,
                        date=date,
                        time=time,
                        qs_row_num_val=quote_plus(str(row_num)),
                        qs_sha_val=quote_plus(sha_normalised),
                        # Querystrings can contain the same key multiple times. We use that
                        # functionality to accumulate a list of SHAs to skip over when
                        # a [Prev] button has been clicked multiple times.
                        qs_skip_keyvals="&".join(
                            [
                                "skip={0}".format(quote_plus(skipee))
                                for skipee in sha_skip_list
                            ]
                        ),
                    ),
                    sublime.LAYOUT_BLOCK,
                    self.handle_phantom_button,
                )
            )

        self.phantom_set.update(phantoms)

    def _view(self):
        return self.view

    def handle_phantom_close_button(self):
        self.phantom_set.update([])

    def extra_cli_args(self, line_num, sha_skip_list):
        args = ["-L", "{0},{0}".format(line_num)]
        for skipped_sha in sha_skip_list:
            args.extend(["--ignore-rev", skipped_sha])
        return args

    # ------------------------------------------------------------

    def phantom_exists_for_region(self, region):
        return any(p.region == region for p in self.phantom_set.phantoms)

class BlameInsertCommitDescription(sublime_plugin.TextCommand):

    # Overrides --------------------------------------------------

    def run(self, edit, desc, scratch_view_name):
        view = self.view
        view.set_scratch(True)
        view.assign_syntax("Packages/Git Formats/Git Log.sublime-syntax")
        view.insert(edit, 0, desc)
        view.set_name(scratch_view_name)
        view.set_read_only(True)
<commit_msg>Revert "Use full-on Git Log syntax for [Show]ing a commit, not just the Diff syntax"<commit_after>from urllib.parse import quote_plus

import sublime
import sublime_plugin

from .base import BaseBlame
from .templates import blame_phantom_css, blame_phantom_html_template

# @todo Make a command to open the latest diff ("CommitDescription") for the current line in a single keystroke.
# @body Currently it takes a keystroke and then a mouse click on "Show"

class Blame(BaseBlame, sublime_plugin.TextCommand):

    # Overrides --------------------------------------------------

    def __init__(self, view):
        super().__init__(view)
        self.phantom_set = sublime.PhantomSet(view, "git-blame")

    def run(self, edit, prevving=False, fixed_row_num=None, sha_skip_list=[]):
        if not self.has_suitable_view():
            self.tell_user_to_save()
            return

        phantoms = []

        if prevving:
            # We'll be getting blame information for the line whose existing phantom's
            # [Prev] button was clicked, regardless of where the text cursor(s)
            # currently are.
            relevant_regions = [sublime.Region(self.view.text_point(fixed_row_num, 0))]
        else:
            # We'll be getting blame information for the lines where text cursor(s)
            # currently are.
            relevant_regions = self.view.sel()

        for region in relevant_regions:
            line_region = self.view.line(region)

            # When this Command is ran for a line with a phantom already visible, we
            # erase the phantom (i.e. toggle it). But if the reason this Command is
            # being ran is because the user is clicking the [Prev] button, just erasing
            # the existing phantom is not sufficient, because we need to then display
            # another phantom with updated content.
            if self.phantom_exists_for_region(line_region) and not prevving:
                continue

            row_num, _ = self.view.rowcol(region.begin())
            line_num = row_num + 1

            full_path = self.view.file_name()

            try:
                blame_output = self.get_blame_text(
                    full_path, line_num=line_num, sha_skip_list=sha_skip_list
                )
            except Exception as e:
                self.communicate_error(e)
                return

            blame = self.parse_line(blame_output)
            if not blame:
                self.communicate_error(
                    "Failed to parse anything for {0}. Has git's output format changed?".format(
                        self.__class__.__name__
                    )
                )
                return
            sha = blame["sha"]
            author = blame["author"]
            date = blame["date"]
            time = blame["time"]

            # The SHA output by `git blame` may have a leading caret to indicate that it
            # is a "boundary commit". That needs to be stripped before passing the SHA
            # back to git CLI commands for other purposes.
            sha_normalised = sha.strip("^")

            if sha_skip_list:
                recently_skipped_sha = sha_skip_list[-1]
                if sha_normalised == recently_skipped_sha:
                    sublime.message_dialog(
                        "No earlier commits affected line {0}".format(line_num)
                    )
                    return

            phantoms.append(
                sublime.Phantom(
                    line_region,
                    blame_phantom_html_template.format(
                        css=blame_phantom_css,
                        sha=sha,
                        sha_not_latest_indicator=" *" if sha_skip_list else "",
                        author=author,
                        date=date,
                        time=time,
                        qs_row_num_val=quote_plus(str(row_num)),
                        qs_sha_val=quote_plus(sha_normalised),
                        # Querystrings can contain the same key multiple times. We use that
                        # functionality to accumulate a list of SHAs to skip over when
                        # a [Prev] button has been clicked multiple times.
                        qs_skip_keyvals="&".join(
                            [
                                "skip={0}".format(quote_plus(skipee))
                                for skipee in sha_skip_list
                            ]
                        ),
                    ),
                    sublime.LAYOUT_BLOCK,
                    self.handle_phantom_button,
                )
            )

        self.phantom_set.update(phantoms)

    def _view(self):
        return self.view

    def handle_phantom_close_button(self):
        self.phantom_set.update([])

    def extra_cli_args(self, line_num, sha_skip_list):
        args = ["-L", "{0},{0}".format(line_num)]
        for skipped_sha in sha_skip_list:
            args.extend(["--ignore-rev", skipped_sha])
        return args

    # ------------------------------------------------------------

    def phantom_exists_for_region(self, region):
        return any(p.region == region for p in self.phantom_set.phantoms)

class BlameInsertCommitDescription(sublime_plugin.TextCommand):

    # Overrides --------------------------------------------------

    def run(self, edit, desc, scratch_view_name):
        view = self.view
        view.set_scratch(True)
        view.assign_syntax("Packages/Diff/Diff.sublime-syntax")
        view.insert(edit, 0, desc)
        view.set_name(scratch_view_name)
        view.set_read_only(True)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Various ligand binding models for use in assays.

"""

#=============================================================================================
# IMPORTS
#=============================================================================================

import numpy as np
import copy

import scipy.optimize
import scipy.integrate

from math import sqrt, exp, log

#=============================================================================================
# Physical constants
#=============================================================================================

Na = 6.02214179e23 # Avogadro's number (number/mol)
kB = Na * 1.3806504e-23 / 4184.0 # Boltzmann constant (kcal/mol/K)
C0 = 1.0 # standard concentration (M)

#=============================================================================================
# Binding models
#=============================================================================================

class BindingModel(object):
    """
    Abstract base class for reaction models.

    """

    def __init__(self):
        pass

#=============================================================================================
# Two-component binding model
#=============================================================================================

class TwoComponentBindingModel(BindingModel):
   """
   Simple two-component association.

   """

   @classmethod
   def equilibrium_concentrations(cls, DeltaG, Ptot, Ltot):
      """
      Compute equilibrium concentrations for simple two-component association.

      Parameters
      ----------
      DeltaG : float
         Reduced free energy of binding (in units of kT)
      Ptot : float or numpy array
         Total protein concentration summed over bound and unbound species, molarity.
      Ltot : float or numpy array
         Total ligand concentration summed over bound and unbound speciesl, molarity.

      Returns
      -------
      P : float or numpy array with same dimensions as Ptot
         Free protein concentration, molarity.
      L : float or numpy array with same dimensions as Ptot
         Free ligand concentration, molarity.
      PL : float or numpy array with same dimensions as Ptot
         Bound complex concentration, molarity.

      """
      # Handle only strictly positive elements---all others are set to zero as constants
      try:
          nonzero_indices = np.where(Ltot > 0)[0]
          zero_indices = np.where(Ltot <= 0)[0]
      except:
          nonzero_indices = range(size[0])
          zero_indices = []
      nnonzero = len(nonzero_indices)
      nzeros = len(zero_indices)

      # Original form:
      #Kd = np.exp(DeltaG)
      #sqrt_arg = (Ptot + Ltot + Kd)**2 - 4*Ptot*Ltot
      #sqrt_arg[sqrt_arg < 0.0] = 0.0
      #PL = 0.5 * ((Ptot + Ltot + Kd) - np.sqrt(sqrt_arg));  # complex concentration (M)

      # Numerically stable variant?
      PL = np.zeros(Ptot.shape)
      logP = np.log(Ptot[nonzero_indices])
      logL = np.log(Ltot[nonzero_indices])
      logPLK = np.logaddexp(np.logaddexp(logP, logL), DeltaG)
      PLK = np.exp(logPLK);
      sqrt_arg = 1.0 - np.exp(np.log(4.0) + logP + logL - 2*logPLK);
      sqrt_arg[sqrt_arg < 0.0] = 0.0 # ensure always positive
      PL[nonzero_indices] = 0.5 * PLK * (1.0 - np.sqrt(sqrt_arg));  # complex concentration (M)

      # Another variant
      #PL = 2*Ptot*Ltot / ((Ptot+Ltot+Kd) + np.sqrt((Ptot + Ltot + Kd)**2 - 4*Ptot*Ltot));  # complex concentration (M)
      # Yet another numerically stable variant?
      #logPLK = np.logaddexp(np.log(Ptot + Ltot),  DeltaG);
      #PLK = np.exp(logPLK);
      #xy = np.exp(np.log(Ptot) + np.log(Ltot) - 2.0*logPLK);
      #chi = 1.0 - 4.0 * xy;
      #chi[chi < 0.0] = 0.0 # prevent square roots of negative numbers
      #PL = 0.5 * PLK * (1 - np.sqrt(chi))

      # Ensure all concentrations are within limits, correcting cases where numerical issues cause problems.
      PL[PL < 0.0] = 0.0 # complex cannot have negative concentration
      #PL_max = np.minimum(Ptot, Ltot)
      #indices = np.where(PL > PL_max)
      #PL[indices] = PL_max[indices]

      # Compute remaining concentrations.
      P = Ptot - PL; # free protein concentration in sample cell after n injections (M)
      L = Ltot - PL; # free ligand concentration in sample cell after n injections (M)
      return [P, L, PL]

#=============================================================================================
# General robust competitive binding model
#=============================================================================================

class GeneralBindingModel(BindingModel):
   """
   General robust binding model for one protein and arbitrary number of competitive ligands.

   """

   @classmethod
   def equilibrium_concentrations(cls, reactions, conservation_equations, tol=1.0e-8):
      """
      Compute the equilibrium concentrations of each complex species for a general set of binding reactions.

      Parameters
      ----------
      reactions : list
          List of binding reactions.
          Each binding reaction is encoded as a tuple of (log equilibrium constant, dict of stoichiometry)
          Example: K_d = [RL] / ([R] [L]) becomes [ (-10, {'RL': -1, 'R' : +1, 'L' : +1}) ]
      conservation_equations : list
          List of mass conservation laws.
          Each mass conservation law is encoded as a tuple of (log total concentration, dict of stoichiometry of all species)
          Example: [R]tot = 10^-6 M = [RL] + [R] and [L]tot = 10^-6 M = [RL] + [L] becomes [ (-6, {'RL' : +1, 'R' : +1}), (-6, {'RL' : +1, 'L' : +1}) ]
      tol : float, optional, default=1.0e-8
          Solution tolerance.

      Returns
      -------
      log_concentrations : dict of str : float
          log_concentrations[species] is the log concentration of specified species

      Examples
      --------

      Simple 1:1 association

      >>> reactions = [ (-10, {'RL': -1, 'R' : +1, 'L' : +1}) ]
      >>> conservation_equations = [ (-6, {'RL' : +1, 'R' : +1}), (-6, {'RL' : +1, 'L' : +1}) ]
      >>> log_concentrations = GeneralBindingModel.equilibrium_concentrations(reactions, conservation_equations)

     Competitive 1:1 association

      >>> reactions = [ (-10, {'RL': -1, 'R' : +1, 'L' : +1}), (-5, {'RP' : -1, 'R' : +1, 'P' : +1}) ]
      >>> conservation_equations = [ (-6, {'RL' : +1, 'RP' : +1, 'R' : +1}), (-6, {'RL' : +1, 'L' : +1}), (-5, {'RP' : +1, 'P' : +1}) ]
      >>> log_concentrations = GeneralBindingModel.equilibrium_concentrations(reactions, conservation_equations)

      TODO
      ----
      * Can we allow the caller to specify initial conditions instead of conservation laws?

      """

      nreactions = len(reactions)
      nconservation = len(conservation_equations)
      nequations = nreactions + nconservation

      # Determine names of all species.
      all_species = set()
      for (log_equilibrium_constant, reaction) in reactions:
          for species in reaction.keys():
              all_species.add(species)
      all_species = list(all_species) # order is now fixed
      nspecies = len(all_species)

      # Construct function with appropriate roots.
      def ftarget(X):
          target = np.zeros([nequations], np.float64)
          jacobian = np.zeros([nequations, nspecies], np.float64)
          equation_index = 0
          # Reactions
          for (log_equilibrium_constant, reaction) in reactions:
              target[equation_index] = - log_equilibrium_constant
              for (species_index, species) in enumerate(all_species):
                  if species in reaction:
                      stoichiometry = reaction[species]
                      target[equation_index] += stoichiometry * X[species_index]
                      jacobian[equation_index][species_index] = stoichiometry
              equation_index += 1
          # Conservation of mass
          from scipy.misc import logsumexp
          for (log_total_concentration, conservation_equation) in conservation_equations:
              target[equation_index] = - log_total_concentration
              log_concentrations = list()
              for (species_index, species) in enumerate(all_species):
                  if species in conservation_equation:
                      stoichiometry = conservation_equation[species]
                      log_concentrations.append(X[species_index] + np.log(stoichiometry))
              log_concentrations = np.array(log_concentrations)
              logsum = logsumexp(log_concentrations)
              target[equation_index] += logsum
              for (species_index, species) in enumerate(all_species):
                  log_concentrations = list()
                  if species in conservation_equation:
                      stoichiometry = conservation_equation[species]
                      jacobian[equation_index, species_index] = stoichiometry * np.exp(X[species_index] - logsum)
              equation_index += 1

          return (target, jacobian)

      # Construct initial guess
      # We assume that all matter is equally spread out among all species via the conservation equations
      from scipy.misc import logsumexp
      LOG_ZERO = -100
      X = LOG_ZERO * np.ones([nspecies], np.float64)
      for (species_index, species) in enumerate(all_species):
          for (log_total_concentration, conservation_equation) in conservation_equations:
              log_total_stoichiometry = np.log(np.sum([stoichiometry for stoichiometry in conservation_equation.values()]))
              if species in conservation_equation:
                  stoichiometry = conservation_equation[species]
                  X[species_index] = logsumexp([X[species_index], log_total_concentration + np.log(stoichiometry) - log_total_stoichiometry])

      # Solve
      from scipy.optimize import root
      options = {'xtol' : tol}
      sol = root(ftarget, X, method='lm', jac=True, tol=tol, options=options)
      if (sol.success == False):
          msg  = "root-finder failed to converge:\n"
          msg += str(sol)
          raise Exception(msg)

      log_concentrations = { all_species[index] : sol.x[index] for index in range(nspecies) }
      return log_concentrations
<commit_msg>added simple competition model to bindingmodels.py<commit_after>#!/usr/bin/env python

"""
Various ligand binding models for use in assays.

"""

#=============================================================================================
# IMPORTS
#=============================================================================================

import numpy as np
import copy

import scipy.optimize
import scipy.integrate

from math import sqrt, exp, log

#=============================================================================================
# Physical constants
#=============================================================================================

Na = 6.02214179e23 # Avogadro's number (number/mol)
kB = Na * 1.3806504e-23 / 4184.0 # Boltzmann constant (kcal/mol/K)
C0 = 1.0 # standard concentration (M)

#=============================================================================================
# Binding models
#=============================================================================================

class BindingModel(object):
    """
    Abstract base class for reaction models.

    """

    def __init__(self):
        pass

#=============================================================================================
# Two-component binding model
#=============================================================================================

class TwoComponentBindingModel(BindingModel):
   """
   Simple two-component association.

   """

   @classmethod
   def equilibrium_concentrations(cls, DeltaG, Ptot, Ltot):
      """
      Compute equilibrium concentrations for simple two-component association.

      Parameters
      ----------
      DeltaG : float
         Reduced free energy of binding (in units of kT)
      Ptot : float or numpy array
         Total protein concentration summed over bound and unbound species, molarity.
      Ltot : float or numpy array
         Total ligand concentration summed over bound and unbound speciesl, molarity.

      Returns
      -------
      P : float or numpy array with same dimensions as Ptot
         Free protein concentration, molarity.
      L : float or numpy array with same dimensions as Ptot
         Free ligand concentration, molarity.
      PL : float or numpy array with same dimensions as Ptot
         Bound complex concentration, molarity.

      """
      # Handle only strictly positive elements---all others are set to zero as constants
      try:
          nonzero_indices = np.where(Ltot > 0)[0]
          zero_indices = np.where(Ltot <= 0)[0]
      except:
          nonzero_indices = range(size[0])
          zero_indices = []
      nnonzero = len(nonzero_indices)
      nzeros = len(zero_indices)

      # Original form:
      #Kd = np.exp(DeltaG)
      #sqrt_arg = (Ptot + Ltot + Kd)**2 - 4*Ptot*Ltot
      #sqrt_arg[sqrt_arg < 0.0] = 0.0
      #PL = 0.5 * ((Ptot + Ltot + Kd) - np.sqrt(sqrt_arg));  # complex concentration (M)

      # Numerically stable variant?
      PL = np.zeros(Ptot.shape)
      logP = np.log(Ptot[nonzero_indices])
      logL = np.log(Ltot[nonzero_indices])
      logPLK = np.logaddexp(np.logaddexp(logP, logL), DeltaG)
      PLK = np.exp(logPLK);
      sqrt_arg = 1.0 - np.exp(np.log(4.0) + logP + logL - 2*logPLK);
      sqrt_arg[sqrt_arg < 0.0] = 0.0 # ensure always positive
      PL[nonzero_indices] = 0.5 * PLK * (1.0 - np.sqrt(sqrt_arg));  # complex concentration (M)

      # Another variant
      #PL = 2*Ptot*Ltot / ((Ptot+Ltot+Kd) + np.sqrt((Ptot + Ltot + Kd)**2 - 4*Ptot*Ltot));  # complex concentration (M)
      # Yet another numerically stable variant?
      #logPLK = np.logaddexp(np.log(Ptot + Ltot),  DeltaG);
      #PLK = np.exp(logPLK);
      #xy = np.exp(np.log(Ptot) + np.log(Ltot) - 2.0*logPLK);
      #chi = 1.0 - 4.0 * xy;
      #chi[chi < 0.0] = 0.0 # prevent square roots of negative numbers
      #PL = 0.5 * PLK * (1 - np.sqrt(chi))

      # Ensure all concentrations are within limits, correcting cases where numerical issues cause problems.
      PL[PL < 0.0] = 0.0 # complex cannot have negative concentration
      #PL_max = np.minimum(Ptot, Ltot)
      #indices = np.where(PL > PL_max)
      #PL[indices] = PL_max[indices]

      # Compute remaining concentrations.
      P = Ptot - PL; # free protein concentration in sample cell after n injections (M)
      L = Ltot - PL; # free ligand concentration in sample cell after n injections (M)
      return [P, L, PL]

#=============================================================================================
# Analytical competitive binding model
#=============================================================================================

class CompetitionBindingModel(BindingModel):
     """
     Analytic solution for competitive binding model problem.
     As described in Wang, Z. X. An exact mathematical expression
     for describing competitive binding of two different ligands to
     a protein molecule. FEBS Lett. 1995, 360, 111−114.
     """

     @classmethod
     def equilibrium_concentrations(cls, Ptot, Ltot, DeltaG_L, Btot, DeltaG_B):
        """

        Compute equilibrium concentrations for analytical competition assay association.
        Parameters
        ----------
        Ptot : float or numpy array
          Total protein concentration summed over bound and unbound species, molarity.
        Ltot : float or numpy array
           Total fluorescent ligand concentration summed over bound and unbound speciesl, molarity.
        DeltaG_L : float
           Reduced free energy of binding of fluorescent L to P (in units of kT)
        Btot : float or numpy array
           Total competitive non-fluorescent ligand concentration summed over bound and unbound speciesl, molarity.
        DeltaG_B : float
           Reduced free energy of binding of non-fluorescent B to P (in units of kT)

        Returns
        -------
        P : float or numpy array with same dimensions as Ptot
           Free protein concentration, molarity.
        L : float or numpy array with same dimensions as Ptot
           Free fluorescent ligand concentration, molarity.
        PL : float or numpy array with same dimensions as Ptot
           Bound fluorescent ligand complex concentration, molarity.
        B : float or numpy array with same dimensions as Ptot
           Free non-fluorescent competitive ligand concentration, molarity.
        PB : float or numpy array with same dimensions as Ptot
           Bound non-fluorescent competitive ligand complex concentration, molarity.
        """

        # Handle only strictly positive elements---all others are set to zero as constants
        try:
            nonzero_indices = np.where(Ltot > 0)[0]
            zero_indices = np.where(Ltot <= 0)[0]
        except:
            nonzero_indices = range(size[0])
            zero_indices = []
        nnonzero = len(nonzero_indices)
        nzeros = len(zero_indices)

        # Original form:
        Kd_L = np.exp(DeltaG_L)
        Kd_B = np.exp(DeltaG_B)

        # P^3 + aP^2 + bP + c = 0
        a = Kd_L + Kd_B + Ltot + Btot - Ptot
        b = Kd_L*Kd_B + Kd_B*(Ltot-Ptot) + Kd_B*(Btot - Ptot)
        c = -Kd_L*Kd_B*Ptot

        # Subsitute P=u-a/3
        # u^3 - qu - r = 0 where
        q = (a**2)/3.0 - b
        r = (-2.0/27.0)*a**3 +(1.0/3.0)*a*b - c

        # Discriminant
        delta = (r**2)/4.0 -(q**3)/27.0

        # 3 roots. Physically meaningful root is u.
        theta = np.arccos((-2*(a**3)+9*a*b-27*c)/(2*np.sqrt((a**2-3*b)**3)))
        u = (2.0/3.0)*np.sqrt(a**2-3*b)*np.cos(theta/3.0)

        # Compute remaining concentrations.
        P = u - a/3.0           # free protein concentration in sample cell after n injections (M)
        PL = P*Ltot/(Kd_L + P)  # fluorescent ligand complex concentration (M)
        PB = P*Btot/(Kd_B + P)  # non-fluorescent ligand complex concentration (M)
        L = Ltot - PL           # free fluorescent ligand concentration in sample cell after n injections (M)
        B = Btot - PB           # free non-fluorescent ligand concentration in sample cell after n injections (M)

        return [P, L, PL, B, PB]

#=============================================================================================
# General robust competitive binding model
#=============================================================================================

class GeneralBindingModel(BindingModel):
   """
   General robust binding model for one protein and arbitrary number of competitive ligands.

   """

   @classmethod
   def equilibrium_concentrations(cls, reactions, conservation_equations, tol=1.0e-8):
      """
      Compute the equilibrium concentrations of each complex species for a general set of binding reactions.

      Parameters
      ----------
      reactions : list
          List of binding reactions.
          Each binding reaction is encoded as a tuple of (log equilibrium constant, dict of stoichiometry)
          Example: K_d = [RL] / ([R] [L]) becomes [ (-10, {'RL': -1, 'R' : +1, 'L' : +1}) ]
      conservation_equations : list
          List of mass conservation laws.
          Each mass conservation law is encoded as a tuple of (log total concentration, dict of stoichiometry of all species)
          Example: [R]tot = 10^-6 M = [RL] + [R] and [L]tot = 10^-6 M = [RL] + [L] becomes [ (-6, {'RL' : +1, 'R' : +1}), (-6, {'RL' : +1, 'L' : +1}) ]
      tol : float, optional, default=1.0e-8
          Solution tolerance.

      Returns
      -------
      log_concentrations : dict of str : float
          log_concentrations[species] is the log concentration of specified species

      Examples
      --------

      Simple 1:1 association

      >>> reactions = [ (-10, {'RL': -1, 'R' : +1, 'L' : +1}) ]
      >>> conservation_equations = [ (-6, {'RL' : +1, 'R' : +1}), (-6, {'RL' : +1, 'L' : +1}) ]
      >>> log_concentrations = GeneralBindingModel.equilibrium_concentrations(reactions, conservation_equations)

     Competitive 1:1 association

      >>> reactions = [ (-10, {'RL': -1, 'R' : +1, 'L' : +1}), (-5, {'RP' : -1, 'R' : +1, 'P' : +1}) ]
      >>> conservation_equations = [ (-6, {'RL' : +1, 'RP' : +1, 'R' : +1}), (-6, {'RL' : +1, 'L' : +1}), (-5, {'RP' : +1, 'P' : +1}) ]
      >>> log_concentrations = GeneralBindingModel.equilibrium_concentrations(reactions, conservation_equations)

      TODO
      ----
      * Can we allow the caller to specify initial conditions instead of conservation laws?

      """

      nreactions = len(reactions)
      nconservation = len(conservation_equations)
      nequations = nreactions + nconservation

      # Determine names of all species.
      all_species = set()
      for (log_equilibrium_constant, reaction) in reactions:
          for species in reaction.keys():
              all_species.add(species)
      all_species = list(all_species) # order is now fixed
      nspecies = len(all_species)

      # Construct function with appropriate roots.
      def ftarget(X):
          target = np.zeros([nequations], np.float64)
          jacobian = np.zeros([nequations, nspecies], np.float64)
          equation_index = 0
          # Reactions
          for (log_equilibrium_constant, reaction) in reactions:
              target[equation_index] = - log_equilibrium_constant
              for (species_index, species) in enumerate(all_species):
                  if species in reaction:
                      stoichiometry = reaction[species]
                      target[equation_index] += stoichiometry * X[species_index]
                      jacobian[equation_index][species_index] = stoichiometry
              equation_index += 1
          # Conservation of mass
          from scipy.misc import logsumexp
          for (log_total_concentration, conservation_equation) in conservation_equations:
              target[equation_index] = - log_total_concentration
              log_concentrations = list()
              for (species_index, species) in enumerate(all_species):
                  if species in conservation_equation:
                      stoichiometry = conservation_equation[species]
                      log_concentrations.append(X[species_index] + np.log(stoichiometry))
              log_concentrations = np.array(log_concentrations)
              logsum = logsumexp(log_concentrations)
              target[equation_index] += logsum
              for (species_index, species) in enumerate(all_species):
                  log_concentrations = list()
                  if species in conservation_equation:
                      stoichiometry = conservation_equation[species]
                      jacobian[equation_index, species_index] = stoichiometry * np.exp(X[species_index] - logsum)
              equation_index += 1

          return (target, jacobian)

      # Construct initial guess
      # We assume that all matter is equally spread out among all species via the conservation equations
      from scipy.misc import logsumexp
      LOG_ZERO = -100
      X = LOG_ZERO * np.ones([nspecies], np.float64)
      for (species_index, species) in enumerate(all_species):
          for (log_total_concentration, conservation_equation) in conservation_equations:
              log_total_stoichiometry = np.log(np.sum([stoichiometry for stoichiometry in conservation_equation.values()]))
              if species in conservation_equation:
                  stoichiometry = conservation_equation[species]
                  X[species_index] = logsumexp([X[species_index], log_total_concentration + np.log(stoichiometry) - log_total_stoichiometry])

      # Solve
      from scipy.optimize import root
      options = {'xtol' : tol}
      sol = root(ftarget, X, method='lm', jac=True, tol=tol, options=options)
      if (sol.success == False):
          msg  = "root-finder failed to converge:\n"
          msg += str(sol)
          raise Exception(msg)

      log_concentrations = { all_species[index] : sol.x[index] for index in range(nspecies) }
      return log_concentrations
<|endoftext|>