<commit_before># Feature extraction code

import os, time, subprocess, csv, shelve, re, pdb

class FeatureVectorAssembler():
    '''Assembles feature vectors from protein pair files, data source lists and gold standard protein pair lists.'''
    def __init__(self,sourcetab):
        #instantiate protein pair parsers
        # first parse the data source table
        # store the directory of the table and it's name
        self.sourcetabdir,self.tabfile = os.path.split(sourcetab)
        # open the table and parse for initialisation options
        c = csv.reader(open(sourcetab), delimiter="\t")
        # iterate over lines adding to list of protein pair parsers
        self.parserinitlist = []
        for line in c:
            #store the information in a dictionary
            d = {}
            d["data path"] = os.path.join(self.sourcetabdir,line[0])
            d["output path"] = os.path.join(self.sourcetabdir,line[1])
            #store options in a dictionary in the dictionary
            # if there are options
            d["options"] = {}
            if line[2] != "":
                options = line[2].split(";")
                for x in options:
                    #split each option to find out which option it is:
                    x = x.split("=")
                    #store it in the dictionary
                    # if there are invalid options this code WILL NOT DETECT THEM
                    d["options"][x[0]]= x[1]
                #update the script directory
                if "script" in d["options"].keys():
                    d["options"]["script"] = os.path.join(self.sourcetabdir,d["options"]["script"])
                #parse protindexes and validexes:
                if "protindexes" in d["options"].keys():
                    d["options"]["protindexes"] = tuple(int(v) for v in re.findall("[0-9]+", d["options"]["protindexes"]))
                if "valindexes" in d["options"].keys():
                    d["options"]["valindexes"] = tuple(int(v) for v in re.findall("[0-9]+", d["options"]["valindexes"]))
            #copy the dictionary into the list
            self.parserinitlist.append(d.copy())
        #then initialise each of these parsers and keep them in a list
        self.parserlist = []
        for parser in self.parserinitlist:
            self.parserlist.append(ProteinPairParser(parser["data path"],
                                                     parser["output path"],
                                                     **parser["options"]))
        return None

    def regenerate(self, force=False):
        '''Calls all known protein parsers and gets them to regenerate their output, if they have to.'''
        for parser in self.parserlist:
            parser.regenerate(force)
        return None

    def assemble(self, pairfile, outputfile, pairlabels=False, delim="\t", missinglabel="missing"):
        '''Assembles a file of feature vectors for each protein pair in a protein pair file supplied.

        Assumes the pairfile is tab delimited.'''
        # first parse the pairfile into a list of frozensets
        pairs = map(lambda l: frozenset(l),csv.reader(open(pairfile), delimiter="\t"))
        # open the file to put the feature vector in
        c = csv.writer(open(outputfile, "w"), delimiter=delim)
        #open all the databases and put them in a dictionary
        dbdict = {}
        for parser in self.parserinitlist:
            dbdict[parser["output path"]] = openpairshelf(parser["output path"])

        # then iterate through the pairs, querying all parser databases and building a list of rows
        rows = []
        for pair in pairs:
            row = []
            if pairlabels==True:
                lpair = list(pair)
                if len(lpair)==1:
                    lpair = lpair*2
                row = row+lpair
            for parser in self.parserinitlist:
                #if there are features there then append them to the row
                try:
                    row.append(dbdict[parser["output path"]][pair])
                except KeyError:
                    row.append([missinglabel])
            #copy the row into the list of rows
            rows.append(row[:])
        #check if lengths of missing labels is correct
        columns = zip(*rows)
        for col,colindex in zip(columns,range(len(columns))):

        #write all the rows to file
        c.writerows(rows)

        #close all the databases
        for parser in self.parserinitlist:
            dbdict[parser["output path"]].close()

        return None

class ProteinPairDB(shelve.DbfilenameShelf):
    '''A simple database for protein pairs using shelve.'''
    def __setitem__(self,key,value):
        #key will be frozenset so make it a list first
        key = list(key)
        #then make it a string
        if len(key) == 1:
            key = key[0]*2
        else:
            key = key[0] + "\t" + key[1]
        shelve.DbfilenameShelf.__setitem__(self,key,value)
        return None

    def __getitem__(self,key):
        #make two strings from the key
        key = list(key)
        if len(key) == 1:
            key1 = key[0]*2
            key1 = key[0]*2
        else:
            key1 = key[0] + "\t" + key[1]
            key2 = key[1] + "\t" + key[0]
        #try the first one
        try:
            value = shelve.DbfilenameShelf.__getitem__(self,key1)
        except KeyError:
            #couldn't find the first key, try the second
            value = shelve.DbfilenameShelf.__getitem__(self,key2)
            #if we don't find this one then error out as usual
        return value

class ProteinPairParser():
    '''Does simple parsing on data files to produce protein pair files with feature values'''
    def __init__(self,datadir,outdir,protindexes=(0,1),valindexes=[2],script=None,csvdelim="\t",ignoreheader=0):
        #first, store the initialisation
        self.datadir = datadir
        self.outdir = outdir
        self.protindexes=protindexes
        # had to hack this together from the list above
        # passing tuple in as default did not work
        self.valindexes=tuple(valindexes)
        self.script=script
        self.csvdelim=csvdelim
        self.ignoreheader=ignoreheader
        return None

    def regenerate(self, force=False):
        '''Regenerate the pair file from the data source
        if the data source is newer than the pair file'''
        # so first check the ages of both files
        datamtime = os.stat(self.datadir)[-2]
        if os.path.isfile(self.outdir):
            pairmtime = os.stat(self.outdir)[-2]
        else:
            #bit of a hack
            pairmtime = 0
        #if the data modification time is greater than output modification time
        if datamtime > pairmtime or force==True:
            # now regenerate the data file according to the options defined above:
            print "data file is newer than pair file"
            #if there's a script to run
            if self.script != None:
                #then execute the script
                retcode=subprocess.call("python2 %s"%self.script, shell=True)
            #first delete out of date file, if it's there
            if os.path.isfile(self.outdir):
                os.remove(self.outdir)
            #perform simple parsing to make a file of just protein pairs and the value we care about
            #and save these using shelve
            db = openpairshelf(self.outdir)
            #open the data file
            c = csv.reader(open(self.datadir), delimiter=self.csvdelim)
            #if the header should be ignored then ignore it
            if self.ignoreheader=="1":
                print "ignoring header"
                c.next()
            for line in c:
                #each line use the protein pair as a key
                #by formatting it as a frozenset
                pair = frozenset([line[self.protindexes[0]],line[self.protindexes[1]]])
                #and the value is indexed by valindexes
                db[pair] = []
                try:
                    for i in self.valindexes:
                        db[pair].append(line[i])
                except TypeError:
                    pdb.set_trace()
            db.close()
        return None

def openpairshelf(filename, flag='c', protocol=None, writeback=False):
    """Returns a ProteinPairDB object, with similar functionality to shelve.open()"""
    return ProteinPairDB(filename, flag, protocol, writeback)
<commit_msg>working version, tested#<commit_after># Feature extraction code

import os, time, subprocess, csv, shelve, re, sys

class FeatureVectorAssembler():
    '''Assembles feature vectors from protein pair files, data source lists and gold standard protein pair lists.'''
    def __init__(self,sourcetab, verbose=False):
        #instantiate protein pair parsers
        # first parse the data source table
        # store the directory of the table and it's name
        self.sourcetabdir,self.tabfile = os.path.split(sourcetab)
        if verbose:
            print "Using %s from top data directory %s."%(self.sourcetabdir,self.tabfile)
        # open the table and parse for initialisation options
        c = csv.reader(open(sourcetab), delimiter="\t")
        # iterate over lines adding to list of protein pair parsers
        if verbose:
            print "Reading data source table:"
        self.parserinitlist = []
        for line in c:
            #store the information in a dictionary
            d = {}
            d["data path"] = os.path.join(self.sourcetabdir,line[0])
            d["output path"] = os.path.join(self.sourcetabdir,line[1])
            #store options in a dictionary in the dictionary
            # if there are options
            d["options"] = {}
            if line[2] != "":
                options = line[2].split(";")
                for x in options:
                    #split each option to find out which option it is:
                    x = x.split("=")
                    #store it in the dictionary
                    # if there are invalid options this code WILL NOT DETECT THEM
                    d["options"][x[0]]= x[1]
                #update the script directory
                if "script" in d["options"].keys():
                    d["options"]["script"] = os.path.join(self.sourcetabdir,d["options"]["script"])
                #parse protindexes and validexes:
                if "protindexes" in d["options"].keys():
                    d["options"]["protindexes"] = tuple(int(v) for v in re.findall("[0-9]+", d["options"]["protindexes"]))
                if "valindexes" in d["options"].keys():
                    d["options"]["valindexes"] = tuple(int(v) for v in re.findall("[0-9]+", d["options"]["valindexes"]))
            #copy the dictionary into the list
            if verbose:
                print "\t"+"Data source: %s to be processed to %s"%(d["data path"],d["output path"])
            self.parserinitlist.append(d.copy())
        #then initialise each of these parsers and keep them in a list
        self.parserlist = []
        if verbose:
            print "Initialising parsers."
        for parser in self.parserinitlist:
            self.parserlist.append(ProteinPairParser(parser["data path"],
                                                     parser["output path"],
                                                     **parser["options"]))
        if verbose:
            print "Finished Initialisation."
        return None

    def regenerate(self, force=False, verbose=False):
        '''Calls all known protein parsers and gets them to regenerate their output, if they have to.'''
        if verbose:
            print "Regenerating parsers:"
        for parser in self.parserlist:
            if verbose:
                print "\t" + "parser %i"%self.parserlist.index(parser)
            parser.regenerate(force,verbose)
        return None

    def assemble(self, pairfile, outputfile, pairlabels=False, delim="\t", missinglabel="missing", verbose=False):
        '''Assembles a file of feature vectors for each protein pair in a protein pair file supplied.

        Assumes the pairfile is tab delimited.'''
        if verbose:
            print "Reading pairfile: %s"%pairfile
        # first parse the pairfile into a list of frozensets
        pairs = map(lambda l: frozenset(l),csv.reader(open(pairfile), delimiter="\t"))
        # open the file to put the feature vector in
        c = csv.writer(open(outputfile, "w"), delimiter=delim)
        #open all the databases and put them in a dictionary
        dbdict = {}
        if verbose:
            print "Opening databases:"
        for parser in self.parserinitlist:
            dbdict[parser["output path"]] = openpairshelf(parser["output path"])
            if verbose:
                print "\t"+"%s open"%parser["output path"]

        if verbose:
            print "Checking feature sizes:"
        #check size of feature in each file
        #will be important later
        featuresizes = {}
        for parser in self.parserinitlist:
            k = dbdict[parser["output path"]].keys()[0]
            featuresizes[parser["output path"]] = len(dbdict[parser["output path"]][k])
            if verbose:
                print "\t"+"Database %s contains features of size %i."%(parser["output path"],featuresizes[parser["output path"]])

        if verbose:
            sys.stdout.write("Writing feature vectors")
            lcount=0
            #counters for each database reporting numbers of missing values
            mcount={}
            for parser in self.parserinitlist:
                mcount[parser["output path"]] = 0
        # then iterate through the pairs, querying all parser databases and building a list of rows
        rows = []
        for pair in pairs:
            row = []
            if pairlabels==True:
                lpair = list(pair)
                if len(lpair)==1:
                    lpair = lpair*2
                row = row+lpair
            for parser in self.parserinitlist:
                #if there are features there then append them to the row
                try:
                    row = row + dbdict[parser["output path"]][pair]
                except KeyError:
                    row = row + [missinglabel]*featuresizes[parser["output path"]]
                    if verbose:
                        mcount[parser["output path"]] += 1
            c.writerow(row)
            if verbose:
                lcount = lcount+1
                if lcount%1000==0:
                    sys.stdout.write(".")
        if verbose:
            sys.stdout.write("\n")
            print "Wrote %i vectors."%lcount
            for parser in self.parserinitlist:
                print "Matched %.2f"%(100.0-100.0*mcount[parser["output path"]]/lcount) + "%" + " of protein pairs in %s to %s"%(pairfile,parser["output path"])
        #close all the databases
        for parser in self.parserinitlist:
            dbdict[parser["output path"]].close()

        return None

class ProteinPairDB(shelve.DbfilenameShelf):
    '''A simple database for protein pairs using shelve.'''
    def __setitem__(self,key,value):
        #key will be frozenset so make it a list first
        key = list(key)
        #then make it a string
        if len(key) == 1:
            key = key[0]*2
        else:
            key = key[0] + "\t" + key[1]
        shelve.DbfilenameShelf.__setitem__(self,key,value)
        return None

    def __getitem__(self,key):
        #make two strings from the key
        key = list(key)
        if len(key) == 1:
            key1 = key[0]*2
            key1 = key[0]*2
        else:
            key1 = key[0] + "\t" + key[1]
            key2 = key[1] + "\t" + key[0]
        #try the first one
        try:
            value = shelve.DbfilenameShelf.__getitem__(self,key1)
        except KeyError:
            #couldn't find the first key, try the second
            value = shelve.DbfilenameShelf.__getitem__(self,key2)
            #if we don't find this one then error out as usual
        return value

    def keys(self):
        #retrieve the string keys used by shelve
        ks = shelve.DbfilenameShelf.keys(self)
        #convert them to frozensets
        ks = map(lambda x: frozenset(x.split("\t")), ks)
        return ks

class ProteinPairParser():
    '''Does simple parsing on data files to produce protein pair files with feature values'''
    def __init__(self,datadir,outdir,protindexes=(0,1),valindexes=[2],script=None,csvdelim="\t",ignoreheader=0):
        #first, store the initialisation
        self.datadir = datadir
        self.outdir = outdir
        self.protindexes=protindexes
        # had to hack this together from the list above
        # passing tuple in as default did not work
        self.valindexes=tuple(valindexes)
        self.script=script
        self.csvdelim=csvdelim
        self.ignoreheader=ignoreheader
        return None

    def regenerate(self, force=False, verbose=False):
        '''Regenerate the pair file from the data source
        if the data source is newer than the pair file'''
        # so first check the ages of both files
        datamtime = os.stat(self.datadir)[-2]
        if os.path.isfile(self.outdir):
            pairmtime = os.stat(self.outdir)[-2]
        else:
            #bit of a hack
            pairmtime = 0
        #if the data modification time is greater than output modification time
        if datamtime > pairmtime or force==True:
            # now regenerate the data file according to the options defined above:
            if verbose and datamtime > pairmtime:
                if pairmtime == 0:
                    print "Database file not found, regenerating at %s from %s."%(self.outdir,self.datadir)
                else:
                    print "Data file %s is newer than processed database %s, regenerating."%(self.datadir,self.outdir)
            if verbose and force:
                print "Forcing regeneration of database %s from data file %s."%(self.outdir,self.datadir)
            #if there's a script to run
            if self.script != None:
                if verbose:
                    print "Executing script: %s."%self.script
                #then execute the script
                retcode=subprocess.call("python2 %s"%self.script, shell=True)
                if verbose:
                    print "Script returned: %s"%retcode
            #first delete out of date file, if it's there
            if os.path.isfile(self.outdir):
                os.remove(self.outdir)
            #perform simple parsing to make a file of just protein pairs and the value we care about
            #and save these using shelve
            db = openpairshelf(self.outdir)
            #open the data file
            c = csv.reader(open(self.datadir), delimiter=self.csvdelim)
            #if the header should be ignored then ignore it
            if self.ignoreheader=="1":
                if verbose:
                    print "Ignoring header."
                c.next()
            if verbose:
                sys.stdout.write("Filling database")
                lcount = 0
            for line in c:
                #each line use the protein pair as a key
                #by formatting it as a frozenset
                pair = frozenset([line[self.protindexes[0]],line[self.protindexes[1]]])
                #and the value is indexed by valindexes
                values = []
                for i in self.valindexes:
                    values.append(line[i])
                db[pair] = values[:]
                if verbose:
                    lcount = lcount+1
                    if lcount%1000==0:
                        sys.stdout.write(".")
            if verbose:
                sys.stdout.write("\n")
                print "Parsed %i lines."%lcount
            db.close()
        return None

def openpairshelf(filename, flag='c', protocol=None, writeback=False):
    """Returns a ProteinPairDB object, with similar functionality to shelve.open()"""
    return ProteinPairDB(filename, flag, protocol, writeback)
<|endoftext|>


<commit_before># coding: utf-8
import numpy as np

from scipy.signal import welch
from spectrum import pburg

from hrv.detrend import polynomial_detrend
from hrv.rri import RRi
from hrv.utils import (validate_rri, _interpolate_rri)

__all__ = ['time_domain', 'frequency_domain', 'non_linear']

@validate_rri
def time_domain(rri):
    """
    time_domain(rri)

    Calculate time-domain indices from an RRi series

    Parameters
    ----------
    rri : array_like
        sequence containing the RRi series

    Returns
    -------
    results : dict
        Dictionary containing the following time domain indices:
            - rmssd: root mean squared of the successive differences
            - sdnn: standard deviation of the RRi series
            - sdsd: standard deviation of the successive differences
            - nn50: number RRi successive differences greater than 50ms
            - pnn50: percentage of RRi successive differences greater than 50ms
            - mri: average value of the RRi series
            - mhr: average value of the heart rate calculated from the
                   RRi sries

    References
    ----------
    - Heart rate variability. (1996). Standards of measurement, physiological
      interpretation, and clinical use. Task Force of the European Society of
      Cardiology and the North American Society of Pacing and
      Electrophysiology. Eur Heart J, 17, 354-381.

    Examples
    --------
    >>> from hrv.classical import time_domain
    >>> rri = [1, 2, 3, 4, 5, 6]
    >>> time_domain(rri)
    {'rmssd': 1.0,
     'sdnn': 1.8708286933869707,
     'sdsd': 0.0,
     'nn50': 0,
     'pnn50': 0.0,
     'mrri': 3.5,
     'mhr': 24500.0}
    """
    # TODO: let user choose interval for pnn50 and nn50.
    diff_rri = np.diff(rri)
    rmssd = np.sqrt(np.mean(diff_rri ** 2))
    sdnn = np.std(rri, ddof=1)  # make it calculates N-1
    sdsd = np.std(diff_rri, ddof=1)
    nn50 = _nn50(rri)
    pnn50 = _pnn50(rri)
    mrri = np.mean(rri)
    mhr = np.mean(60 / (rri / 1000.0))

    return dict(zip(['rmssd', 'sdnn', 'sdsd', 'nn50', 'pnn50', 'mrri', 'mhr'],
                    [rmssd, sdnn, sdsd, nn50, pnn50, mrri, mhr]))

def _nn50(rri):
    return sum(abs(np.diff(rri)) > 50)

def _pnn50(rri):
    return _nn50(rri) / len(rri) * 100

# TODO: create nperseg, noverlap and detrend arguments
def frequency_domain(rri, time=None, fs=4.0, method='welch',
                     interp_method='cubic', detrend='constant',
                     vlf_band=(0, 0.04), lf_band=(0.04, 0.15),
                     hf_band=(0.15, 0.4), **kwargs):

    if isinstance(rri, RRi):
        time = rri.time if time is None else time
        detrend = detrend if not rri.detrended else False
        interp_method = interp_method if not rri.interpolated else None

    if interp_method is not None:
        rri = _interpolate_rri(rri, time, fs, interp_method)

    if method == 'welch':
        fxx, pxx = welch(x=rri, fs=fs, detrend=detrend, **kwargs)
    elif method == 'ar':
        if detrend:
            rri = polynomial_detrend(rri, degree=1)
        fxx, pxx = _calc_pburg_psd(rri=rri,  fs=fs, **kwargs)

    return _auc(fxx, pxx, vlf_band, lf_band, hf_band)

def _auc(fxx, pxx, vlf_band, lf_band, hf_band):
    vlf_indexes = np.logical_and(fxx >= vlf_band[0], fxx < vlf_band[1])
    lf_indexes = np.logical_and(fxx >= lf_band[0], fxx < lf_band[1])
    hf_indexes = np.logical_and(fxx >= hf_band[0], fxx < hf_band[1])

    vlf = np.trapz(y=pxx[vlf_indexes], x=fxx[vlf_indexes])
    lf = np.trapz(y=pxx[lf_indexes], x=fxx[lf_indexes])
    hf = np.trapz(y=pxx[hf_indexes], x=fxx[hf_indexes])
    total_power = vlf + lf + hf
    lf_hf = lf / hf
    lfnu = (lf / (total_power - vlf)) * 100
    hfnu = (hf / (total_power - vlf)) * 100

    return dict(zip(['total_power', 'vlf', 'lf', 'hf', 'lf_hf', 'lfnu',
                    'hfnu'], [total_power, vlf, lf, hf, lf_hf, lfnu, hfnu]))

def _calc_pburg_psd(rri, fs, order=16, nfft=None):
    burg = pburg(data=rri, order=order, NFFT=nfft, sampling=fs)
    burg.scale_by_freq = False
    burg()
    return np.array(burg.frequencies()), burg.psd

@validate_rri
def non_linear(rri):
    sd1, sd2 = _poincare(rri)
    return dict(zip(['sd1', 'sd2'], [sd1, sd2]))

def _poincare(rri):
    diff_rri = np.diff(rri)
    sd1 = np.sqrt(np.std(diff_rri, ddof=1) ** 2 * 0.5)
    sd2 = np.sqrt(2 * np.std(rri, ddof=1) ** 2 - 0.5 * np.std(diff_rri,
                                                              ddof=1) ** 2)
    return sd1, sd2
<commit_msg>Start frequency domain docstring<commit_after># coding: utf-8
import numpy as np

from scipy.signal import welch
from spectrum import pburg

from hrv.detrend import polynomial_detrend
from hrv.rri import RRi
from hrv.utils import (validate_rri, _interpolate_rri)

__all__ = ['time_domain', 'frequency_domain', 'non_linear']

@validate_rri
def time_domain(rri):
    """
    time_domain(rri)

    Calculate time-domain indices from an RRi series

    Parameters
    ----------
    rri : array_like
        sequence containing the RRi series

    Returns
    -------
    results : dict
        Dictionary containing the following time domain indices:
            - rmssd: root mean squared of the successive differences
            - sdnn: standard deviation of the RRi series
            - sdsd: standard deviation of the successive differences
            - nn50: number RRi successive differences greater than 50ms
            - pnn50: percentage of RRi successive differences greater than 50ms
            - mri: average value of the RRi series
            - mhr: average value of the heart rate calculated from the
                   RRi sries

    References
    ----------
    - Heart rate variability. (1996). Standards of measurement, physiological
      interpretation, and clinical use. Task Force of the European Society of
      Cardiology and the North American Society of Pacing and
      Electrophysiology. Eur Heart J, 17, 354-381.

    Examples
    --------
    >>> from hrv.classical import time_domain
    >>> rri = [1, 2, 3, 4, 5, 6]
    >>> time_domain(rri)
    {'rmssd': 1.0,
     'sdnn': 1.8708286933869707,
     'sdsd': 0.0,
     'nn50': 0,
     'pnn50': 0.0,
     'mrri': 3.5,
     'mhr': 24500.0}
    """
    # TODO: let user choose interval for pnn50 and nn50.
    diff_rri = np.diff(rri)
    rmssd = np.sqrt(np.mean(diff_rri ** 2))
    sdnn = np.std(rri, ddof=1)  # make it calculates N-1
    sdsd = np.std(diff_rri, ddof=1)
    nn50 = _nn50(rri)
    pnn50 = _pnn50(rri)
    mrri = np.mean(rri)
    mhr = np.mean(60 / (rri / 1000.0))

    return dict(zip(['rmssd', 'sdnn', 'sdsd', 'nn50', 'pnn50', 'mrri', 'mhr'],
                    [rmssd, sdnn, sdsd, nn50, pnn50, mrri, mhr]))

def _nn50(rri):
    return sum(abs(np.diff(rri)) > 50)

def _pnn50(rri):
    return _nn50(rri) / len(rri) * 100

# TODO: create nperseg, noverlap and detrend arguments
def frequency_domain(rri, time=None, fs=4.0, method='welch',
                     interp_method='cubic', detrend='constant',
                     vlf_band=(0, 0.04), lf_band=(0.04, 0.15),
                     hf_band=(0.15, 0.4), **kwargs):
    """
    Estimate the Power Spectral Density (PSD) of an RRi series and
    calculate the area under the curve (AUC) of the Very Low, Low, and High
    frequency bands. The PSD can be estimated using non-parametric
    (Welch's method) or parametric (Burg's method) approaches. The AUC
    is calculated using the trapezoidal method (numpy.trapz).

    Parameters
    ----------
    rri : array_like
        Sequence containing the RRi series
    time : array_like, optional
        Sequence containing the time associated with the RRi series.
        When not provided time is created from the cumulative sum of the
        values from the RRi series
    method : str, optional
        The method for Power Spectral Density estimation. 'welch' (default),
        'ar' (spectrum, see Cokelaer et al., 2017)
    interp_method : str {'cubic', 'linear'}, optional
        Interpolation funtion applied to the RRi series. If RRi series
        is already interpolated this step is skipped. 'cubic' (default),
        'linear'
    detrend : str or function, optional
       Detrend method applied to the RRi series. Defaults to 'constant'.
       If the rri is an RRiDetrend object this step is skipped.
       See scipy.signal.welch for more information
    vlf_band : tuple (inferior_bound, superior_bound)
        Frenquency interval of the Very Low frequency components of the
        estimated PSD. Defaults to (0, 0.04)
    lf_band : tuple (inferior_bound, superior_bound)
        Frenquency interval of the Low frequency components of the estimated
        PSD. Defaults to (0.04, 0.15)
    hf_band : tuple (inferior_bound, superior_bound)
        Frenquency interval of the High frequency components of the estimated
        PSD. Defaults to (0.15, 0.4)

    Returns
    -------
    results : dict
        Dictionary containing the following frequency domain indices:
            - total power: root mean squared of the successive differences
            - vlf: standard deviation of the RRi series
            - lf: standard deviation of the successive differences
            - hf: number RRi successive differences greater than 50ms
            - lf/hf: percentage of RRi successive differences greater than 50ms
            - lfnu: average value of the RRi series
            - hfnu: average value of the heart rate calculated from the
                    RRi sries

    References
    ----------
    - Heart rate variability. (1996). Standards of measurement, physiological
      interpretation, and clinical use. Task Force of the European Society of
      Cardiology and the North American Society of Pacing and
      Electrophysiology. Eur Heart J, 17, 354-381.
    - Cokelaer et al., (2017). ‘Spectrum’: Spectral Analysis in Python.
      Journal of Open Source Software, 2(18), 348, doi:10.21105/joss.00348

    Examples
    --------
    >>> from hrv.io import read_from_text
    >>> from hrv.classical import frequency_domain
    >>> rri = read_from_text('path/to/file.txt')
    >>> frequency_domain(rri)
    {'total_power': 3376.2107048316066,
     'vlf': 447.31367706350915,
     'lf': 1217.5000069660707,
     'hf': 1711.3970208020266,
     'lf_hf': 0.71140710902693,
     'lfnu': 41.56854936937952,
     'hfnu': 58.43145063062048}
    """

    if isinstance(rri, RRi):
        time = rri.time if time is None else time
        detrend = detrend if not rri.detrended else False
        interp_method = interp_method if not rri.interpolated else None

    if interp_method is not None:
        rri = _interpolate_rri(rri, time, fs, interp_method)

    if method == 'welch':
        fxx, pxx = welch(x=rri, fs=fs, detrend=detrend, **kwargs)
    elif method == 'ar':
        if detrend:
            rri = polynomial_detrend(rri, degree=1)
        fxx, pxx = _calc_pburg_psd(rri=rri,  fs=fs, **kwargs)

    return _auc(fxx, pxx, vlf_band, lf_band, hf_band)

def _auc(fxx, pxx, vlf_band, lf_band, hf_band):
    vlf_indexes = np.logical_and(fxx >= vlf_band[0], fxx < vlf_band[1])
    lf_indexes = np.logical_and(fxx >= lf_band[0], fxx < lf_band[1])
    hf_indexes = np.logical_and(fxx >= hf_band[0], fxx < hf_band[1])

    vlf = np.trapz(y=pxx[vlf_indexes], x=fxx[vlf_indexes])
    lf = np.trapz(y=pxx[lf_indexes], x=fxx[lf_indexes])
    hf = np.trapz(y=pxx[hf_indexes], x=fxx[hf_indexes])
    total_power = vlf + lf + hf
    lf_hf = lf / hf
    lfnu = (lf / (total_power - vlf)) * 100
    hfnu = (hf / (total_power - vlf)) * 100

    return dict(zip(['total_power', 'vlf', 'lf', 'hf', 'lf_hf', 'lfnu',
                    'hfnu'], [total_power, vlf, lf, hf, lf_hf, lfnu, hfnu]))

def _calc_pburg_psd(rri, fs, order=16, nfft=None):
    burg = pburg(data=rri, order=order, NFFT=nfft, sampling=fs)
    burg.scale_by_freq = False
    burg()
    return np.array(burg.frequencies()), burg.psd

@validate_rri
def non_linear(rri):
    sd1, sd2 = _poincare(rri)
    return dict(zip(['sd1', 'sd2'], [sd1, sd2]))

def _poincare(rri):
    diff_rri = np.diff(rri)
    sd1 = np.sqrt(np.std(diff_rri, ddof=1) ** 2 * 0.5)
    sd2 = np.sqrt(2 * np.std(rri, ddof=1) ** 2 - 0.5 * np.std(diff_rri,
                                                              ddof=1) ** 2)
    return sd1, sd2
<|endoftext|>


<commit_before>from CodernityDB.database import RecordNotFound
from flask import Flask, abort, jsonify, request
from converters import DateConverter
from db import get_db

db = None
pwsd = None

app = Flask(__name__)
app.url_map.converters['date'] = DateConverter

WILL_BE_STATES = ('yes', 'probably', 'maybe')

@app.route("/<date:date>", methods=['GET'])
def get_day(date):
    print date
    try:
        record = db.get('days', date)
    except RecordNotFound:
        return 'day', 404
    return jsonify(record)

@app.route("/<date:date>", methods=['PUT', 'POST'])
def update_day(date):
    if request.headers['password-key'] != pwsd:
        return 'wrong password', 403
    try:
        record = db.get('days', date, with_doc=True)
    except RecordNotFound:
        record = None
    query = request.json
    data = {k: v for k, v in query.items() if v and k in ('name', 'desc')}

    if record is None:
        if request.method == 'PUT':
            return 'day', 404
        data.update({
            'date': date,
            'users': {}
        })
        db.insert(data)
    else:
        record.update(data)
        db.update(record)
    return data, 201

@app.route("/<date:date>/user/<string:name>", methods=['GET'])
def get_user(date, name):
    try:
        record = db.get('days', date)
    except RecordNotFound:
        return 'day', 404
    if name not in record['users']:
        return 'user', 404

    return jsonify(record)

@app.route("/<date:date>/user/<string:name>", methods=['PUT', 'POST'])
def update_user(date, name):
    if request.headers['password-key'] != pwsd:
        return 'wrong password', 403
    try:
        record = db.get('days', date)
    except RecordNotFound:
        return 'day', 404

    query = request.json
    user = record['users'].get(name, {})
    if request.method == 'PUT' and not user:
        return 'user', 404

    if query.get('hour') not in xrange(1, 24 + 1):
        return 'hour is invalid', 400

    if query.get('minutes') not in xrange(1, 60 + 1):
        return 'minutes is invalid', 400

    if query.get('state') not in WILL_BE_STATES:
        return 'will_be state is invalid', 400

    db.update(record)
    return 201

@app.route("/<date:past>/<date:pre>", methods=['GET'])
def range_view(past, pre):
    return jsonify(db.get_many('days', past=past, end=pre))

if __name__ == "__main__":
    with open('pwsd') as f:
        pwsd = f.read()
    db = get_db('/tmp/firemark')
    app.run(debug=True)
<commit_msg>modify views<commit_after>from hsonliner.models import User, Event, Participant, Token
from hsonliner.converters import DateConverter

from flask import Flask, abort, jsonify, request
from datetime import datetime, timedelta
from functools import wraps

app = Flask(__name__)
app.url_map.converters['date'] = DateConverter

def session_scope():
    return app.config['SESSION']().scope()

app.session = lambda: app.config['SESSION'].scope()

WILL_BE_STATES = ('yes', 'probably', 'maybe')

def login_required(func):

    @wraps(func)
    def inner(**kwargs):
        token_hash = request.headers.get('token')

        if token_hash is None:
            return 'token not found', 403

        with session_scope() as session:
            token = session.query(Token).filter_by(hash=token_hash)
            user = session.query(User).filter_by(id=token.user)
            token.expire_date += timedelta(minutes=10)

        return func(user, **kwargs)

    return inner

@app.route("/login/<string:name>", methods=['POST'])
def login(name):
    pswd = User.generate_hash(request.data, app.config['SALT'])
    with session_scope() as session:
        user_id = (session.query(User.id)
                   .filter_by(name=name, pswd=pswd).scalar())

    if user_id is None:
        return 'user', 403

    token = Token(
        user_id=user_id,
        expire_data=datetime.now() + timedelta(minutes=10)
    )

    with session_scope() as session:
        session.add(token)

    return jsonify({'token': token.hash})

@app.route("/<date:date>", methods=['GET'])
def get_day(date):
    try:
        record = db.get('days', date)
    except RecordNotFound:
        return 'day', 404
    return jsonify(record)

@login_required
@app.route("/<date:date>", methods=['PUT', 'POST'])
def update_day(user, date):
    if request.headers['password-key'] != pwsd:
        return 'wrong password', 403
    try:
        record = db.get('days', date, with_doc=True)
    except RecordNotFound:
        record = None
    query = request.json
    data = {k: v for k, v in query.items() if v and k in ('name', 'desc')}

    if record is None:
        if request.method == 'PUT':
            return 'day', 404
        data.update({
            'date': date,
            'users': {}
        })
        db.insert(data)
    else:
        record.update(data)
        db.update(record)
    return data, 201

@login_required
@app.route("/<date:date>/user/<string:name>", methods=['GET'])
def get_user(date, name):
    try:
        record = db.get('days', date)
    except RecordNotFound:
        return 'day', 404
    if name not in record['users']:
        return 'user', 404

    return jsonify(record)

@login_required
@app.route("/<date:date>/user/<string:name>", methods=['PUT', 'POST'])
def update_user(date, name):
    if request.headers['password-key'] != pwsd:
        return 'wrong password', 403
    try:
        record = db.get('days', date)
    except RecordNotFound:
        return 'day', 404

    query = request.json
    user = record['users'].get(name, {})
    if request.method == 'PUT' and not user:
        return 'user', 404

    if query.get('hour') not in xrange(1, 24 + 1):
        return 'hour is invalid', 400

    if query.get('minutes') not in xrange(1, 60 + 1):
        return 'minutes is invalid', 400

    if query.get('state') not in WILL_BE_STATES:
        return 'will_be state is invalid', 400

    db.update(record)
    return 201

@login_required
@app.route("/<date:past>/<date:pre>", methods=['GET'])
def range_view(past, pre):
    return jsonify(db.get_many('days', past=past, end=pre))

<|endoftext|>


<commit_before>"""Open Drug Discovery Toolkit
==============================
Universal and easy to use resource for various drug discovery tasks,
ie docking, virutal screening, rescoring.

    Attributes
    ----------
    toolkit : module,
        Toolkits backend module, currenlty OpenBabel [ob] and RDKit [rdk].
        This setting is toolkit-wide, and sets given toolkit as default
"""
from __future__ import absolute_import
import os
import subprocess
import warnings
try:
    from oddt.toolkits import ob
except ImportError:
    ob = None
try:
    from oddt.toolkits import rdk
except ImportError:
    rdk = None

toolkit = None
if 'ODDT_TOOLKIT' in os.environ:
    if os.environ['ODDT_TOOLKIT'] == 'ob':
        if ob is None:
            warnings.warn('OpenBabel toolkit is forced by ODDT_TOOLKIT, '
                          'but can\'t be imported')
        toolkit = ob
    elif os.environ['ODDT_TOOLKIT'] == 'rdk':
        if rdk is None:
            warnings.warn('RDKit toolkit is forced by ODDT_TOOLKIT, '
                          'but can\'t be imported')
        toolkit = rdk
elif ob:
    toolkit = ob
elif rdk:
    toolkit = rdk
else:
    warnings.warn('No toolkit is present. Install OpenBabel or RDKit')

def get_version():
    home = os.path.dirname(__file__)
    git_v = None
    v = '0.3.2'
    if os.path.isdir(home + '/../.git'):
        try:
            git_v = str(subprocess.check_output(['git',
                                                 'describe',
                                                 '--tags'], cwd=home).strip())
        except subprocess.CalledProcessError:  # catch errors, eg. no git installed
            pass
    if git_v:
        v = git_v
    return v

__version__ = get_version()
__all__ = ['toolkit']

def random_seed(i):
    """
    Set global random seed for all underlying components.
    Use 'brute-force' approach, by setting undelying libraries' seeds.

    Parameters
    ----------
        i: int
            integer used as seed for random number generators
    """
    from numpy.random import seed as np_seed
    from random import seed as python_seed

    # python's random module
    python_seed(i)
    # numpy random module
    np_seed(i)
<commit_msg>Add more valid ODDT_TOOLKIT values and be strict about wrong ones<commit_after>"""Open Drug Discovery Toolkit
==============================
Universal and easy to use resource for various drug discovery tasks,
ie docking, virutal screening, rescoring.

    Attributes
    ----------
    toolkit : module,
        Toolkits backend module, currenlty OpenBabel [ob] and RDKit [rdk].
        This setting is toolkit-wide, and sets given toolkit as default
"""
from __future__ import absolute_import
import os
import subprocess
import warnings
try:
    from oddt.toolkits import ob
except ImportError:
    ob = None
try:
    from oddt.toolkits import rdk
except ImportError:
    rdk = None

toolkit = None
if 'ODDT_TOOLKIT' in os.environ:
    if os.environ['ODDT_TOOLKIT'] in ['ob', 'openbabel']:
        if ob is None:
            warnings.warn('OpenBabel toolkit is forced by ODDT_TOOLKIT, '
                          'but can\'t be imported')
        toolkit = ob
    elif os.environ['ODDT_TOOLKIT'] in ['rdk', 'rdkit']:
        if rdk is None:
            warnings.warn('RDKit toolkit is forced by ODDT_TOOLKIT, '
                          'but can\'t be imported')
        toolkit = rdk
    else:
        raise EnvironmentError('ODDT_TOOLKIT is set to invalid value: "%s". '
                               'Use one of ["ob", "openbabel"] for OpenBabel '
                               'or ["rdk", "rdkit"] for RDKit' % os.environ['ODDT_TOOLKIT'])

elif ob:
    toolkit = ob
elif rdk:
    toolkit = rdk
else:
    warnings.warn('No toolkit is present. Install OpenBabel or RDKit')

def get_version():
    home = os.path.dirname(__file__)
    git_v = None
    v = '0.3.2'
    if os.path.isdir(home + '/../.git'):
        try:
            git_v = str(subprocess.check_output(['git',
                                                 'describe',
                                                 '--tags'], cwd=home).strip())
        except subprocess.CalledProcessError:  # catch errors, eg. no git installed
            pass
    if git_v:
        v = git_v
    return v

__version__ = get_version()
__all__ = ['toolkit']

def random_seed(i):
    """
    Set global random seed for all underlying components.
    Use 'brute-force' approach, by setting undelying libraries' seeds.

    Parameters
    ----------
        i: int
            integer used as seed for random number generators
    """
    from numpy.random import seed as np_seed
    from random import seed as python_seed

    # python's random module
    python_seed(i)
    # numpy random module
    np_seed(i)
<|endoftext|>


<commit_before>__authors__ = "Tim Savage"
__author_email__ = "tim@savage.company"
__copyright__ = "Copyright (C) 2014 Tim Savage"
__version__ = "0.10b8"

from odin.fields import *  # noqa
from odin.fields.composite import *  # noqa
from odin.fields.virtual import *  # noqa
from odin.mapping import *  # noqa
from odin.resources import Resource  # noqa
from odin.adapters import ResourceAdapter  # noqa

# Disable logging if an explicit handler is not added
import logging
try:
    logging.getLogger('odin').addHandler(logging.NullHandler())
except AttributeError:
    pass  # Fallback for python 2.6
<commit_msg>PEP8 fix<commit_after>__authors__ = "Tim Savage"
__author_email__ = "tim@savage.company"
__copyright__ = "Copyright (C) 2014 Tim Savage"
__version__ = "0.10b8"

from odin.fields import *  # noqa
from odin.fields.composite import *  # noqa
from odin.fields.virtual import *  # noqa
from odin.mapping import *  # noqa
from odin.resources import Resource  # noqa
from odin.adapters import ResourceAdapter  # noqa

# Disable logging if an explicit handler is not added
try:
    import logging
    logging.getLogger('odin').addHandler(logging.NullHandler())
except AttributeError:
    pass  # Fallback for python 2.6
<|endoftext|>


<commit_before>import datetime
from django.db import models
try:
    import json
except ImportError:
    from django.utils import simplejson as json
from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _

JSON = 1
XML = 2
FORMAT_CHOICES = (
    (JSON, "JSON"),
    (XML, "XML"),
)

class ProviderRule(models.Model):
    name = models.CharField(_("name"), max_length=128, null=True, blank=True)
    regex = models.CharField(_("regex"), max_length=2000)
    endpoint = models.CharField(_("endpoint"), max_length=2000)
    format = models.IntegerField(_("format"), choices=FORMAT_CHOICES)

    def __unicode__(self):
        return self.name or self.endpoint

class StoredOEmbed(models.Model):
    match = models.TextField(_("match"))
    max_width = models.IntegerField(_("max width"))
    max_height = models.IntegerField(_("max height"))
    html = models.TextField(_("html"))
    json = models.TextField(_("json"))
    date_added = models.DateTimeField(
        _("date added"), default=now)

    class Meta:
        ordering = ('-max_width',) # larger ones take precedence
        verbose_name = u'Stored OEmbed'
        verbose_name_plural = u'Stored OEmbeds'

    def __unicode__(self):
        return self.match

    def get_json(self, name):
        """ Convenience for JSON properties; e.g. get_json('thumbnail_url') """
        return json.loads(self.json).get(name, None)
<commit_msg>Update imports (json in Py2.6+); removed unused datetime import<commit_after>import json
from django.db import models
from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _

JSON = 1
XML = 2
FORMAT_CHOICES = (
    (JSON, "JSON"),
    (XML, "XML"),
)

class ProviderRule(models.Model):
    name = models.CharField(_("name"), max_length=128, null=True, blank=True)
    regex = models.CharField(_("regex"), max_length=2000)
    endpoint = models.CharField(_("endpoint"), max_length=2000)
    format = models.IntegerField(_("format"), choices=FORMAT_CHOICES)

    def __unicode__(self):
        return self.name or self.endpoint

class StoredOEmbed(models.Model):
    match = models.TextField(_("match"))
    max_width = models.IntegerField(_("max width"))
    max_height = models.IntegerField(_("max height"))
    html = models.TextField(_("html"))
    json = models.TextField(_("json"))
    date_added = models.DateTimeField(
        _("date added"), default=now)

    class Meta:
        ordering = ('-max_width',) # larger ones take precedence
        verbose_name = u'Stored OEmbed'
        verbose_name_plural = u'Stored OEmbeds'

    def __unicode__(self):
        return self.match

    def get_json(self, name):
        """ Convenience for JSON properties; e.g. get_json('thumbnail_url') """
        return json.loads(self.json).get(name, None)
<|endoftext|>


<commit_before>#*********************************************************************
#
# (C) Copyright Broadcom Corporation 2013-2014
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
#*********************************************************************

#
# Implements Matches object
#

from ryu.lib import mac
from ofdpa.utils import Utils

class Matches():

    def __init__(self, *args, **kwargs):
        super(Matches, self).__init__(*args, **kwargs)

    @staticmethod
    def create_matches(dp, config):
        #
        #TODO: vlan mask
        matches = dp.ofproto_parser.OFPMatch()
        #
        # matches config is a list
        print "matches config: %s" % config
        #
        for key in config.keys():
            #
            #print "key: %s" % key
            #print "val: %s" % config[key]
            #
            if key[0] == '#':
                continue

            val = config[key]
            #
            if (key == "in_port"):
                matches.set_in_port(int(val))

            elif (key == "in_phy_port"):
                matches.set_in_phy_port(int(val))

            elif (key == "metadata"):
                matches.set_metadata(int(val))

            elif (key == "eth_dst"):
                matches.set_dl_dst(mac.haddr_to_bin(val))

            elif (key == "eth_dst_mask"):
                haddr = mac.haddr_to_bin(config["eth_dst"])
                mask = mac.haddr_to_bin(val)
                matches.set_dl_dst_masked(haddr,mask)

            elif (key == "eth_src"):
                matches.set_dl_src(mac.haddr_to_bin(val))

            elif (key == "eth_src_mask"):
                haddr = mac.haddr_to_bin(config["eth_src"])
                mask = mac.haddr_to_bin(val)
                matches.set_dl_src_masked(mac,mask)

            elif (key == "eth_type"):
                matches.set_dl_type(int(val,16))

            elif (key == "vlan_vid"):
                lst = val.split(',')
                if(len(lst) == 1):
                    vlan_vid = Utils.to_int(lst[0])
                    matches.set_vlan_vid(vlan_vid)
                elif(len(lst) == 2):
                    vlan_vid = Utils.to_int(lst[0])
                    vlan_vid_mask = Utils.to_int(lst[1])
                    matches.set_vlan_vid_masked(vlan_vid, vlan_vid_mask)
                else:
                    print key, ": only two values allowed, key ignored"

            elif (key == "vlan_pcp"):
                matches.set_vlan_pcp(int(val))

            elif (key == "ip_dscp"):
                matches.set_ip_dscp(int(val))

            elif (key == "ip_ecn"):
                raise Exception("Wrong match name:", key)

            elif (key == "ip_proto"):
                matches.set_ip_proto(int(val))

            elif (key == "ipv4_src"):
                matches.set_ipv4_src(Matches.ipv4_to_int(val))

            elif (key == "ipv4_src_mask"):
                addr = Matches.ipv4_to_int(config["ipv4_src"])
                mask = Matches.mask_ntob(int(val))
                matches.set_ipv4_src_masked(addr, mask)

            elif (key == "ipv4_dst"):
                matches.set_ipv4_dst(Matches.ipv4_to_int(val))

            elif (key == "ipv4_dst_mask"):
                addr = Matches.ipv4_to_int(config["ipv4_dst"])
                mask = Matches.mask_ntob(int(val))
                matches.set_ipv4_dst_masked(addr, mask)

            elif (key == "tcp_src"):
                matches.set_tcp_src(int(val))

            elif (key == "tcp_dst"):
                matches.set_tcp_dst(int(val))

            elif (key == "udp_src"):
                matches.set_udp_src(int(val))

            elif (key == "udp_dst"):
                matches.set_udp_dst(int(val))

            elif (key == "sctp_src"):
                 matches.set_sctp_src(int(val))

            elif (key == "sctp_dst"):
                 matches.set_sctp_dst(int(val))

            elif (key == "icmpv4_type"):
                 matches.set_icmpv4_type(int(val))

            elif (key == "icmpv4_code"):
                 matches.set_icmpv4_code(int(val))

            elif (key == "arp_op"):
                raise Exception("Wrong match name:", key)

            elif (key == "arp_spa"):
                raise Exception("Wrong match name:", key)

            elif (key == "arp_tpa"):
                raise Exception("Wrong match name:", key)

            elif (key == "arp_sha"):
                raise Exception("Wrong match name:", key)

            elif (key == "arp_tha"):
                raise Exception("Wrong match name:", key)

            elif (key == "ipv6_src"):
                matches.set_ipv6_src(Matches.ipv6_to_int(val))

            elif (key == "ipv6_src_mask"):
                ipv6_src_int = Matches.ipv6_to_int(config["ipv6_src"])
                mask_int = Matches.ipv6_to_int(mask)
                match.set_ipv6_src_masked(ipv6_src_int, mask_int)

            elif (key == "ipv6_dst"):
                matches.set_ipv6_dst(Matches.ipv6_to_int(val))

            elif (key == "ipv6_dst_mask"):
                ipv6_dst_int = Matches.ipv6_to_int(config["ipv6_dst"])
                mask_int = Matches.ipv6_to_int(mask)
                match.set_ipv6_dst_masked(ipv6_dst_int, mask_int)

            elif (key == "ipv6_flabel"):
                lst = val.split(',')
                flabel_int = Utils.to_int(lst[0])
                if(len(lst) == 1):
                    matches.set_ipv6_flabel(flabel_int)
                elif(len(lst) == 2):
                    flabel_mask_int = Utils.to_int(lst[1])
                    matches.set_ipv6_flabel_masked(flabel_int, flabel_mask_int)
                else:
                    print key, ": only two values allowed, key ignored"

            elif (key == "icmpv6_type"):
                 matches.set_icmpv6_type(int(val))

            elif (key == "icmpv6_code"):
                 matches.set_icmpv6_code(int(val))

            elif (key == "ipv6_nd_target"):
                raise Exception("Wrong match name:", key)

            elif (key == "ipv6_nd_sll"):
                raise Exception("Wrong match name:", key)

            elif (key == "ipv6_nd_tll"):
                raise Exception("Wrong match name:", key)

            elif (key == "mpls_label"):
                raise Exception("Wrong match name:", key)

            elif (key == "mpls_tc"):
                raise Exception("Wrong match name:", key)

            elif (key == "mpls_bos"):
                raise Exception("Wrong match name:", key)

            elif (key == "pbb_isid"):
                raise Exception("Wrong match name:", key)

            elif (key == "tunnel_id"):
                matches.set_tunnel_id(int(val))

            elif (key == "ipv6_exthdr"):
                raise Exception("Wrong match name:", key)

            else:
                raise Exception("Wrong match name:", key)

        #print "matches: %s" % matches
        return matches

    @staticmethod
    def ipv4_to_int(string):
        ip = string.split('.')
        assert len(ip) == 4
        i = 0
        for b in ip:
            b = int(b)
            i = (i << 8) | b
        return i

    @staticmethod
    def ipv6_to_int(string):
        ip = string.split(':')
        assert len(ip) == 8
        return [int(x, 16) for x in ip]

    @staticmethod
    def ipv4_text_to_int(ip_text):
        if ip_text == 0:
            return ip_text
        assert isinstance(ip_text, str)
        return struct.unpack('!I', addrconv.ipv4.text_to_bin(ip_text))[0]

    @staticmethod
    def mask_ntob(mask, err_msg=None):
        try:
            return (0xffffffff << (32 - mask)) & 0xffffffff
        except ValueError:
            msg = 'illegal netmask'
            if err_msg is not None:
                msg = '%s %s' % (err_msg, msg)
            raise ValueError(msg)

<commit_msg>change mask convert<commit_after>#*********************************************************************
#
# (C) Copyright Broadcom Corporation 2013-2014
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
#*********************************************************************

#
# Implements Matches object
#

from ryu.lib import mac
from ofdpa.utils import Utils

class Matches():

    def __init__(self, *args, **kwargs):
        super(Matches, self).__init__(*args, **kwargs)

    @staticmethod
    def create_matches(dp, config):
        #
        #TODO: vlan mask
        matches = dp.ofproto_parser.OFPMatch()
        #
        # matches config is a list
        print "matches config: %s" % config
        #
        for key in config.keys():
            #
            #print "key: %s" % key
            #print "val: %s" % config[key]
            #
            if key[0] == '#':
                continue

            val = config[key]
            #
            if (key == "in_port"):
                matches.set_in_port(int(val))

            elif (key == "in_phy_port"):
                matches.set_in_phy_port(int(val))

            elif (key == "metadata"):
                matches.set_metadata(int(val))

            elif (key == "eth_dst"):
                matches.set_dl_dst(mac.haddr_to_bin(val))

            elif (key == "eth_dst_mask"):
                haddr = mac.haddr_to_bin(config["eth_dst"])
                mask = mac.haddr_to_bin(val)
                matches.set_dl_dst_masked(haddr,mask)

            elif (key == "eth_src"):
                matches.set_dl_src(mac.haddr_to_bin(val))

            elif (key == "eth_src_mask"):
                haddr = mac.haddr_to_bin(config["eth_src"])
                mask = mac.haddr_to_bin(val)
                matches.set_dl_src_masked(mac,mask)

            elif (key == "eth_type"):
                matches.set_dl_type(int(val,16))

            elif (key == "vlan_vid"):
                lst = val.split(',')
                if(len(lst) == 1):
                    vlan_vid = Utils.to_int(lst[0])
                    matches.set_vlan_vid(vlan_vid)
                elif(len(lst) == 2):
                    vlan_vid = Utils.to_int(lst[0])
                    vlan_vid_mask = Utils.to_int(lst[1])
                    matches.set_vlan_vid_masked(vlan_vid, vlan_vid_mask)
                else:
                    print key, ": only two values allowed, key ignored"

            elif (key == "vlan_pcp"):
                matches.set_vlan_pcp(int(val))

            elif (key == "ip_dscp"):
                matches.set_ip_dscp(int(val))

            elif (key == "ip_ecn"):
                raise Exception("Wrong match name:", key)

            elif (key == "ip_proto"):
                matches.set_ip_proto(int(val))

            elif (key == "ipv4_src"):
                matches.set_ipv4_src(Matches.ipv4_to_int(val))

            elif (key == "ipv4_src_mask"):
                addr = Matches.ipv4_to_int(config["ipv4_src"])
                #mask = Matches.mask_ntob(int(val))
                mask = Matches.ipv4_to_int(val)
                matches.set_ipv4_src_masked(addr, mask)

            elif (key == "ipv4_dst"):
                matches.set_ipv4_dst(Matches.ipv4_to_int(val))

            elif (key == "ipv4_dst_mask"):
                addr = Matches.ipv4_to_int(config["ipv4_dst"])
                #mask = Matches.mask_ntob(int(val))
                mask = Matches.ipv4_to_int(val)
                matches.set_ipv4_dst_masked(addr, mask)

            elif (key == "tcp_src"):
                matches.set_tcp_src(int(val))

            elif (key == "tcp_dst"):
                matches.set_tcp_dst(int(val))

            elif (key == "udp_src"):
                matches.set_udp_src(int(val))

            elif (key == "udp_dst"):
                matches.set_udp_dst(int(val))

            elif (key == "sctp_src"):
                 matches.set_sctp_src(int(val))

            elif (key == "sctp_dst"):
                 matches.set_sctp_dst(int(val))

            elif (key == "icmpv4_type"):
                 matches.set_icmpv4_type(int(val))

            elif (key == "icmpv4_code"):
                 matches.set_icmpv4_code(int(val))

            elif (key == "arp_op"):
                raise Exception("Wrong match name:", key)

            elif (key == "arp_spa"):
                raise Exception("Wrong match name:", key)

            elif (key == "arp_tpa"):
                raise Exception("Wrong match name:", key)

            elif (key == "arp_sha"):
                raise Exception("Wrong match name:", key)

            elif (key == "arp_tha"):
                raise Exception("Wrong match name:", key)

            elif (key == "ipv6_src"):
                matches.set_ipv6_src(Matches.ipv6_to_int(val))

            elif (key == "ipv6_src_mask"):
                ipv6_src_int = Matches.ipv6_to_int(config["ipv6_src"])
                mask_int = Matches.ipv6_to_int(mask)
                match.set_ipv6_src_masked(ipv6_src_int, mask_int)

            elif (key == "ipv6_dst"):
                matches.set_ipv6_dst(Matches.ipv6_to_int(val))

            elif (key == "ipv6_dst_mask"):
                ipv6_dst_int = Matches.ipv6_to_int(config["ipv6_dst"])
                mask_int = Matches.ipv6_to_int(mask)
                match.set_ipv6_dst_masked(ipv6_dst_int, mask_int)

            elif (key == "ipv6_flabel"):
                lst = val.split(',')
                flabel_int = Utils.to_int(lst[0])
                if(len(lst) == 1):
                    matches.set_ipv6_flabel(flabel_int)
                elif(len(lst) == 2):
                    flabel_mask_int = Utils.to_int(lst[1])
                    matches.set_ipv6_flabel_masked(flabel_int, flabel_mask_int)
                else:
                    print key, ": only two values allowed, key ignored"

            elif (key == "icmpv6_type"):
                 matches.set_icmpv6_type(int(val))

            elif (key == "icmpv6_code"):
                 matches.set_icmpv6_code(int(val))

            elif (key == "ipv6_nd_target"):
                raise Exception("Wrong match name:", key)

            elif (key == "ipv6_nd_sll"):
                raise Exception("Wrong match name:", key)

            elif (key == "ipv6_nd_tll"):
                raise Exception("Wrong match name:", key)

            elif (key == "mpls_label"):
                raise Exception("Wrong match name:", key)

            elif (key == "mpls_tc"):
                raise Exception("Wrong match name:", key)

            elif (key == "mpls_bos"):
                raise Exception("Wrong match name:", key)

            elif (key == "pbb_isid"):
                raise Exception("Wrong match name:", key)

            elif (key == "tunnel_id"):
                matches.set_tunnel_id(int(val))

            elif (key == "ipv6_exthdr"):
                raise Exception("Wrong match name:", key)

            else:
                raise Exception("Wrong match name:", key)

        #print "matches: %s" % matches
        return matches

    @staticmethod
    def ipv4_to_int(string):
        ip = string.split('.')
        assert len(ip) == 4
        i = 0
        for b in ip:
            b = int(b)
            i = (i << 8) | b
        return i

    @staticmethod
    def ipv6_to_int(string):
        ip = string.split(':')
        assert len(ip) == 8
        return [int(x, 16) for x in ip]

    @staticmethod
    def ipv4_text_to_int(ip_text):
        if ip_text == 0:
            return ip_text
        assert isinstance(ip_text, str)
        return struct.unpack('!I', addrconv.ipv4.text_to_bin(ip_text))[0]

    @staticmethod
    def mask_ntob(mask, err_msg=None):
        try:
            return (0xffffffff << (32 - mask)) & 0xffffffff
        except ValueError:
            msg = 'illegal netmask'
            if err_msg is not None:
                msg = '%s %s' % (err_msg, msg)
            raise ValueError(msg)

<|endoftext|>


<commit_before>"""html2markdown converts an html string to markdown while preserving unsupported markup."""
# TODO:
#	implement standard
 (i.e. without attributes)
import bs4
from bs4 import BeautifulSoup
import re

import sys
if sys.version_info[0] > 2:
	unicode = str

_supportedTags = {
	# NOTE: will be ignored if they have unsupported attributes (cf. _supportedAttributes)
	'blockquote',
	'p',
	'a',
	'h1','h2','h3','h4','h5','h6',
	'strong','b',
	'em','i',
	'ul','ol','li',
	'br',
	'img',
	'pre','code',
	'hr'
}
_supportedAttributes = (
	'a href',
	'a title',
	'img alt',
	'img src',
	'img title',
)

_inlineTags = {
	# these can be mixed with markdown (when unprocessed)
	# block tags will be surrounded by newlines and be unprocessed inside
	# (unless supported tag + supported attribute[s])
	'a',
	'abbr',
	'acronym',
	'audio',
	'b',
	'bdi',
	'bdo',
	'big',
	#'br',
	'button',
	#'canvas',
	'cite',
	'code',
	'data',
	'datalist',
	'del',
	'dfn',
	'em',
	#'embed',
	'i',
	#'iframe',
	#'img',
	#'input',
	'ins',
	'kbd',
	'label',
	'map',
	'mark',
	'meter',
	#'noscript',
	'object',
	#'output',
	'picture',
	#'progress',
	'q',
	'ruby',
	's',
	'samp',
	#'script',
	'select',
	'slot',
	'small',
	'span',
	'strike',
	'strong',
	'sub',
	'sup',
	'svg',
	'template',
	'textarea',
	'time',
	'u',
	'tt',
	'var',
	#'video',
	'wbr',
}

def _supportedAttrs(tag):
	sAttrs = [attr.split(' ')[1] for attr in _supportedAttributes if attr.split(' ')[0]==tag.name]
	for attr in tag.attrs:
		if attr not in sAttrs:
			return False
	return True

def _recursivelyValid(tag):
	# not all tags require this property
	# requires:

asdf

	# does not:

asdf

	children = tag.find_all(recursive = False)
	for child in children:
		if not _recursivelyValid(child):
			return False
	if tag.name == '[document]':
		return True
	elif tag.name in _inlineTags:
		return True
	elif tag.name not in _supportedTags:
		return False
	if not _supportedAttrs(tag):
		return False
	return True

_escapeCharSequence = tuple(r'\`*_[]#')
_escapeCharRegexStr = '([{}])'.format(''.join(re.escape(c) for c in _escapeCharSequence))
_escapeCharSub = re.compile(_escapeCharRegexStr).sub

def _escapeCharacters(tag):
	"""non-recursively escape underlines and asterisks
	in the tag"""
	for i,c in enumerate(tag.contents):
		if type(c) != bs4.element.NavigableString:
			continue
		c.replace_with(_escapeCharSub(r'\\\1', c))

def _breakRemNewlines(tag):
	"""non-recursively break spaces and remove newlines in the tag"""
	for i,c in enumerate(tag.contents):
		if type(c) != bs4.element.NavigableString:
			continue
		c.replace_with(re.sub(r' {2,}', ' ', c).replace('\n',''))

def _markdownify(tag, _listType=None, _blockQuote=False, _listIndex=1):
	"""recursively converts a tag into markdown"""
	children = tag.find_all(recursive=False)

	if tag.name == '[document]':
		for child in children:
			_markdownify(child)
		return

	if tag.name not in _supportedTags or not _supportedAttrs(tag):
		if tag.name not in _inlineTags:
			tag.insert_before('\n\n')
			tag.insert_after('\n\n')
		else:
			_escapeCharacters(tag)
			for child in children:
				_markdownify(child)
		return
	if tag.name not in ('pre', 'code'):
		_escapeCharacters(tag)
		_breakRemNewlines(tag)
	if tag.name == 'p':
		if tag.string != None:
			if tag.string.strip() == u'':
				tag.string = u'\xa0'
				tag.unwrap()
				return
		if not _blockQuote:
			tag.insert_before('\n\n')
			tag.insert_after('\n\n')
		else:
			tag.insert_before('\n')
			tag.insert_after('\n')
		tag.unwrap()

		for child in children:
			_markdownify(child)
	elif tag.name == 'br':
		tag.string = '  \n'
		tag.unwrap()
	elif tag.name == 'img':
		alt = ''
		title = ''
		if tag.has_attr('alt'):
			alt = tag['alt']
		if tag.has_attr('title') and tag['title']:
			title = ' "%s"' % tag['title']
		tag.string = '![%s](%s%s)' % (alt, tag['src'], title)
		tag.unwrap()
	elif tag.name == 'hr':
		tag.string = '\n---\n'
		tag.unwrap()
	elif tag.name == 'pre':
		tag.insert_before('\n\n')
		tag.insert_after('\n\n')
		if tag.code:
			if not _supportedAttrs(tag.code):
				return
			for child in tag.code.find_all(recursive=False):
				if child.name != 'br':
					return
			# code block
			for br in tag.code.find_all('br'):
				br.string = '\n'
				br.unwrap()
			tag.code.unwrap()
			lines = unicode(tag).strip().split('\n')
			lines[0] = lines[0][5:]
			lines[-1] = lines[-1][:-6]
			if not lines[-1]:
				lines.pop()
			for i,line in enumerate(lines):
				line = line.replace(u'\xa0', ' ')
				lines[i] = '    %s' % line
			tag.replace_with(BeautifulSoup('\n'.join(lines), 'html.parser'))
		return
	elif tag.name == 'code':
		# inline code
		if children:
			return
		tag.insert_before('`` ')
		tag.insert_after(' ``')
		tag.unwrap()
	elif _recursivelyValid(tag):
		if tag.name == 'blockquote':
			# ! FIXME: hack
			tag.insert_before('<<
>>')
			tag.unwrap()
			for child in children:
				_markdownify(child, _blockQuote=True)
			return
		elif tag.name == 'a':
			# process children first
			for child in children:
				_markdownify(child)
			if not tag.has_attr('href'):
				return
			if tag.string != tag.get('href') or tag.has_attr('title'):
				title = ''
				if tag.has_attr('title'):
					title = ' "%s"' % tag['title']
				tag.string = '[%s](%s%s)' % (BeautifulSoup(unicode(tag), 'html.parser').string,
					tag.get('href', ''),
					title)
			else:
				# ! FIXME: hack
				tag.string = '<<<FLOATING LINK: %s>>>' % tag.string
			tag.unwrap()
			return
		elif tag.name == 'h1':
			tag.insert_before('\n\n# ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h2':
			tag.insert_before('\n\n## ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h3':
			tag.insert_before('\n\n### ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h4':
			tag.insert_before('\n\n#### ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h5':
			tag.insert_before('\n\n##### ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h6':
			tag.insert_before('\n\n###### ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name in ('ul', 'ol'):
			tag.insert_before('\n\n')
			tag.insert_after('\n\n')
			tag.unwrap()
			for i, child in enumerate(children):
				_markdownify(child, _listType=tag.name, _listIndex=i+1)
			return
		elif tag.name == 'li':
			if not _listType:
				#
 outside of list; ignore
				return
			if _listType == 'ul':
				tag.insert_before('*   ')
			else:
				tag.insert_before('%d.   ' % _listIndex)
			for child in children:
				_markdownify(child)
			for c in tag.contents:
				if type(c) != bs4.element.NavigableString:
					continue
				c.replace_with('\n    '.join(c.split('\n')))
			tag.insert_after('\n')
			tag.unwrap()
			return
		elif tag.name in ('strong','b'):
			tag.insert_before('__')
			tag.insert_after('__')
			tag.unwrap()
		elif tag.name in ('em','i'):
			tag.insert_before('_')
			tag.insert_after('_')
			tag.unwrap()
		for child in children:
			_markdownify(child)

def convert(html):
	"""converts an html string to markdown while preserving unsupported markup."""
	bs = BeautifulSoup(html, 'html.parser')
	_markdownify(bs)
	ret = unicode(bs).replace(u'\xa0', ' ')
	ret = re.sub(r'\n{3,}', r'\n\n', ret)
	# ! FIXME: hack
	ret = re.sub(r'<<<FLOATING LINK: (.+)>>>', r'<\1>', ret)
	# ! FIXME: hack
	sp = re.split(r'(<<<BLOCKQUOTE: .*?>>>)', ret, flags=re.DOTALL)
	for i,e in enumerate(sp):
		if e[:len('<<<BLOCKQUOTE:')] == '<<<BLOCKQUOTE:':
			sp[i] = '> ' + e[len('<<<BLOCKQUOTE:') : -len('>>>')]
			sp[i] = sp[i].replace('\n', '\n> ')
	ret = ''.join(sp)
	return ret.strip('\n')
<commit_msg>Add disclaimer.<commit_after># -*- coding:utf8 -*-
"""html2markdown converts an html string to markdown while preserving unsupported markup."""
#
# Copyright 2017-2018 David Lönnhager (dlon)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished
# to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

import bs4
from bs4 import BeautifulSoup
import re

import sys
if sys.version_info[0] > 2:
	unicode = str

_supportedTags = {
	# NOTE: will be ignored if they have unsupported attributes (cf. _supportedAttributes)
	'blockquote',
	'p',
	'a',
	'h1','h2','h3','h4','h5','h6',
	'strong','b',
	'em','i',
	'ul','ol','li',
	'br',
	'img',
	'pre','code',
	'hr'
}
_supportedAttributes = (
	'a href',
	'a title',
	'img alt',
	'img src',
	'img title',
)

_inlineTags = {
	# these can be mixed with markdown (when unprocessed)
	# block tags will be surrounded by newlines and be unprocessed inside
	# (unless supported tag + supported attribute[s])
	'a',
	'abbr',
	'acronym',
	'audio',
	'b',
	'bdi',
	'bdo',
	'big',
	#'br',
	'button',
	#'canvas',
	'cite',
	'code',
	'data',
	'datalist',
	'del',
	'dfn',
	'em',
	#'embed',
	'i',
	#'iframe',
	#'img',
	#'input',
	'ins',
	'kbd',
	'label',
	'map',
	'mark',
	'meter',
	#'noscript',
	'object',
	#'output',
	'picture',
	#'progress',
	'q',
	'ruby',
	's',
	'samp',
	#'script',
	'select',
	'slot',
	'small',
	'span',
	'strike',
	'strong',
	'sub',
	'sup',
	'svg',
	'template',
	'textarea',
	'time',
	'u',
	'tt',
	'var',
	#'video',
	'wbr',
}

def _supportedAttrs(tag):
	sAttrs = [attr.split(' ')[1] for attr in _supportedAttributes if attr.split(' ')[0]==tag.name]
	for attr in tag.attrs:
		if attr not in sAttrs:
			return False
	return True

def _recursivelyValid(tag):
	# not all tags require this property
	# requires:

asdf

	# does not:

asdf

	children = tag.find_all(recursive = False)
	for child in children:
		if not _recursivelyValid(child):
			return False
	if tag.name == '[document]':
		return True
	elif tag.name in _inlineTags:
		return True
	elif tag.name not in _supportedTags:
		return False
	if not _supportedAttrs(tag):
		return False
	return True

_escapeCharSequence = tuple(r'\`*_[]#')
_escapeCharRegexStr = '([{}])'.format(''.join(re.escape(c) for c in _escapeCharSequence))
_escapeCharSub = re.compile(_escapeCharRegexStr).sub

def _escapeCharacters(tag):
	"""non-recursively escape underlines and asterisks
	in the tag"""
	for i,c in enumerate(tag.contents):
		if type(c) != bs4.element.NavigableString:
			continue
		c.replace_with(_escapeCharSub(r'\\\1', c))

def _breakRemNewlines(tag):
	"""non-recursively break spaces and remove newlines in the tag"""
	for i,c in enumerate(tag.contents):
		if type(c) != bs4.element.NavigableString:
			continue
		c.replace_with(re.sub(r' {2,}', ' ', c).replace('\n',''))

def _markdownify(tag, _listType=None, _blockQuote=False, _listIndex=1):
	"""recursively converts a tag into markdown"""
	children = tag.find_all(recursive=False)

	if tag.name == '[document]':
		for child in children:
			_markdownify(child)
		return

	if tag.name not in _supportedTags or not _supportedAttrs(tag):
		if tag.name not in _inlineTags:
			tag.insert_before('\n\n')
			tag.insert_after('\n\n')
		else:
			_escapeCharacters(tag)
			for child in children:
				_markdownify(child)
		return
	if tag.name not in ('pre', 'code'):
		_escapeCharacters(tag)
		_breakRemNewlines(tag)
	if tag.name == 'p':
		if tag.string != None:
			if tag.string.strip() == u'':
				tag.string = u'\xa0'
				tag.unwrap()
				return
		if not _blockQuote:
			tag.insert_before('\n\n')
			tag.insert_after('\n\n')
		else:
			tag.insert_before('\n')
			tag.insert_after('\n')
		tag.unwrap()

		for child in children:
			_markdownify(child)
	elif tag.name == 'br':
		tag.string = '  \n'
		tag.unwrap()
	elif tag.name == 'img':
		alt = ''
		title = ''
		if tag.has_attr('alt'):
			alt = tag['alt']
		if tag.has_attr('title') and tag['title']:
			title = ' "%s"' % tag['title']
		tag.string = '![%s](%s%s)' % (alt, tag['src'], title)
		tag.unwrap()
	elif tag.name == 'hr':
		tag.string = '\n---\n'
		tag.unwrap()
	elif tag.name == 'pre':
		tag.insert_before('\n\n')
		tag.insert_after('\n\n')
		if tag.code:
			if not _supportedAttrs(tag.code):
				return
			for child in tag.code.find_all(recursive=False):
				if child.name != 'br':
					return
			# code block
			for br in tag.code.find_all('br'):
				br.string = '\n'
				br.unwrap()
			tag.code.unwrap()
			lines = unicode(tag).strip().split('\n')
			lines[0] = lines[0][5:]
			lines[-1] = lines[-1][:-6]
			if not lines[-1]:
				lines.pop()
			for i,line in enumerate(lines):
				line = line.replace(u'\xa0', ' ')
				lines[i] = '    %s' % line
			tag.replace_with(BeautifulSoup('\n'.join(lines), 'html.parser'))
		return
	elif tag.name == 'code':
		# inline code
		if children:
			return
		tag.insert_before('`` ')
		tag.insert_after(' ``')
		tag.unwrap()
	elif _recursivelyValid(tag):
		if tag.name == 'blockquote':
			# ! FIXME: hack
			tag.insert_before('<<
>>')
			tag.unwrap()
			for child in children:
				_markdownify(child, _blockQuote=True)
			return
		elif tag.name == 'a':
			# process children first
			for child in children:
				_markdownify(child)
			if not tag.has_attr('href'):
				return
			if tag.string != tag.get('href') or tag.has_attr('title'):
				title = ''
				if tag.has_attr('title'):
					title = ' "%s"' % tag['title']
				tag.string = '[%s](%s%s)' % (BeautifulSoup(unicode(tag), 'html.parser').string,
					tag.get('href', ''),
					title)
			else:
				# ! FIXME: hack
				tag.string = '<<<FLOATING LINK: %s>>>' % tag.string
			tag.unwrap()
			return
		elif tag.name == 'h1':
			tag.insert_before('\n\n# ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h2':
			tag.insert_before('\n\n## ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h3':
			tag.insert_before('\n\n### ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h4':
			tag.insert_before('\n\n#### ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h5':
			tag.insert_before('\n\n##### ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name == 'h6':
			tag.insert_before('\n\n###### ')
			tag.insert_after('\n\n')
			tag.unwrap()
		elif tag.name in ('ul', 'ol'):
			tag.insert_before('\n\n')
			tag.insert_after('\n\n')
			tag.unwrap()
			for i, child in enumerate(children):
				_markdownify(child, _listType=tag.name, _listIndex=i+1)
			return
		elif tag.name == 'li':
			if not _listType:
				#
 outside of list; ignore
				return
			if _listType == 'ul':
				tag.insert_before('*   ')
			else:
				tag.insert_before('%d.   ' % _listIndex)
			for child in children:
				_markdownify(child)
			for c in tag.contents:
				if type(c) != bs4.element.NavigableString:
					continue
				c.replace_with('\n    '.join(c.split('\n')))
			tag.insert_after('\n')
			tag.unwrap()
			return
		elif tag.name in ('strong','b'):
			tag.insert_before('__')
			tag.insert_after('__')
			tag.unwrap()
		elif tag.name in ('em','i'):
			tag.insert_before('_')
			tag.insert_after('_')
			tag.unwrap()
		for child in children:
			_markdownify(child)

def convert(html):
	"""converts an html string to markdown while preserving unsupported markup."""
	bs = BeautifulSoup(html, 'html.parser')
	_markdownify(bs)
	ret = unicode(bs).replace(u'\xa0', ' ')
	ret = re.sub(r'\n{3,}', r'\n\n', ret)
	# ! FIXME: hack
	ret = re.sub(r'<<<FLOATING LINK: (.+)>>>', r'<\1>', ret)
	# ! FIXME: hack
	sp = re.split(r'(<<<BLOCKQUOTE: .*?>>>)', ret, flags=re.DOTALL)
	for i,e in enumerate(sp):
		if e[:len('<<<BLOCKQUOTE:')] == '<<<BLOCKQUOTE:':
			sp[i] = '> ' + e[len('<<<BLOCKQUOTE:') : -len('>>>')]
			sp[i] = sp[i].replace('\n', '\n> ')
	ret = ''.join(sp)
	return ret.strip('\n')
<|endoftext|>


<commit_before>from celery import task
from oms_pds.pds.models import Profile
from bson import ObjectId
from pymongo import Connection
from django.conf import settings
import time
from datetime import date
import json
import pdb

from oms_pds.pds.models import Profile

"""the MONGODB_DATABASE_MULTIPDS setting is set by extract-user-middleware in cases where we need multiple PDS instances within one PDS service """

connection = Connection(
    host=getattr(settings, "MONGODB_HOST", None),
    port=getattr(settings, "MONGODB_PORT", None)
)

@task()
def add(x, y):
    print "Test"
    return x + y

@task()
def activityForLastHour():
    profiles = Profile.objects.all()

    print "Entered aggregation"

    aggregates = {}

    for profile in profiles:
        # Get the mongo store for the given user
        dbName = "User_" + str(profile.id)
        collection = connection[dbName]["funf"]

        # funf timestamps are represented as millis since epoch

        currentTime = time.mktime(time.gmtime())
        oneHourAgo = currentTime - (3600 * 24 * 30)

        #currentTime *= 1000
        #oneHourAgo *= 1000

        lowActivityIntervals = highActivityIntervals = totalIntervals = 0

        print dbName
        print profile.id, profile.uuid

        for data in collection.find({ "key": { "$regex" : "ActivityProbe$" }, "time": { "$gte" : oneHourAgo }}):
            dataValue = json.loads(data.value)
            print dataValue
            totalIntervals += dataValue["total_intervals"]
            lowActivityIntervals += dataValue["low_activity_intervals"]
            highActivityIntervals += dataValue["high_activity_intervals"]

        aggregates[profile.uuid] = { "total": totalIntervals, "low": lowActivityIntervals, "high": highActivityIntervals }

    return aggregates

@task()
def activityForToday():
    profiles = Profile.objects.all()

    print "Entered aggregation"

    aggregates = {}

    # Note: left off setting midnight to loop over hours until now...
    currentTime = time.mktime(time.gmtime())
    # Interesting way of getting midnight for the day of the current GM time.... is there a better way?
    midnight = time.mktime(date.fromtimestamp(currentTime).timetuple())

    # funf timestamps are represented as millis since epoch
    #currentTime *= 1000
    #midnight *= 1000

    for profile in profiles:
        # Get the mongo store for the given user
        dbName = "User_" + str(profile.id)
        collection = connection[dbName]["funf"]
        aggregates[profile.uuid] = {}

        print dbName
        print profile.id, profile.uuid
        pdb.set_trace()
        for offsetFromMidnight in range(int(midnight), int(currentTime), 3600):
            hour = (offsetFromMidnight - midnight) / 3600
            lowActivityIntervals = highActivityIntervals = totalIntervals = 0

            for data in collection.find({ "key": { "$regex" : "ActivityProbe$" }, "time": { "$gte" : offsetFromMidnight, "$lt": offsetFromMidnight + 3600 }}):
                #pdb.set_trace()
                dataValue = data["value"]
                print dataValue
                totalIntervals += dataValue["total_intervals"]
                lowActivityIntervals += dataValue["low_activity_intervals"]
                highActivityIntervals += dataValue["high_activity_intervals"]

            aggregates[profile.uuid][hour] = { "total": totalIntervals, "low": lowActivityIntervals, "high": highActivityIntervals }

    return aggregates
        <commit_msg>Cleaning up activity aggregation task<commit_after>from celery import task
from oms_pds.pds.models import Profile
from bson import ObjectId
from pymongo import Connection
from django.conf import settings
import time
from datetime import date
import json
import pdb

from oms_pds.pds.models import Profile

"""the MONGODB_DATABASE_MULTIPDS setting is set by extract-user-middleware in cases where we need multiple PDS instances within one PDS service """

connection = Connection(
    host=getattr(settings, "MONGODB_HOST", None),
    port=getattr(settings, "MONGODB_PORT", None)
)

@task()
def add(x, y):
    print "Test"
    return x + y

@task()
def activityForLastHour():
    profiles = Profile.objects.all()

    print "Entered aggregation"

    aggregates = {}

    for profile in profiles:
        # Get the mongo store for the given user
        dbName = "User_" + str(profile.id)
        collection = connection[dbName]["funf"]

        # funf timestamps are represented as millis since epoch

        currentTime = time.mktime(time.gmtime())
        oneHourAgo = currentTime - (3600 * 24 * 30)

        #currentTime *= 1000
        #oneHourAgo *= 1000

        lowActivityIntervals = highActivityIntervals = totalIntervals = 0

        print dbName
        print profile.id, profile.uuid

        for data in collection.find({ "key": { "$regex" : "ActivityProbe$" }, "time": { "$gte" : oneHourAgo }}):
            dataValue = json.loads(data.value)
            print dataValue
            totalIntervals += dataValue["total_intervals"]
            lowActivityIntervals += dataValue["low_activity_intervals"]
            highActivityIntervals += dataValue["high_activity_intervals"]

        aggregates[profile.uuid] = { "total": totalIntervals, "low": lowActivityIntervals, "high": highActivityIntervals }

    return aggregates

def activityForTimeRange(collection, start, end):
    lowActivityIntervals = highActivityIntervals = totalIntervals = 0

    for data in collection.find({ "key": { "$regex" : "ActivityProbe$" }, "time": { "$gte" : start, "$lt":end }}):
        #pdb.set_trace()
        dataValue = data["value"]
        totalIntervals += dataValue["total_intervals"]
        lowActivityIntervals += dataValue["low_activity_intervals"]
        highActivityIntervals += dataValue["high_activity_intervals"]

    return { "total": totalIntervals, "low": lowActivityIntervals, "high": highActivityIntervals }

@task()
def activityForToday():
    profiles = Profile.objects.all()
    aggregates = {}

    # Note: left off setting midnight to loop over hours until now...
    currentTime = time.mktime(time.gmtime())
    # Interesting way of getting midnight for the day of the current GM time.... is there a better way?
    midnight = time.mktime(date.fromtimestamp(currentTime - (3600 * 24 * 2)).timetuple())

    for profile in profiles:
        # Get the mongo store for the given user
        dbName = "User_" + str(profile.id)
        collection = connection[dbName]["funf"]
        aggregates[profile.uuid] = {}

        for offsetFromMidnight in range(int(midnight), int(currentTime), 3600):
            hour = (offsetFromMidnight - midnight) / 3600

            aggregates[profile.uuid][hour] = activityForTimeRange(collection, offsetFromMidnight, offsetFromMidnight + 3600)

    return aggregates
        <|endoftext|>


<commit_before>from deuces.deuces import Card
from convenience import find_pcts
import sys

card_str = sys.argv[1]
assert len(card_str) == 8

p1 = [Card.new('As'), Card.new('Kc')]
p2 = [Card.new('7h'), Card.new('7d')]
print find_pcts(p1, p2, start_b = [], iter = 10000)
<commit_msg>Successfully parse command line hands description.<commit_after># usage: python one_time_eval.py as8sqdtc

from deuces.deuces import Card
from convenience import find_pcts, pr
import sys

s = sys.argv[1]
assert len(s) == 8
cards = [s[0:2], s[2:4], s[4:6], s[6:8]]
for i, ci in enumerate(cards):
    cards[i] = ci[0].upper() + ci[1].lower()

p1 = [Card.new(cards[0]), Card.new(cards[1])]
p2 = [Card.new(cards[2]), Card.new(cards[3])]
pr(p1)
pr(p2)
print find_pcts(p1, p2, start_b = [], iter = 10000)
<|endoftext|>