<commit_before>#!/usr/bin/env python
# pypreprocessor.py

__author__ = 'Evan Plaice'
__version__ = '0.6.0'
#modified by hendiol

# changed by hendiol at 18.01.2017: added reset_internal for processing several files after each other
# changed by hendiol at 11.04.2017: trying to get nested #ifdefs handeld

#develop nested further

import sys
import os
import traceback
import imp

class preprocessor:
    def __init__(self, inFile=sys.argv[0], outFile='',
                 defines=[], removeMeta=False, escapeChar = '#'):
        # public variables
        self.defines = defines
        self.input = inFile
        self.output = outFile
        self.removeMeta = removeMeta
        self.escapeChar = escapeChar
        # private variables
        self.__linenum = 0
        self.__excludeblock = False
        self.__ifblocks = []
        self.__ifconditions = []
        self.__evalsquelch = True
        self.__outputBuffer = ''

    # reseting internal things to parse a second file
    def reset_internal(self):
        self.__linenum = 0
        self.__excludeblock = False
        self.__ifblocks = []
        self.__ifconditions = []
        self.__evalsquelch = True
        self.__outputBuffer = ''

    # the #define directive
    def define(self, define):
        self.defines.append(define)

    # the #undef directive
    def undefine(self, define):
        # re-map the defines list excluding the define specified in the args
        self.defines[:] = [x for x in self.defines if x != define]

    def search_defines(self, define):
        if define in self.defines:
            return True
        else:
            return False

    # the #ifdef directive
    def compare_defines_and_conditions(self, defines, conditions):
        # if defines and conditions lists have no intersecting values (ie. else = true)
        if not [val for val in defines if val in conditions]:
            return True
        else:
            return False

    def __if(self):
        value = bool(self.__ifblocks)
        for ib in self.__ifblocks: #may better use list comprehension
            value=value and ib
        return not value    #not: because True means removing

 # evaluate
    def lexer(self, line):
    # return values are (squelch, metadata)
        if not self.__ifblocks and self.__excludeblock is False:
            # squelch the preprocessor parse on the first
            # pass to prevent preprocessor infinite loop
            if 'pypreprocessor.parse()' in line:
                return True, True
            if line[:1] != self.escapeChar:
                return False, False
        # handle #define directives
        if line[:7] == self.escapeChar + 'define':
            if len(line.split()) != 2:
                self.exit_error(self.escapeChar + 'define')
            else:
                self.define(line.split()[1])
                return False, True
        # handle #undef directives
        if line[:6] == self.escapeChar + 'undef':
            if len(line.split()) != 2:
                self.exit_error(self.escapeChar + 'undef')
            else:
                self.undefine(line.split()[1])
                return False, True
        # handle #endif directives
        if line[:6] == self.escapeChar + 'endif':
            if len(line.split()) != 1:
                self.exit_error(self.escapeChar + 'endif')
            else:
                if len(self.__ifconditions)>1:
                    self.__ifblocks.pop(-1)
                    self.__ifcondition=self.__ifconditions.pop(-1)
                else:
                    self.__ifblocks = []
                    self.__ifconditions = []
                return False, True

        # handle #else directives
        if line[:5] == self.escapeChar + 'else':
            if len(line.split()) != 1:
                self.exit_error(self.escapeChar + 'else')
            else:
                self.__ifblocks[-1]=not(self.search_defines(self.__ifconditions[-1]))
            return False, True

        # handle #endexclude directives
        if line[:11] == self.escapeChar + 'endexclude':
            if len(line.split()) != 1:
                self.exit_error(self.escapeChar + 'endexclude')
            else:
                self.__excludeblock = False
                return False, True
        # handle #exclude directives
        if line[:8] == self.escapeChar + 'exclude':
            if len(line.split()) != 1:
                self.exit_error(self.escapeChar + 'exclude')
            else:
                self.__excludeblock = True
        # process the excludeblock
        if self.__excludeblock is True:
            return True, False
        # handle #ifdef directives
        if line[:6] == self.escapeChar + 'ifdef':
            if len(line.split()) != 2:
                self.exit_error(self.escapeChar + 'ifdef')
            else:
                self.__ifblocks.append(self.search_defines(line.split()[1]))
                self.__ifconditions.append(line.split()[1])
        # handle #else directives
        if line[:5] == self.escapeChar + 'else':
            if len(line.split()) != 1:
                self.exit_error(self.escapeChar + 'else')
        # process the ifblock
        if self.__ifblocks: # is True:
            # evaluate and process an #ifdef
            if line[:6] == self.escapeChar + 'ifdef':
                return False, True
            # evaluate and process the #else
            elif line[:5] == self.escapeChar + 'else':
                return False, True
            else:
                return self.__if(), False #self.__evalsquelch
        else:
            return False, False

    # error handling
    def exit_error(self, directive):
        print('File: "' + self.input + '", line ' + str(self.__linenum))
        print('SyntaxError: Invalid ' + directive + ' directive')
        sys.exit(1)
    def rewrite_traceback(self):
        trace = traceback.format_exc().splitlines()
        index = 0
        for line in trace:
            if index == (len(trace) - 2):
                print(line.replace("<string>", self.input))
            else:
                print(line)
            index += 1

    # parsing/processing
    def parse(self):
        # open the input file
        input_file = open(os.path.join(self.input),'r')
        try:
            # process the input file
            for line in input_file:
                self.__linenum += 1
                # to squelch or not to squelch
                print('__________________')
                print('lexer',self.__linenum)
                print('Def',self.defines)
                print('Cond',self.__ifconditions)
                print('Ifs',self.__ifblocks)
                print('Eval',self.__evalsquelch)
                print(self.__if())
                #
                squelch, metaData = self.lexer(line)
                #
                print('----')
                print('Def',self.defines)
                print('Cond',self.__ifconditions)
                print('Ifs',self.__ifblocks)
                print('Eval',self.__evalsquelch)
                print(self.__if())
                print(squelch, metaData)
                print(line)
                #
                # process and output
                if self.removeMeta is True:
                    if metaData is True or squelch is True:
                        continue
                if squelch is True:
                    self.__outputBuffer += self.escapeChar + line
                    continue
                if squelch is False:
                    self.__outputBuffer += line
                    continue
        finally:
            input_file.close()
        self.post_process()

    # post-processor
    def post_process(self):
        try:
            # open file for output (no auto-run)
            if self.output != '':
                self.run = False
                output_file = open(self.output, 'w')
            # open tmp file
            else:
                self.run = True
                self.output = 'tmp_' + os.path.basename(self.input)
                output_file = open(self.output, 'w')
            # write post-processed code to file
            output_file.write(self.__outputBuffer)
        finally:
            output_file.close()
        # resolve postprocess stage depending on the mode
        '''
        if self.run == False:
            sys.exit(0)
        else:
        #'''
        if self.run == True:
            # if this module is loaded as a library override the import
            if imp.lock_held() is True:
                    self.override_import()
            else:
                self.on_the_fly()
                # break execution so python doesn't
                # run the rest of the pre-processed code
                sys.exit(0)

    # postprocessor - override an import
    def override_import(self):
        try:
            moduleName = self.input.split('.')[0]
            tmpModuleName = self.output.split('.')[0]
            del sys.modules[moduleName]
            sys.modules[tmpModuleName] = __import__(tmpModuleName)
            sys.modules[moduleName] = __import__(tmpModuleName)
        except:
            self.rewrite_traceback()
        finally:
            # remove tmp (.py & .pyc) files
            os.remove(self.output)
            os.remove(self.output + 'c')

    # postprocessor - on-the-fly execution
    def on_the_fly(self):
        try:
            exec(open(self.output,"rb").read())
        except:
            self.rewrite_traceback()
        finally:
            # remove tmp file
            os.remove(self.output)

pypreprocessor = preprocessor()<commit_msg>Changing the order makes things more simple to read and faster ????<commit_after>#!/usr/bin/env python
# pypreprocessor.py

__author__ = 'Evan Plaice'
__version__ = '0.6.0'
#modified by hendiol

# changed by hendiol at 18.01.2017: added reset_internal for processing several files after each other
# changed by hendiol at 11.04.2017: trying to get nested #ifdefs handeld

#develop nested further

import sys
import os
import traceback
import imp

class preprocessor:
    def __init__(self, inFile=sys.argv[0], outFile='',
                 defines=[], removeMeta=False, escapeChar = '#'):
        # public variables
        self.defines = defines
        self.input = inFile
        self.output = outFile
        self.removeMeta = removeMeta
        self.escapeChar = escapeChar
        # private variables
        self.__linenum = 0
        self.__excludeblock = False
        self.__ifblocks = []
        self.__ifconditions = []
        self.__evalsquelch = True
        self.__outputBuffer = ''

    # reseting internal things to parse a second file
    def reset_internal(self):
        self.__linenum = 0
        self.__excludeblock = False
        self.__ifblocks = []
        self.__ifconditions = []
        self.__evalsquelch = True
        self.__outputBuffer = ''

    # the #define directive
    def define(self, define):
        self.defines.append(define)

    # the #undef directive
    def undefine(self, define):
        # re-map the defines list excluding the define specified in the args
        self.defines[:] = [x for x in self.defines if x != define]

    def search_defines(self, define):
        if define in self.defines:
            return True
        else:
            return False

    # the #ifdef directive
    def compare_defines_and_conditions(self, defines, conditions):
        # if defines and conditions lists have no intersecting values (ie. else = true)
        if not [val for val in defines if val in conditions]:
            return True
        else:
            return False

    def __if(self):
        value = bool(self.__ifblocks)
        for ib in self.__ifblocks: #may better use list comprehension
            value=value and ib
        return not value    #not: because True means removing

 # evaluate
    def lexer(self, line):
    # return values are (squelch, metadata)
        if not self.__ifblocks and self.__excludeblock is False:
            # squelch the preprocessor parse on the first
            # pass to prevent preprocessor infinite loop
            if 'pypreprocessor.parse()' in line:
                return True, True
            if line[:1] != self.escapeChar:
                return False, False
        # handle #define directives
        if line[:7] == self.escapeChar + 'define':
            if len(line.split()) != 2:
                self.exit_error(self.escapeChar + 'define')
            else:
                self.define(line.split()[1])
                return False, True
        # handle #undef directives
        elif line[:6] == self.escapeChar + 'undef':
            if len(line.split()) != 2:
                self.exit_error(self.escapeChar + 'undef')
            else:
                self.undefine(line.split()[1])
                return False, True
        # handle #exclude directives
        elif line[:8] == self.escapeChar + 'exclude':
            if len(line.split()) != 1:
                self.exit_error(self.escapeChar + 'exclude')
            else:
                self.__excludeblock = True
        # handle #endexclude directives
        elif line[:11] == self.escapeChar + 'endexclude':
            if len(line.split()) != 1:
                self.exit_error(self.escapeChar + 'endexclude')
            else:
                self.__excludeblock = False
                return False, True
        # handle #endif directives
        if line[:6] == self.escapeChar + 'endif':
            if len(line.split()) != 1:
                self.exit_error(self.escapeChar + 'endif')
            else:
                if len(self.__ifconditions)>1:
                    self.__ifblocks.pop(-1)
                    self.__ifcondition=self.__ifconditions.pop(-1)
                else:
                    self.__ifblocks = []
                    self.__ifconditions = []
                return False, True

        # handle #else directives
        if line[:5] == self.escapeChar + 'else':
            if len(line.split()) != 1:
                self.exit_error(self.escapeChar + 'else')
            else:
                self.__ifblocks[-1]=not(self.search_defines(self.__ifconditions[-1]))
            return False, True

        # process the excludeblock
        if self.__excludeblock is True:
            return True, False
        # handle #ifdef directives
        if line[:6] == self.escapeChar + 'ifdef':
            if len(line.split()) != 2:
                self.exit_error(self.escapeChar + 'ifdef')
            else:
                self.__ifblocks.append(self.search_defines(line.split()[1]))
                self.__ifconditions.append(line.split()[1])
        # handle #else directives
        if line[:5] == self.escapeChar + 'else':
            if len(line.split()) != 1:
                self.exit_error(self.escapeChar + 'else')
        # process the ifblock
        if self.__ifblocks: # is True:
            # evaluate and process an #ifdef
            if line[:6] == self.escapeChar + 'ifdef':
                return False, True
            # evaluate and process the #else
            elif line[:5] == self.escapeChar + 'else':
                return False, True
            else:
                return self.__if(), False #self.__evalsquelch
        else:
            return False, False

    # error handling
    def exit_error(self, directive):
        print('File: "' + self.input + '", line ' + str(self.__linenum))
        print('SyntaxError: Invalid ' + directive + ' directive')
        sys.exit(1)
    def rewrite_traceback(self):
        trace = traceback.format_exc().splitlines()
        index = 0
        for line in trace:
            if index == (len(trace) - 2):
                print(line.replace("<string>", self.input))
            else:
                print(line)
            index += 1

    # parsing/processing
    def parse(self):
        # open the input file
        input_file = open(os.path.join(self.input),'r')
        try:
            # process the input file
            for line in input_file:
                self.__linenum += 1
                # to squelch or not to squelch
                print('__________________')
                print('lexer',self.__linenum)
                print('Def',self.defines)
                print('Cond',self.__ifconditions)
                print('Ifs',self.__ifblocks)
                print('Eval',self.__evalsquelch)
                print(self.__if())
                #
                squelch, metaData = self.lexer(line)
                #
                print('----')
                print('Def',self.defines)
                print('Cond',self.__ifconditions)
                print('Ifs',self.__ifblocks)
                print('Eval',self.__evalsquelch)
                print(self.__if())
                print(squelch, metaData)
                print(line)
                #
                # process and output
                if self.removeMeta is True:
                    if metaData is True or squelch is True:
                        continue
                if squelch is True:
                    self.__outputBuffer += self.escapeChar + line
                    continue
                if squelch is False:
                    self.__outputBuffer += line
                    continue
        finally:
            input_file.close()
        self.post_process()

    # post-processor
    def post_process(self):
        try:
            # open file for output (no auto-run)
            if self.output != '':
                self.run = False
                output_file = open(self.output, 'w')
            # open tmp file
            else:
                self.run = True
                self.output = 'tmp_' + os.path.basename(self.input)
                output_file = open(self.output, 'w')
            # write post-processed code to file
            output_file.write(self.__outputBuffer)
        finally:
            output_file.close()
        # resolve postprocess stage depending on the mode
        '''
        if self.run == False:
            sys.exit(0)
        else:
        #'''
        if self.run == True:
            # if this module is loaded as a library override the import
            if imp.lock_held() is True:
                    self.override_import()
            else:
                self.on_the_fly()
                # break execution so python doesn't
                # run the rest of the pre-processed code
                sys.exit(0)

    # postprocessor - override an import
    def override_import(self):
        try:
            moduleName = self.input.split('.')[0]
            tmpModuleName = self.output.split('.')[0]
            del sys.modules[moduleName]
            sys.modules[tmpModuleName] = __import__(tmpModuleName)
            sys.modules[moduleName] = __import__(tmpModuleName)
        except:
            self.rewrite_traceback()
        finally:
            # remove tmp (.py & .pyc) files
            os.remove(self.output)
            os.remove(self.output + 'c')

    # postprocessor - on-the-fly execution
    def on_the_fly(self):
        try:
            exec(open(self.output,"rb").read())
        except:
            self.rewrite_traceback()
        finally:
            # remove tmp file
            os.remove(self.output)

pypreprocessor = preprocessor()<|endoftext|>


<commit_before>#!/usr/bin/env python
# pypreprocessor.py

__author__ = 'Evan Plaice'
__coauthor__ = 'Hendi O L'
__version__ = '0.74'

import sys
import os
import traceback
import imp

class preprocessor:
    def __init__(self, inFile=sys.argv[0], outFile='',
                 defines=[], removeMeta=False, escape = '#', run=True, resume=False, save = True):
        # public variables
        self.defines = defines
        self.input = inFile
        self.output = outFile
        self.removeMeta = removeMeta
        self.escape = escape
        self.run = run
        self.resume = resume
        self.save = save
        self.readEncoding = sys.stdin.encoding
        self.writeEncoding = sys.stdout.encoding
        # private variables
        self.__linenum = 0
        self.__excludeblock = False
        self.__ifblocks = []
        self.__ifconditions = []
        self.__evalsquelch = True
        self.__outputBuffer = ''

    # reseting internal things to parse a second file
    def __reset_internal(self):
        self.__linenum = 0
        self.__excludeblock = False
        self.__ifblocks = []
        self.__ifconditions = []
        self.__evalsquelch = True
        self.__outputBuffer = ''

    # the #define directive
    def define(self, define):
        self.defines.append(define)

    # the #undef directive
    def undefine(self, define):
        # re-map the defines list excluding the define specified in the args
        self.defines[:] = [x for x in self.defines if x != define]

    # search: if define is defined
    def search_defines(self, define):
        if define in self.defines:
            return True
        else:
            return False

    #returning: validness of #ifdef #else block
    def __if(self):
        value = bool(self.__ifblocks)
        for ib in self.__ifblocks:
           value*=ib   #* represents and: value = value and ib
        return not value    #not: because True means removing

    # evaluate
    def lexer(self, line):
    # return values are (squelch, metadata)
        if not (self.__ifblocks or self.__excludeblock):
            if 'pypreprocessor.parse()' in line:
                return True, True
            #this block only for faster processing (not necessary)
            elif line[:len(self.escape)] != self.escape:
                return False, False
        # handle #define directives
        if line[:len(self.escape) + 6] == self.escape + 'define':
            if len(line.split()) != 2:
                self.exit_error(self.escape + 'define')
            else:
                self.define(line.split()[1])
                return False, True
        # handle #undef directives
        elif line[:len(self.escape) + 5] == self.escape + 'undef':
            if len(line.split()) != 2:
                self.exit_error(self.escape + 'undef')
            else:
                self.undefine(line.split()[1])
                return False, True
        # handle #exclude directives
        elif line[:len(self.escape) + 7] == self.escape + 'exclude':
            if len(line.split()) != 1:
                self.exit_error(self.escape + 'exclude')
            else:
                self.__excludeblock = True
                return False, True
        # handle #endexclude directives
        elif line[:len(self.escape) + 10] == self.escape + 'endexclude':
            if len(line.split()) != 1:
                self.exit_error(self.escape + 'endexclude')
            else:
                self.__excludeblock = False
                return False, True
        # handle #ifnotdef directives (is the same as: #ifdef X #else)
        elif line[:len(self.escape) + 8] == self.escape + 'ifdefnot':
            if len(line.split()) != 2:
                self.exit_error(self.escape + 'ifdefnot')
            else:
                self.__ifblocks.append(not(self.search_defines(line.split()[1])))
                self.__ifconditions.append(line.split()[1])
                return False, True
        # handle #ifdef directives
        elif line[:len(self.escape) + 5] == self.escape + 'ifdef':
            if len(line.split()) != 2:
                self.exit_error(self.escape + 'ifdef')
            else:
                self.__ifblocks.append(self.search_defines(line.split()[1]))
                self.__ifconditions.append(line.split()[1])
                return False, True
        # handle #else...
        # handle #elseif directives
        elif line[:len(self.escape) + 6] == self.escape + 'elseif':
            if len(line.split()) != 2:
                self.exit_error(self.escape + 'elseif')
            else:
                self.__ifblocks[-1]=not(self.__ifblocks[-1])#self.search_defines(self.__ifconditions[-1]))
                self.__ifblocks.append(self.search_defines(line.split()[1]))
                self.__ifconditions.append(line.split()[1])
            return False, True
        # handle #else directives
        elif line[:len(self.escape) + 4] == self.escape + 'else':
            if len(line.split()) != 1:
                self.exit_error(self.escape + 'else')
            else:
                self.__ifblocks[-1]=not(self.__ifblocks[-1])#self.search_defines(self.__ifconditions[-1]))
            return False, True
        # handle #endif..
        # handle #endififdef
        elif line[:len(self.escape) + 10] == self.escape + 'endififdef':
            if len(line.split()) != 2:
                self.exit_error(self.escape + 'endififdef')
            else:
                if len(self.__ifconditions)>=1:
                    self.__ifblocks.pop(-1)
                    self.__ifcondition=self.__ifconditions.pop(-1)
                else:
                    self.__ifblocks = []
                    self.__ifconditions = []
                self.__ifblocks.append(self.search_defines(line.split()[1]))
                self.__ifconditions.append(line.split()[1])
                return False, True
        # handle #endifall directives
        elif line[:len(self.escape) + 8] == self.escape + 'endifall':
            if len(line.split()) != 1:
                self.exit_error(self.escape + 'endifall')
            else:
                self.__ifblocks = []
                self.__ifconditions = []
                return False, True
        # handle #endif and #endif numb directives
        elif line[:len(self.escape) + 5] == self.escape + 'endif':
            if len(line.split()) != 1:
                self.exit_error(self.escape + 'endif number')
            else:
                try:
                    number=int(line[6:])
                except ValueError as VE:
                    #print('ValueError',VE)
                    #self.exit_error(self.escape + 'endif number')
                    number=1
                if len(self.__ifconditions)>number:
                    for i in range(0,number):
                        self.__ifblocks.pop(-1)
                        self.__ifcondition=self.__ifconditions.pop(-1)
                elif len(self.__ifconditions) == number:
                    self.__ifblocks = []
                    self.__ifconditions = []
                else:
                    print('Warning try to remove more blocks than present', self.input, self.__linenum)
                    self.__ifblocks = []
                    self.__ifconditions = []
                return False, True
        else: #No directive --> execute
            # process the excludeblock
            if self.__excludeblock is True:
                return True, False
            # process the ifblock
            elif self.__ifblocks: # is True:
                return self.__if(), False
            #here can add other stuff for deleting comnments eg
            else:
                return False, False

    # error handling
    def exit_error(self, directive):
        print('File: "' + self.input + '", line ' + str(self.__linenum))
        print('SyntaxError: Invalid ' + directive + ' directive')
        sys.exit(1)
    def rewrite_traceback(self):
        trace = traceback.format_exc().splitlines()
        index = 0
        for line in trace:
            if index == (len(trace) - 2):
                print(line.replace("<string>", self.input))
            else:
                print(line)
            index += 1

    # parsing/processing
    def parse(self):
        # open the input file
        input_file = open(os.path.join(self.input),'r', encoding=self.readEncoding)
        try:
            # process the input file
            for line in input_file:
                self.__linenum += 1
                # to squelch or not to squelch
                squelch, metaData = self.lexer(line)
                # process and output
                if self.removeMeta is True:
                    if metaData is True or squelch is True:
                        continue
                if squelch is True:
                    if(metaData):
                        self.__outputBuffer += self.escape + line
                    else:
                        self.__outputBuffer += self.escape[0] + line
                    continue
                if squelch is False:
                    self.__outputBuffer += line
                    continue
        finally:
            input_file.close()
            #Warnings for unclosed #ifdef blocks
            if self.__ifblocks:
                print('Warning: Number of unclosed Ifdefblocks: ',len(self.__ifblocks))
                print('Can cause unwished behaviour in the preprocessed code, preprocessor is safe')
                if input('Do you want more Information? ').lower() in ('yes','true','t','1'):
                    print('Name of input and output file: ',self.input,' ',self.output)
                    for i, item in enumerate(self.__ifconditions):
                        if (item in self.defines) != self.__ifblocks[i]:
                            cond = ' else '
                        else:
                            cond = ' if '
                        print('Block:',item, ' is in condition: ',cond )

        self.post_process()

    # post-processor
    def post_process(self):
        try:
            # set file name
            if self.output == '':
                self.output = self.input[0:-len(self.input.split('.')[-1])-1]+'_out.'+self.input.split('.')[-1]
            # open file for output
            output_file = open(self.output, 'w', encoding=self.writeEncoding)
            # write post-processed code to file
            output_file.write(self.__outputBuffer)
        finally:
            output_file.close()

        if(self.run):
            # if this module is loaded as a library override the import
            if imp.lock_held() is True:
                self.override_import()
                return
            else:
                self.on_the_fly()
        if(self.resume):
            self.__reset_internal()
            pass
        else:
            # break execution so python doesn't
            # run the rest of the pre-processed code
            sys.exit(0)

    # postprocessor - override an import
    def override_import(self):
        try:
            moduleName = self.input.split('.')[0]
            tmpModuleName = self.output.split('.')[0]
            del sys.modules[moduleName]
            sys.modules[tmpModuleName] = __import__(tmpModuleName)
            sys.modules[moduleName] = __import__(tmpModuleName)
        except:
            self.rewrite_traceback()
        finally:
            # remove tmp (.py & .pyc) files
            os.remove(self.output)
            os.remove(self.output + 'c')

    # postprocessor - on-the-fly execution
    def on_the_fly(self):
        try:
            exec(open(self.output,"r", encoding=self.readEncoding).read())
        except:
            self.rewrite_traceback()
        finally:
            if(not self.save):
                # remove tmp file
                os.remove(self.output)

pypreprocessor = preprocessor()<commit_msg>make exec run in global<commit_after>#!/usr/bin/env python
# pypreprocessor.py

__author__ = 'Evan Plaice'
__coauthor__ = 'Hendi O L'
__version__ = '0.74'

import sys
import os
import traceback
import imp

class preprocessor:
    def __init__(self, inFile=sys.argv[0], outFile='',
                 defines=[], removeMeta=False, escape = '#', run=True, resume=False, save = True):
        # public variables
        self.defines = defines
        self.input = inFile
        self.output = outFile
        self.removeMeta = removeMeta
        self.escape = escape
        self.run = run
        self.resume = resume
        self.save = save
        self.readEncoding = sys.stdin.encoding
        self.writeEncoding = sys.stdout.encoding
        # private variables
        self.__linenum = 0
        self.__excludeblock = False
        self.__ifblocks = []
        self.__ifconditions = []
        self.__evalsquelch = True
        self.__outputBuffer = ''

    # reseting internal things to parse a second file
    def __reset_internal(self):
        self.__linenum = 0
        self.__excludeblock = False
        self.__ifblocks = []
        self.__ifconditions = []
        self.__evalsquelch = True
        self.__outputBuffer = ''

    # the #define directive
    def define(self, define):
        self.defines.append(define)

    # the #undef directive
    def undefine(self, define):
        # re-map the defines list excluding the define specified in the args
        self.defines[:] = [x for x in self.defines if x != define]

    # search: if define is defined
    def search_defines(self, define):
        if define in self.defines:
            return True
        else:
            return False

    #returning: validness of #ifdef #else block
    def __if(self):
        value = bool(self.__ifblocks)
        for ib in self.__ifblocks:
           value*=ib   #* represents and: value = value and ib
        return not value    #not: because True means removing

    # evaluate
    def lexer(self, line):
    # return values are (squelch, metadata)
        if not (self.__ifblocks or self.__excludeblock):
            if 'pypreprocessor.parse()' in line:
                return True, True
            #this block only for faster processing (not necessary)
            elif line[:len(self.escape)] != self.escape:
                return False, False
        # handle #define directives
        if line[:len(self.escape) + 6] == self.escape + 'define':
            if len(line.split()) != 2:
                self.exit_error(self.escape + 'define')
            else:
                self.define(line.split()[1])
                return False, True
        # handle #undef directives
        elif line[:len(self.escape) + 5] == self.escape + 'undef':
            if len(line.split()) != 2:
                self.exit_error(self.escape + 'undef')
            else:
                self.undefine(line.split()[1])
                return False, True
        # handle #exclude directives
        elif line[:len(self.escape) + 7] == self.escape + 'exclude':
            if len(line.split()) != 1:
                self.exit_error(self.escape + 'exclude')
            else:
                self.__excludeblock = True
                return False, True
        # handle #endexclude directives
        elif line[:len(self.escape) + 10] == self.escape + 'endexclude':
            if len(line.split()) != 1:
                self.exit_error(self.escape + 'endexclude')
            else:
                self.__excludeblock = False
                return False, True
        # handle #ifnotdef directives (is the same as: #ifdef X #else)
        elif line[:len(self.escape) + 8] == self.escape + 'ifdefnot':
            if len(line.split()) != 2:
                self.exit_error(self.escape + 'ifdefnot')
            else:
                self.__ifblocks.append(not(self.search_defines(line.split()[1])))
                self.__ifconditions.append(line.split()[1])
                return False, True
        # handle #ifdef directives
        elif line[:len(self.escape) + 5] == self.escape + 'ifdef':
            if len(line.split()) != 2:
                self.exit_error(self.escape + 'ifdef')
            else:
                self.__ifblocks.append(self.search_defines(line.split()[1]))
                self.__ifconditions.append(line.split()[1])
                return False, True
        # handle #else...
        # handle #elseif directives
        elif line[:len(self.escape) + 6] == self.escape + 'elseif':
            if len(line.split()) != 2:
                self.exit_error(self.escape + 'elseif')
            else:
                self.__ifblocks[-1]=not(self.__ifblocks[-1])#self.search_defines(self.__ifconditions[-1]))
                self.__ifblocks.append(self.search_defines(line.split()[1]))
                self.__ifconditions.append(line.split()[1])
            return False, True
        # handle #else directives
        elif line[:len(self.escape) + 4] == self.escape + 'else':
            if len(line.split()) != 1:
                self.exit_error(self.escape + 'else')
            else:
                self.__ifblocks[-1]=not(self.__ifblocks[-1])#self.search_defines(self.__ifconditions[-1]))
            return False, True
        # handle #endif..
        # handle #endififdef
        elif line[:len(self.escape) + 10] == self.escape + 'endififdef':
            if len(line.split()) != 2:
                self.exit_error(self.escape + 'endififdef')
            else:
                if len(self.__ifconditions)>=1:
                    self.__ifblocks.pop(-1)
                    self.__ifcondition=self.__ifconditions.pop(-1)
                else:
                    self.__ifblocks = []
                    self.__ifconditions = []
                self.__ifblocks.append(self.search_defines(line.split()[1]))
                self.__ifconditions.append(line.split()[1])
                return False, True
        # handle #endifall directives
        elif line[:len(self.escape) + 8] == self.escape + 'endifall':
            if len(line.split()) != 1:
                self.exit_error(self.escape + 'endifall')
            else:
                self.__ifblocks = []
                self.__ifconditions = []
                return False, True
        # handle #endif and #endif numb directives
        elif line[:len(self.escape) + 5] == self.escape + 'endif':
            if len(line.split()) != 1:
                self.exit_error(self.escape + 'endif number')
            else:
                try:
                    number=int(line[6:])
                except ValueError as VE:
                    #print('ValueError',VE)
                    #self.exit_error(self.escape + 'endif number')
                    number=1
                if len(self.__ifconditions)>number:
                    for i in range(0,number):
                        self.__ifblocks.pop(-1)
                        self.__ifcondition=self.__ifconditions.pop(-1)
                elif len(self.__ifconditions) == number:
                    self.__ifblocks = []
                    self.__ifconditions = []
                else:
                    print('Warning try to remove more blocks than present', self.input, self.__linenum)
                    self.__ifblocks = []
                    self.__ifconditions = []
                return False, True
        else: #No directive --> execute
            # process the excludeblock
            if self.__excludeblock is True:
                return True, False
            # process the ifblock
            elif self.__ifblocks: # is True:
                return self.__if(), False
            #here can add other stuff for deleting comnments eg
            else:
                return False, False

    # error handling
    def exit_error(self, directive):
        print('File: "' + self.input + '", line ' + str(self.__linenum))
        print('SyntaxError: Invalid ' + directive + ' directive')
        sys.exit(1)
    def rewrite_traceback(self):
        trace = traceback.format_exc().splitlines()
        index = 0
        for line in trace:
            if index == (len(trace) - 2):
                print(line.replace("<string>", self.input))
            else:
                print(line)
            index += 1

    # parsing/processing
    def parse(self):
        # open the input file
        input_file = open(os.path.join(self.input),'r', encoding=self.readEncoding)
        try:
            # process the input file
            for line in input_file:
                self.__linenum += 1
                # to squelch or not to squelch
                squelch, metaData = self.lexer(line)
                # process and output
                if self.removeMeta is True:
                    if metaData is True or squelch is True:
                        continue
                if squelch is True:
                    if(metaData):
                        self.__outputBuffer += self.escape + line
                    else:
                        self.__outputBuffer += self.escape[0] + line
                    continue
                if squelch is False:
                    self.__outputBuffer += line
                    continue
        finally:
            input_file.close()
            #Warnings for unclosed #ifdef blocks
            if self.__ifblocks:
                print('Warning: Number of unclosed Ifdefblocks: ',len(self.__ifblocks))
                print('Can cause unwished behaviour in the preprocessed code, preprocessor is safe')
                if input('Do you want more Information? ').lower() in ('yes','true','t','1'):
                    print('Name of input and output file: ',self.input,' ',self.output)
                    for i, item in enumerate(self.__ifconditions):
                        if (item in self.defines) != self.__ifblocks[i]:
                            cond = ' else '
                        else:
                            cond = ' if '
                        print('Block:',item, ' is in condition: ',cond )

        self.post_process()

    # post-processor
    def post_process(self):
        try:
            # set file name
            if self.output == '':
                self.output = self.input[0:-len(self.input.split('.')[-1])-1]+'_out.'+self.input.split('.')[-1]
            # open file for output
            output_file = open(self.output, 'w', encoding=self.writeEncoding)
            # write post-processed code to file
            output_file.write(self.__outputBuffer)
        finally:
            output_file.close()

        if(self.run):
            # if this module is loaded as a library override the import
            if imp.lock_held() is True:
                self.override_import()
                return
            else:
                self.on_the_fly()
        if(self.resume):
            self.__reset_internal()
            pass
        else:
            # break execution so python doesn't
            # run the rest of the pre-processed code
            sys.exit(0)

    # postprocessor - override an import
    def override_import(self):
        try:
            moduleName = self.input.split('.')[0]
            tmpModuleName = self.output.split('.')[0]
            del sys.modules[moduleName]
            sys.modules[tmpModuleName] = __import__(tmpModuleName)
            sys.modules[moduleName] = __import__(tmpModuleName)
        except:
            self.rewrite_traceback()
        finally:
            # remove tmp (.py & .pyc) files
            os.remove(self.output)
            os.remove(self.output + 'c')

    # postprocessor - on-the-fly execution
    def on_the_fly(self):
        try:
            exec(open(self.output,"r", encoding=self.readEncoding).read(), globals())
        except:
            self.rewrite_traceback()
        finally:
            if(not self.save):
                # remove tmp file
                os.remove(self.output)

pypreprocessor = preprocessor()<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
Module to search the SAO/NASA Astrophysics Data System

:author: Magnus Persson <magnusp@vilhelm.nu>

"""

from astropy.table import Table

from ..query import BaseQuery
from ..utils import commons, async_to_sync
from ..utils.class_or_instance import class_or_instance

from .utils import _get_data_from_xml
from . import conf

from xml.dom import minidom

__all__ = ['ADS', 'ADSClass']

@async_to_sync
class ADSClass(BaseQuery):

    SERVER = conf.server
    QUERY_ADVANCED_PATH = conf.advanced_path
    QUERY_SIMPLE_PATH = conf.simple_path
    TIMEOUT = conf.timeout

    QUERY_SIMPLE_URL = SERVER + QUERY_SIMPLE_PATH
    QUERY_ADVANCED_URL = SERVER + QUERY_ADVANCED_PATH

    def __init__(self, *args):
        """ set some parameters """
        super(ADSClass, self).__init__()

    @class_or_instance
    def query_simple(self, query_string, get_query_payload=False,
                     get_raw_response=False, cache=True):
        """
        Basic query.  Uses a string and the ADS generic query.
        """
        request_payload = self._args_to_payload(query_string)

        response = self._request(method='POST', url=self.QUERY_SIMPLE_URL,
                                 data=request_payload, timeout=self.TIMEOUT,
                                 cache=cache)

        response.raise_for_status()

        # primarily for debug purposes, but also useful if you want to send
        # someone a URL linking directly to the data
        if get_query_payload:
            return request_payload
        if get_raw_response:
            return response
        # parse the XML response into AstroPy Table
        resulttable = self._parse_response(response)

        return resulttable

    def _parse_response(self, response):

        encoded_content = response.text.encode(response.encoding)

        xmlrepr = minidom.parseString(encoded_content)
        # Check if there are any results!

        # get the list of hits
        hitlist = xmlrepr.childNodes[0].childNodes
        hitlist = hitlist[1::2]  # every second hit is a "line break"

        # Grab the various fields
        titles = _get_data_from_xml(hitlist, 'title')
        bibcode = _get_data_from_xml(hitlist, 'bibcode')
        journal = _get_data_from_xml(hitlist, 'journal')
        volume = _get_data_from_xml(hitlist, 'volume')
        pubdate = _get_data_from_xml(hitlist, 'pubdate')
        page = _get_data_from_xml(hitlist, 'page')
        score = _get_data_from_xml(hitlist, 'score')
        citations = _get_data_from_xml(hitlist, 'citations')
        abstract = _get_data_from_xml(hitlist, 'abstract')
        doi = _get_data_from_xml(hitlist, 'DOI')
        eprintid = _get_data_from_xml(hitlist, 'eprintid')
        authors = _get_data_from_xml(hitlist, 'author')
        # put into AstroPy Table
        t = Table()
        t['title'] = titles
        t['bibcode'] = bibcode
        t['journal'] = journal
        t['volume'] = volume
        t['pubdate'] = pubdate
        t['page'] = page
        t['score'] = score
        t['citations'] = citations
        t['abstract'] = abstract
        t['doi'] = doi
        t['eprintid'] = eprintid
        t['authors'] = authors

        return t

    def _args_to_payload(self, query_string):
        # convert arguments to a valid requests payload
        # i.e. a dictionary
        return {'qsearch': query_string, 'data_type': 'XML'}

ADS = ADSClass()
<commit_msg>Remove unused import<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
Module to search the SAO/NASA Astrophysics Data System

:author: Magnus Persson <magnusp@vilhelm.nu>

"""

from astropy.table import Table

from ..query import BaseQuery
from ..utils import async_to_sync
from ..utils.class_or_instance import class_or_instance

from .utils import _get_data_from_xml
from . import conf

from xml.dom import minidom

__all__ = ['ADS', 'ADSClass']

@async_to_sync
class ADSClass(BaseQuery):

    SERVER = conf.server
    QUERY_ADVANCED_PATH = conf.advanced_path
    QUERY_SIMPLE_PATH = conf.simple_path
    TIMEOUT = conf.timeout

    QUERY_SIMPLE_URL = SERVER + QUERY_SIMPLE_PATH
    QUERY_ADVANCED_URL = SERVER + QUERY_ADVANCED_PATH

    def __init__(self, *args):
        """ set some parameters """
        super(ADSClass, self).__init__()

    @class_or_instance
    def query_simple(self, query_string, get_query_payload=False,
                     get_raw_response=False, cache=True):
        """
        Basic query.  Uses a string and the ADS generic query.
        """
        request_payload = self._args_to_payload(query_string)

        response = self._request(method='POST', url=self.QUERY_SIMPLE_URL,
                                 data=request_payload, timeout=self.TIMEOUT,
                                 cache=cache)

        response.raise_for_status()

        # primarily for debug purposes, but also useful if you want to send
        # someone a URL linking directly to the data
        if get_query_payload:
            return request_payload
        if get_raw_response:
            return response
        # parse the XML response into AstroPy Table
        resulttable = self._parse_response(response)

        return resulttable

    def _parse_response(self, response):

        encoded_content = response.text.encode(response.encoding)

        xmlrepr = minidom.parseString(encoded_content)
        # Check if there are any results!

        # get the list of hits
        hitlist = xmlrepr.childNodes[0].childNodes
        hitlist = hitlist[1::2]  # every second hit is a "line break"

        # Grab the various fields
        titles = _get_data_from_xml(hitlist, 'title')
        bibcode = _get_data_from_xml(hitlist, 'bibcode')
        journal = _get_data_from_xml(hitlist, 'journal')
        volume = _get_data_from_xml(hitlist, 'volume')
        pubdate = _get_data_from_xml(hitlist, 'pubdate')
        page = _get_data_from_xml(hitlist, 'page')
        score = _get_data_from_xml(hitlist, 'score')
        citations = _get_data_from_xml(hitlist, 'citations')
        abstract = _get_data_from_xml(hitlist, 'abstract')
        doi = _get_data_from_xml(hitlist, 'DOI')
        eprintid = _get_data_from_xml(hitlist, 'eprintid')
        authors = _get_data_from_xml(hitlist, 'author')
        # put into AstroPy Table
        t = Table()
        t['title'] = titles
        t['bibcode'] = bibcode
        t['journal'] = journal
        t['volume'] = volume
        t['pubdate'] = pubdate
        t['page'] = page
        t['score'] = score
        t['citations'] = citations
        t['abstract'] = abstract
        t['doi'] = doi
        t['eprintid'] = eprintid
        t['authors'] = authors

        return t

    def _args_to_payload(self, query_string):
        # convert arguments to a valid requests payload
        # i.e. a dictionary
        return {'qsearch': query_string, 'data_type': 'XML'}

ADS = ADSClass()
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
VizieR Query Tool
-----------------

:Author: Julien Woillez (jwoillez@gmail.com)
"""

import httplib
import cStringIO
import numpy
from astropy.io.vo.table import parse
from astropy.table import Table

def vizquery(query, server="vizier.cfa.harvard.edu"):
    """
    ## Documentation
    #
    # * http://vizier.u-strasbg.fr/doc/asu-summary.htx
    # * http://vizier.u-strasbg.fr/vizier/vizHelp/menu.htx
    """

    # Always add calculated _RAJ2000 & _DEJ2000 to the query.
    # This is used for cross correlations between queries
    if query.has_key('-out.add'):
        query["-out.add"] += ['_RAJ2000', '_DEJ2000']
    else:
        query["-out.add"]  = ['_RAJ2000', '_DEJ2000']

    # Assemble the actual query
    body = []
    for (key,value) in query.items():
        if type(value) is str:
            body += ["%s=%s"%(key, value)]
        elif type(value) is Table: # Value is a table, convert it to a string, list of positions
            pos = []
            for elem in numpy.array(value, copy=False):
                pos += ["%.8f%+.8f"%(elem['_RAJ2000'],elem['_DEJ2000'])] # Position with the format: _RAJ2000+_DEJ2000
            body += ["-out.add=_q"] # This calculated index is a reference to the input table
            body += ["%s=%s"%(key, "<<;"+";".join(pos))] # The proper convention: <<;pos1;pos2;pos3
        elif type(value) is list: # Value is a list, join it with commas
            body += ["%s=%s"%(key, ",".join(value))]
        else:
            raise Exception, "Don't know how to handle %s"%repr(value)
    body = "\r\n".join(body)

    # Fetch the VOTABLE corresponding to the query
    http = httplib.HTTPConnection(server)
    http.request("POST", "/viz-bin/votable", body=body)
    resp = http.getresponse()
    voTable = parse(cStringIO.StringIO(resp.read()), pedantic=False)

    # Convert VOTABLE into a list of astropy Table.
    tableList = []
    for voTreeTable in voTable.iter_tables():
        if len(voTreeTable.array)>0:
            # Table names come from the VOTABLE fields
            names = []
            for field in voTreeTable.fields:
                names += [field.name.encode('ascii')]
            # Table data come from the VOTABLE record array
            data = voTreeTable.array
            # Add Table to the list
            tableList += [Table(data, names)]

    # Merge the Table list
    table = tableList[0]
    if len(tableList)>1:
        for t in tableList[1:]:
            if len(t)>0:
                for row in t:
                    table.add_row(row)

    return table

if __name__ == '__main__':

    #Find all AGNs in Veron & Cetty with Vmag in [5.0; 11.0]
    query = {}
    query["-source"] = "VII/258/vv10"
    query["-out"] = ["Name", "Sp", "Vmag"]
    query["Vmag"] = "5.0..11.0"
    table1 = vizquery(query)

    #Find sources in 2MASS matching the AGNs positions to within 2 arcsec
    query = {}
    query["-source"] = "II/246/out"
    query["-out"] = ["RAJ2000", "DEJ2000", "2MASS", "Kmag"]
    query["-c.rs"] = "2"
    query["-c"] = table1
    table2 = vizquery(query)

    print(table1)
    print(table2)

<commit_msg>Changed VizieR server default to "vizier.u-strasbg.fr".<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
VizieR Query Tool
-----------------

:Author: Julien Woillez (jwoillez@gmail.com)
"""

import httplib
import cStringIO
import numpy
from astropy.io.vo.table import parse
from astropy.table import Table

def vizquery(query, server="vizier.u-strasbg.fr"):
    """
    ## Documentation
    #
    # * http://vizier.u-strasbg.fr/doc/asu-summary.htx
    # * http://vizier.u-strasbg.fr/vizier/vizHelp/menu.htx
    """

    # Always add calculated _RAJ2000 & _DEJ2000 to the query.
    # This is used for cross correlations between queries
    if query.has_key('-out.add'):
        query["-out.add"] += ['_RAJ2000', '_DEJ2000']
    else:
        query["-out.add"]  = ['_RAJ2000', '_DEJ2000']

    # Assemble the actual query
    body = []
    for (key,value) in query.items():
        if type(value) is str:
            body += ["%s=%s"%(key, value)]
        elif type(value) is Table: # Value is a table, convert it to a string, list of positions
            pos = []
            for elem in numpy.array(value, copy=False):
                pos += ["%.8f%+.8f"%(elem['_RAJ2000'],elem['_DEJ2000'])] # Position with the format: _RAJ2000+_DEJ2000
            body += ["-out.add=_q"] # This calculated index is a reference to the input table
            body += ["%s=%s"%(key, "<<;"+";".join(pos))] # The proper convention: <<;pos1;pos2;pos3
        elif type(value) is list: # Value is a list, join it with commas
            body += ["%s=%s"%(key, ",".join(value))]
        else:
            raise Exception, "Don't know how to handle %s"%repr(value)
    body = "\r\n".join(body)

    # Fetch the VOTABLE corresponding to the query
    http = httplib.HTTPConnection(server)
    http.request("POST", "/viz-bin/votable", body=body)
    resp = http.getresponse()
    voTable = parse(cStringIO.StringIO(resp.read()), pedantic=False)

    # Convert VOTABLE into a list of astropy Table.
    tableList = []
    for voTreeTable in voTable.iter_tables():
        if len(voTreeTable.array)>0:
            # Table names come from the VOTABLE fields
            names = []
            for field in voTreeTable.fields:
                names += [field.name.encode('ascii')]
            # Table data come from the VOTABLE record array
            data = voTreeTable.array
            # Add Table to the list
            tableList += [Table(data, names)]

    # Merge the Table list
    table = tableList[0]
    if len(tableList)>1:
        for t in tableList[1:]:
            if len(t)>0:
                for row in t:
                    table.add_row(row)

    return table

if __name__ == '__main__':

    #Find all AGNs in Veron & Cetty with Vmag in [5.0; 11.0]
    query = {}
    query["-source"] = "VII/258/vv10"
    query["-out"] = ["Name", "Sp", "Vmag"]
    query["Vmag"] = "5.0..11.0"
    table1 = vizquery(query)

    #Find sources in 2MASS matching the AGNs positions to within 2 arcsec
    query = {}
    query["-source"] = "II/246/out"
    query["-out"] = ["RAJ2000", "DEJ2000", "2MASS", "Kmag"]
    query["-c.rs"] = "2"
    query["-c"] = table1
    table2 = vizquery(query)

    print(table1)
    print(table2)

<|endoftext|>


<commit_before>from numpy import *
import numpy as np
import os,platform
import lib_utilities as util

def get_marginal_rates(times,rates,grid):
	mr = np.zeros(len(grid))
	for i in range(len(times)-1):
		grid_ind = np.intersect1d((grid<=times[i]).nonzero()[0],(grid>=times[i+1]).nonzero()[0])
		mr[grid_ind.astype(int)] = rates[i]
	return mr

#def print_R_vec(name,v):
#	new_v=[]
#	for j in range(0,len(v)):
#		value=v[j]
#		if isnan(v[j]): value="NA"
#		new_v.append(value)
#
#	vec="%s=c(%s, " % (name,new_v[0])
#	for j in range(1,len(v)-1): vec += "%s," % (new_v[j])
#	vec += "%s)"  % (new_v[j+1])
#	return vec
#
#def calcHPD(data, level) :
#	assert (0 < level < 1)
#	d = list(data)
#	d.sort()
#	nData = len(data)
#	nIn = int(round(level * nData))
#	if nIn < 2 :
#		raise RuntimeError("not enough data")
#	i = 0
#	r = d[i+nIn-1] - d[i]
#	for k in range(len(d) - (nIn - 1)) :
#		rk = d[k+nIn-1] - d[k]
#		if rk < r :
#			r = rk
#			i = k
#	assert 0 <= i <= i+nIn-1 < len(d)
#	return (d[i], d[i+nIn-1])

def r_plot_code(res,wd,name_file,alpha=0.5,plot_title="RTT plot"):
	data  = "library(scales)\ntrans=%s" % (alpha)
	if platform.system() == "Windows" or platform.system() == "Microsoft":
		data+= "\n\npdf(file='%s\%s_RTT.pdf',width=0.6*9, height=0.6*14)\npar(mfrow=c(2,1))" % (wd,name_file) # 9
	else:
		data+= "\n\npdf(file='%s/%s_RTT.pdf',width=0.6*9, height=0.6*14)\npar(mfrow=c(2,1))" % (wd,name_file) # 9
	data += util.print_R_vec('\nage',   -res[:,0])
	data += util.print_R_vec('\nL_mean', res[:,1])
	data += util.print_R_vec('\nL_hpd_m',res[:,2])
	data += util.print_R_vec('\nL_hpd_M',res[:,3])
	data += util.print_R_vec('\nM_mean', res[:,4])
	data += util.print_R_vec('\nM_hpd_m',res[:,5])
	data += util.print_R_vec('\nM_hpd_M',res[:,6])
	max_x_axis,min_x_axis = -max(res[:,0]),-min(res[:,0])
	data += "\nplot(age,age,type = 'n', ylim = c(0, %s), xlim = c(%s,%s), ylab = 'Speciation rate', xlab = 'Ma',main='%s' )" \
		% (1.1*max(res[:,3]),max_x_axis,min_x_axis,plot_title)
	data += """\nlines(rev(age), rev(L_mean), col = "#4c4cec", lwd=3)"""
	data += """\npolygon(c(age, rev(age)), c(L_hpd_M, rev(L_hpd_m)), col = alpha("#4c4cec",trans), border = NA)"""
	data += "\nplot(age,age,type = 'n', ylim = c(0, %s), xlim = c(%s,%s), ylab = 'Extinction rate', xlab = 'Ma' )" \
		% (1.1*max(res[:,6]),max_x_axis,min_x_axis)
	data += """\nlines(rev(age), rev(M_mean), col = "#e34a33", lwd=3)"""
	data += """\npolygon(c(age, rev(age)), c(M_hpd_M, rev(M_hpd_m)), col = alpha("#e34a33",trans), border = NA)"""
	data += "\nn <- dev.off()"
	return data

def RTTplot_high_res(f,grid_cell_size=1.,burnin=0,max_age=0):
	wd = "%s" % os.path.dirname(f)
	name_file=os.path.splitext(os.path.basename(f))[0]
	t=loadtxt(f, skiprows=max(1,burnin))
	head = next(open(f)).split() # should be faster
	sp_ind = [head.index(s) for s in head if "lambda_" in s]
	ex_ind = [head.index(s) for s in head if "mu_" in s]
	root_ind  = head.index("root_age")
	death_ind = head.index("death_age")
	sp_shift_ind = [head.index(s) for s in head if "shift_sp_" in s]
	ex_shift_ind = [head.index(s) for s in head if "shift_ex_" in s]
	min_root_age = min(t[:,root_ind])
	if max_age> 0: min_root_age=max_age
	max_death_age= max(t[:,death_ind])
	n_bins= int((min_root_age-max_death_age)/grid_cell_size)
	grid = np.linspace(min_root_age,max_death_age,n_bins)
	n_samples = np.shape(t)[0]
	m_sp_matrix = np.zeros((n_samples,n_bins))
	m_ex_matrix = np.zeros((n_samples,n_bins))
	print "Extracting marginal rates..."
	for i in range(n_samples):
		l_shift_times = np.array([min_root_age]+ list(t[i,sp_shift_ind])+[max_death_age])
		l_rates = t[i,sp_ind]
		m_sp_matrix[i] = get_marginal_rates(l_shift_times,l_rates,grid)
		m_shift_times = np.array([min_root_age]+ list(t[i,ex_shift_ind])+[max_death_age])
		m_rates = t[i,ex_ind]
		m_ex_matrix[i] = get_marginal_rates(m_shift_times,m_rates,grid)

	res = np.zeros((n_bins,7)) # times, l, lM, lm, m, mM, mm
	print "Calculating HPDs..."
	for i in range(n_bins):
		l_HPD  = list(util.calcHPD(m_sp_matrix[:,i],0.95))
		l_mean = mean(m_sp_matrix[:,i])
		m_HPD  = list(util.calcHPD(m_ex_matrix[:,i],0.95))
		m_mean = mean(m_ex_matrix[:,i])
		res[i] = np.array([grid[i],l_mean]+l_HPD+[m_mean]+m_HPD)

	Rfile=r_plot_code(res,wd,name_file)
	out="%s/%s_RTT.r" % (wd,name_file)
	newfile = open(out, "w")
	newfile.writelines(Rfile)
	newfile.close()
	print "\nAn R script with the source for the RTT plot was saved as: %sRTT.r\n(in %s)" % (name_file, wd)
	if platform.system() == "Windows" or platform.system() == "Microsoft":
		cmd="cd %s; Rscript %s\%s_RTT.r" % (wd,wd,name_file)
	else:
		cmd="cd %s; Rscript %s/%s_RTT.r" % (wd,wd,name_file)
	os.system(cmd)
	print "done\n"

# args
#burnin=0
#grid_cell_size=.1 # in Myr (approximated)
#f= "/Users/daniele/Desktop/try/catalina_temp/par_est/combined_10mcmc_files.log"
#max_age=23.03
#RTTplot_high_res(f,grid_cell_size,burnin,max_age)<commit_msg>function to plot RTT at any resolution from a pyrate analysis<commit_after>from numpy import *
import numpy as np
import os,platform
import lib_utilities as util
import csv

def get_marginal_rates(times,rates,grid):
	mr = np.zeros(len(grid))
	for i in range(len(times)-1):
		grid_ind = np.intersect1d((grid<=times[i]).nonzero()[0],(grid>=times[i+1]).nonzero()[0])
		mr[grid_ind.astype(int)] = rates[i]
	return mr

def r_plot_code(res,wd,name_file,alpha=0.5,plot_title="RTT plot"):
	data  = "library(scales)\ntrans=%s" % (alpha)
	if platform.system() == "Windows" or platform.system() == "Microsoft":
		data+= "\n\npdf(file='%s\%s_RTT.pdf',width=0.6*9, height=0.6*14)\npar(mfrow=c(2,1))" % (wd,name_file) # 9
	else:
		data+= "\n\npdf(file='%s/%s_RTT.pdf',width=0.6*9, height=0.6*14)\npar(mfrow=c(2,1))" % (wd,name_file) # 9
	data += util.print_R_vec('\nage',   -res[:,0])
	data += util.print_R_vec('\nL_mean', res[:,1])
	data += util.print_R_vec('\nL_hpd_m',res[:,2])
	data += util.print_R_vec('\nL_hpd_M',res[:,3])
	data += util.print_R_vec('\nM_mean', res[:,4])
	data += util.print_R_vec('\nM_hpd_m',res[:,5])
	data += util.print_R_vec('\nM_hpd_M',res[:,6])
	max_x_axis,min_x_axis = -max(res[:,0]),-min(res[:,0])
	data += "\nplot(age,age,type = 'n', ylim = c(0, %s), xlim = c(%s,%s), ylab = 'Speciation rate', xlab = 'Ma',main='%s' )" \
		% (1.1*max(res[:,3]),max_x_axis,min_x_axis,plot_title)
	data += """\nlines(rev(age), rev(L_mean), col = "#4c4cec", lwd=3)"""
	data += """\npolygon(c(age, rev(age)), c(L_hpd_M, rev(L_hpd_m)), col = alpha("#4c4cec",trans), border = NA)"""
	data += "\nplot(age,age,type = 'n', ylim = c(0, %s), xlim = c(%s,%s), ylab = 'Extinction rate', xlab = 'Ma' )" \
		% (1.1*max(res[:,6]),max_x_axis,min_x_axis)
	data += """\nlines(rev(age), rev(M_mean), col = "#e34a33", lwd=3)"""
	data += """\npolygon(c(age, rev(age)), c(M_hpd_M, rev(M_hpd_m)), col = alpha("#e34a33",trans), border = NA)"""
	data += "\nn <- dev.off()"
	return data

def RTTplot_high_res(f,grid_cell_size=1.,burnin=0,max_age=0):
	wd = "%s" % os.path.dirname(f)
	name_file=os.path.splitext(os.path.basename(f))[0]
	t=loadtxt(f, skiprows=max(1,burnin))
	head = next(open(f)).split() # should be faster
	sp_ind = [head.index(s) for s in head if "lambda_" in s]
	ex_ind = [head.index(s) for s in head if "mu_" in s]
	root_ind  = head.index("root_age")
	death_ind = head.index("death_age")
	sp_shift_ind = [head.index(s) for s in head if "shift_sp_" in s]
	ex_shift_ind = [head.index(s) for s in head if "shift_ex_" in s]
	min_root_age = min(t[:,root_ind])
	if max_age> 0: min_root_age=max_age
	max_death_age= max(t[:,death_ind])
	n_bins= int((min_root_age-max_death_age)/grid_cell_size)
	grid = np.linspace(min_root_age,max_death_age,n_bins)
	n_samples = np.shape(t)[0]
	m_sp_matrix = np.zeros((n_samples,n_bins))
	m_ex_matrix = np.zeros((n_samples,n_bins))
	print "Extracting marginal rates..."
	for i in range(n_samples):
		l_shift_times = np.array([min_root_age]+ list(t[i,sp_shift_ind])+[max_death_age])
		l_rates = t[i,sp_ind]
		m_sp_matrix[i] = get_marginal_rates(l_shift_times,l_rates,grid)
		m_shift_times = np.array([min_root_age]+ list(t[i,ex_shift_ind])+[max_death_age])
		m_rates = t[i,ex_ind]
		m_ex_matrix[i] = get_marginal_rates(m_shift_times,m_rates,grid)

	res = np.zeros((n_bins,7)) # times, l, lM, lm, m, mM, mm
	print "Calculating HPDs..."
	for i in range(n_bins):
		l_HPD  = list(util.calcHPD(m_sp_matrix[:,i],0.95))
		l_mean = mean(m_sp_matrix[:,i])
		m_HPD  = list(util.calcHPD(m_ex_matrix[:,i],0.95))
		m_mean = mean(m_ex_matrix[:,i])
		res[i] = np.array([grid[i],l_mean]+l_HPD+[m_mean]+m_HPD)

	Rfile=r_plot_code(res,wd,name_file)
	out="%s/%s_RTT.r" % (wd,name_file)
	newfile = open(out, "w")
	newfile.writelines(Rfile)
	newfile.close()
	print "\nAn R script with the source for the RTT plot was saved as: %sRTT.r\n(in %s)" % (name_file, wd)
	if platform.system() == "Windows" or platform.system() == "Microsoft":
		cmd="cd %s; Rscript %s\%s_RTT.r" % (wd,wd,name_file)
	else:
		cmd="cd %s; Rscript %s/%s_RTT.r" % (wd,wd,name_file)
	os.system(cmd)

	save_HR_log_file=1
	if save_HR_log_file==1:
		print "Saving log file..."
		out="%s/%s_HR_marginal.log" % (wd,name_file)
		logfile = open(out, "w")
		head = ["iteration"]+["l_%s" % (i) for i in range(n_bins)]
		head +=["m_%s" % (i) for i in range(n_bins)]
		wlog=csv.writer(logfile, delimiter='\t')
		wlog.writerow(head)
		for i in range(n_samples):
			l = [i]+ list(m_sp_matrix[i,:]) + list(m_ex_matrix[i,:])
			wlog.writerow(l)
			logfile.flush()

	print "done\n"

# args
#burnin=0
#grid_cell_size=.1 # in Myr (approximated)
#f= "/Users/daniele/Desktop/try/catalina_temp/par_est/combined_10mcmc_files.log"
#max_age=23.03
#RTTplot_high_res(f,grid_cell_size,burnin,max_age)<|endoftext|>


<commit_before># MP-specific cache management
from pysnmp.proto import error

class AbstractMessageProcessingModel:
    messageProcessingModelID = None
    __stateReference = __msgID = 0L
    def __init__(self):
        self.__msgIdIndex = {}
        self.__stateReferenceIndex = {}
        self.__sendPduHandleIdx = {}
        # Message expiration mechanics
        self.__expirationQueue = {}
        self.__expirationTimer = 0L

    def prepareOutgoingMessage(self, msgAndPduDsp, mibInstrumController,
                               **kwargs): pass
    def prepareResponseMessage(self, msgAndPduDsp, mibInstrumController,
                               **kwargs): pass
    def prepareDataElements(self, msgAndPduDsp, mibInstrumController,
                            wholeMsg): pass

    def _newStateReference(self):
        AbstractMessageProcessingModel.__stateReference = (
            AbstractMessageProcessingModel.__stateReference + 1
            )
        return self.__stateReference

    # Server mode cache handling

    def _cachePushByStateRef(self, stateReference, **msgInfo):
        if self.__stateReferenceIndex.has_key(stateReference):
            raise error.ProtocolError(
                'Cache dup for stateReference=%s at %s' %
                (stateReference, self)
                )
        expireAt = self.__expirationTimer+50
        self.__stateReferenceIndex[stateReference] = ( msgInfo, expireAt )

        # Schedule to expire
        if not self.__expirationQueue.has_key(expireAt):
            self.__expirationQueue[expireAt] = {}
        if not self.__expirationQueue[expireAt].has_key('stateReference'):
            self.__expirationQueue[expireAt]['stateReference'] = {}
        self.__expirationQueue[expireAt]['stateReference'][stateReference] = 1
        self.__expireCaches()

    def _cachePopByStateRef(self, stateReference):
        cacheInfo = self.__stateReferenceIndex.get(stateReference)
        if cacheInfo is None:
            raise error.ProtocolError(
                'Cache miss for stateReference=%s at %s' %
                (stateReference, self)
                )
        del self.__stateReferenceIndex[stateReference]
        cacheEntry, expireAt = cacheInfo
        del self.__expirationQueue[expireAt]['stateReference'][stateReference]
        return cacheEntry

    # Client mode cache handling

    def _newMsgID(self):
        AbstractMessageProcessingModel.__msgID = (
            AbstractMessageProcessingModel.__msgID + 1
            )
        return self.__msgID

    def _cachePushByMsgId(self, msgId, **msgInfo):
        if self.__msgIdIndex.has_key(msgId):
            raise error.ProtocolError(
                'Cache dup for msgId=%s at %s' % (msgId, self)
                )
        expireAt = self.__expirationTimer+50
        self.__msgIdIndex[msgId] = ( msgInfo, expireAt )

        self.__sendPduHandleIdx[msgInfo['sendPduHandle']] = msgId

        # Schedule to expire
        if not self.__expirationQueue.has_key(expireAt):
            self.__expirationQueue[expireAt] = {}
        if not self.__expirationQueue[expireAt].has_key('msgId'):
            self.__expirationQueue[expireAt]['msgId'] = {}
        self.__expirationQueue[expireAt]['msgId'][msgId] = 1
        self.__expireCaches()

    def _cachePopByMsgId(self, msgId):
        cacheInfo = self.__msgIdIndex.get(msgId)
        if cacheInfo is None:
            raise error.ProtocolError(
                'Cache miss for msgId=%s at %s' % (msgId, self)
                )
        msgInfo, expireAt = cacheInfo
        del self.__sendPduHandleIdx[msgInfo['sendPduHandle']]
        del self.__msgIdIndex[msgId]
        cacheEntry, expireAt = cacheInfo
        del self.__expirationQueue[expireAt]['msgId'][msgId]
        return cacheEntry

    def _cachePopBySendPduHandle(self, sendPduHandle):
        self._cachePopByMsgId(self.__sendPduHandleIdx[sendPduHandle])

    def __expireCaches(self):
        # Uses internal clock to expire pending messages
        if self.__expirationQueue.has_key(self.__expirationTimer):
            cacheInfo = self.__expirationQueue[self.__expirationTimer]
            if cacheInfo.has_key('stateReference'):
                for stateReference in cacheInfo['stateReference'].keys():
                    del self.__stateReferenceIndex[stateReference]
            if cacheInfo.has_key('msgId'):
                for msgId in cacheInfo['msgId'].keys():
                    del self.__msgIdIndex[msgId]
            del self.__expirationQueue[self.__expirationTimer]
        self.__expirationTimer = self.__expirationTimer + 1

    def releaseStateInformation(self, sendPduHandle):
        self._cachePopBySendPduHandle(sendPduHandle)

<commit_msg>ignore cache misses (may need rework)<commit_after># MP-specific cache management
from pysnmp.proto import error

class AbstractMessageProcessingModel:
    messageProcessingModelID = None
    __stateReference = __msgID = 0L
    def __init__(self):
        self.__msgIdIndex = {}
        self.__stateReferenceIndex = {}
        self.__sendPduHandleIdx = {}
        # Message expiration mechanics
        self.__expirationQueue = {}
        self.__expirationTimer = 0L

    def prepareOutgoingMessage(self, msgAndPduDsp, mibInstrumController,
                               **kwargs): pass
    def prepareResponseMessage(self, msgAndPduDsp, mibInstrumController,
                               **kwargs): pass
    def prepareDataElements(self, msgAndPduDsp, mibInstrumController,
                            wholeMsg): pass

    def _newStateReference(self):
        AbstractMessageProcessingModel.__stateReference = (
            AbstractMessageProcessingModel.__stateReference + 1
            )
        return self.__stateReference

    # Server mode cache handling

    def _cachePushByStateRef(self, stateReference, **msgInfo):
        if self.__stateReferenceIndex.has_key(stateReference):
            raise error.ProtocolError(
                'Cache dup for stateReference=%s at %s' %
                (stateReference, self)
                )
        expireAt = self.__expirationTimer+50
        self.__stateReferenceIndex[stateReference] = ( msgInfo, expireAt )

        # Schedule to expire
        if not self.__expirationQueue.has_key(expireAt):
            self.__expirationQueue[expireAt] = {}
        if not self.__expirationQueue[expireAt].has_key('stateReference'):
            self.__expirationQueue[expireAt]['stateReference'] = {}
        self.__expirationQueue[expireAt]['stateReference'][stateReference] = 1
        self.__expireCaches()

    def _cachePopByStateRef(self, stateReference):
        cacheInfo = self.__stateReferenceIndex.get(stateReference)
        if cacheInfo is None:
            raise error.ProtocolError(
                'Cache miss for stateReference=%s at %s' %
                (stateReference, self)
                )
        del self.__stateReferenceIndex[stateReference]
        cacheEntry, expireAt = cacheInfo
        del self.__expirationQueue[expireAt]['stateReference'][stateReference]
        return cacheEntry

    # Client mode cache handling

    def _newMsgID(self):
        AbstractMessageProcessingModel.__msgID = (
            AbstractMessageProcessingModel.__msgID + 1
            )
        return self.__msgID

    def _cachePushByMsgId(self, msgId, **msgInfo):
        if self.__msgIdIndex.has_key(msgId):
            raise error.ProtocolError(
                'Cache dup for msgId=%s at %s' % (msgId, self)
                )
        expireAt = self.__expirationTimer+50
        self.__msgIdIndex[msgId] = ( msgInfo, expireAt )

        self.__sendPduHandleIdx[msgInfo['sendPduHandle']] = msgId

        # Schedule to expire
        if not self.__expirationQueue.has_key(expireAt):
            self.__expirationQueue[expireAt] = {}
        if not self.__expirationQueue[expireAt].has_key('msgId'):
            self.__expirationQueue[expireAt]['msgId'] = {}
        self.__expirationQueue[expireAt]['msgId'][msgId] = 1
        self.__expireCaches()

    def _cachePopByMsgId(self, msgId):
        cacheInfo = self.__msgIdIndex.get(msgId)
        if cacheInfo is None:
            raise error.ProtocolError(
                'Cache miss for msgId=%s at %s' % (msgId, self)
                )
        msgInfo, expireAt = cacheInfo
        del self.__sendPduHandleIdx[msgInfo['sendPduHandle']]
        del self.__msgIdIndex[msgId]
        cacheEntry, expireAt = cacheInfo
        del self.__expirationQueue[expireAt]['msgId'][msgId]
        return cacheEntry

    def _cachePopBySendPduHandle(self, sendPduHandle):
        if self.__sendPduHandleIdx.has_key(sendPduHandle):
            self._cachePopByMsgId(self.__sendPduHandleIdx[sendPduHandle])

    def __expireCaches(self):
        # Uses internal clock to expire pending messages
        if self.__expirationQueue.has_key(self.__expirationTimer):
            cacheInfo = self.__expirationQueue[self.__expirationTimer]
            if cacheInfo.has_key('stateReference'):
                for stateReference in cacheInfo['stateReference'].keys():
                    del self.__stateReferenceIndex[stateReference]
            if cacheInfo.has_key('msgId'):
                for msgId in cacheInfo['msgId'].keys():
                    del self.__msgIdIndex[msgId]
            del self.__expirationQueue[self.__expirationTimer]
        self.__expirationTimer = self.__expirationTimer + 1

    def releaseStateInformation(self, sendPduHandle):
        try:
            self._cachePopBySendPduHandle(sendPduHandle)
        except error.ProtocolError:
            pass # XXX maybe these should all follow some scheme?

<|endoftext|>


<commit_before>from collections import namedtuple
import os
import re
import shutil
import yaml

Credential = namedtuple("Credential", "name login password comment")

class CredentialExistsError(Exception):
    pass

class CredentialNotFoundError(Exception):
    pass

def expandpath(path, name, login):
    return os.path.join(path, name, "{}.pyssword".format(login))

def content(credential):
    return yaml.dump(credential)

def asdict(credential):
    return credential._asdict()

def asstring(credential):
    return "{} {} {}".format(
        credential.name,
        credential.login,
        credential.comment
    )

def exists(path, name, login):
    return True if os.path.isfile(expandpath(path, name, login)) else False

def clean(path, name, login):
    if exists(path, name, login):
        os.remove(expandpath(path, name, login))
    credential_dir = os.path.dirname(expandpath(path, name, login))
    if not os.listdir(credential_dir):
        shutil.rmtree(credential_dir)

def splitname(fullname):
    rgx = re.compile(r"(?:(?P<login>.+)?@)?(?P<name>.+)")
    if rgx.match(fullname):
        name = rgx.match(fullname).group("name")
        login = rgx.match(fullname).group("login")
        return name, login
    else:
        raise ValueError("Not a valid name")

def asfullname(name, login):
    return "{}@{}".format(login if login else "", name)
<commit_msg>Fix fullname syntax to handle login with (at) signs<commit_after>from collections import namedtuple
import os
import re
import shutil
import yaml

Credential = namedtuple("Credential", "name login password comment")

class CredentialExistsError(Exception):
    pass

class CredentialNotFoundError(Exception):
    pass

def expandpath(path, name, login):
    return os.path.join(path, name, "{}.pyssword".format(login))

def content(credential):
    return yaml.dump(credential)

def asdict(credential):
    return credential._asdict()

def asstring(credential):
    return "{} {} {}".format(
        credential.name,
        credential.login,
        credential.comment
    )

def exists(path, name, login):
    return True if os.path.isfile(expandpath(path, name, login)) else False

def clean(path, name, login):
    if exists(path, name, login):
        os.remove(expandpath(path, name, login))
    credential_dir = os.path.dirname(expandpath(path, name, login))
    if not os.listdir(credential_dir):
        shutil.rmtree(credential_dir)

def splitname(fullname):
    rgx = re.compile(r"(?:(?P<login>.+?@?.+)?@)?(?P<name>.+)")
    if rgx.match(fullname):
        name = rgx.match(fullname).group("name")
        login = rgx.match(fullname).group("login")
        return name, login
    else:
        raise ValueError("Not a valid name")

def asfullname(name, login):
    return "{}@{}".format(login if login else "", name)
<|endoftext|>


<commit_before># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
from __future__ import unicode_literals

from decimal import Decimal
import hashlib

import six
import typepy

import dataproperty as dp

from .error import InvalidDataError

class TableData(object):
    """
    Class to represent a table data structure.

    :param str table_name: Name of the table.
    :param list header_list: Table header names.
    :param list record_list: Table data records.
    """

    @property
    def table_name(self):
        """
        :return: Name of the table.
        :rtype: str
        """

        return self.__table_name

    @property
    def header_list(self):
        """
        :return: Table header names.
        :rtype: list
        """

        return self.__header_list

    @property
    def value_matrix(self):
        """
        :return: Table data records.
        :rtype: list
        """

        return self.__record_list

    @property
    def record_list(self):
        # alias property of value_matrix. this method will be deleted in the
        # future

        return self.value_matrix

    def __init__(
            self, table_name, header_list, record_list, is_strip_quote=False):

        self.__dp_extractor = dp.DataPropertyExtractor()
        self.__dp_extractor.strip_str_header = '"'
        if is_strip_quote:
            self.__dp_extractor.strip_str_value = '"'

        self.__table_name = table_name
        self.__header_list = header_list
        self.__record_list = self.__to_record_list(record_list)

    def __repr__(self):
        return "table_name={}, header_list={}, record_list={}".format(
            self.table_name, self.header_list, self.value_matrix)

    def __eq__(self, other):
        return all([
            self.table_name == other.table_name,
            self.header_list == other.header_list,
            all([
                all([
                    self.__compare_helper(lhs, rhs)
                    for lhs, rhs in zip(lhs_list, rhs_list)
                ])
                for lhs_list, rhs_list
                in zip(self.value_matrix, other.value_matrix)
            ]),
        ])

    def __ne__(self, other):
        return any([
            self.table_name != other.table_name,
            self.header_list != other.header_list,
            any([
                any([
                    not self.__compare_helper(lhs, rhs)
                    for lhs, rhs in zip(lhs_list, rhs_list)
                ])
                for lhs_list, rhs_list
                in zip(self.value_matrix, other.value_matrix)
            ]),
        ])

    def __hash__(self):
        body = (
            self.table_name +
            six.text_type(self.header_list) +
            six.text_type(self.value_matrix)
        )
        return hashlib.sha1(body.encode("utf-8")).hexdigest()

    def is_empty_header(self):
        """
        :return: |True| if the data :py:attr:`.header_list` is empty.
        :rtype: bool
        """

        return typepy.is_empty_sequence(self.header_list)

    def is_empty_record(self):
        """
        :return: |True| if the data :py:attr:`.value_matrix` is empty.
        :rtype: bool
        """

        return typepy.is_empty_sequence(self.value_matrix)

    def is_empty(self):
        """
        :return:
            |True| if the data :py:attr:`.header_list` or
            :py:attr:`.value_matrix` is empty.
        :rtype: bool
        """

        return any([self.is_empty_header(), self.is_empty_record()])

    def as_dict(self):
        """
        :return: Table data as a |dict| instance.
        :rtype: dict

        :Examples:

            .. code:: python

                from pytablereader import TableData

                TableData(
                    table_name="sample",
                    header_list=["a", "b"],
                    record_list=[[1, 2], [3.3, 4.4]]
                ).as_dict()

            .. code:: json

                {'sample': [{'a': 1, 'b': 2}, {'a': 3.3, 'b': 4.4}]}
        """

        self.__dp_extractor.float_type = float

        dict_body = []
        for value_list in self.value_matrix:
            if typepy.is_empty_sequence(value_list):
                continue

            dict_record = [
                (header, self.__dp_extractor.to_dataproperty(value).data)
                for header, value in zip(self.header_list, value_list)
                if value is not None
            ]

            if typepy.is_empty_sequence(dict_record):
                continue

            dict_body.append(dict(dict_record))

        return {self.table_name: dict_body}

    def asdict(self):

        # alias to as_dict method.
        # this method will be deleted in the future.
        return self.as_dict()

    def as_dataframe(self):
        """
        :return: Table data as a ``pandas.DataFrame`` instance.
        :rtype: pandas.DataFrame

        :Examples:

            :ref:`example-as-dataframe`

        .. note::
            ``pandas`` package required to execute this method.
        """

        import pandas

        dataframe = pandas.DataFrame(self.value_matrix)
        if not self.is_empty_header():
            dataframe.columns = self.header_list

        return dataframe

    @staticmethod
    def from_dataframe(dataframe, table_name=""):
        """
        Initialize TableData instance from a pandas.DataFrame instance.

        :param pandas.DataFrame dataframe:
        :param str table_name: Table name to create.
        """

        return TableData(
            table_name=table_name,
            header_list=list(dataframe.columns.values),
            record_list=dataframe.values.tolist())

    def __compare_helper(self, lhs, rhs):
        from typepy.type import Nan

        if Nan(lhs).is_type() and Nan(rhs).is_type():
            return True

        return lhs == rhs

    def __to_record(self, values):
        """
        Convert values to a record.

        :param values: Value to be converted.
        :type values: |dict|/|namedtuple|/|list|/|tuple|
        :raises ValueError: If the ``value`` is invalid.
        """

        try:
            # dictionary to list
            return [
                self.__dp_extractor.to_dataproperty(values.get(header)).data
                for header in self.header_list
            ]
        except AttributeError:
            pass

        try:
            # namedtuple to list
            dict_value = values._asdict()
            return [
                self.__dp_extractor.to_dataproperty(
                    dict_value.get(header)).data
                for header in self.header_list
            ]
        except AttributeError:
            pass

        try:
            return [
                self.__dp_extractor.to_dataproperty(value).data
                for value in values
            ]
        except TypeError:
            raise InvalidDataError(
                "record must be a list or tuple: actual={}".format(values))

    def __to_record_list(self, record_list):
        """
        Convert matrix to records
        """

        self.__dp_extractor.float_type = Decimal

        if typepy.is_empty_sequence(self.header_list):
            return record_list

        return [
            self.__to_record(record)
            for record in record_list
        ]
<commit_msg>Modify deprecation message<commit_after># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
from __future__ import unicode_literals

from decimal import Decimal
import hashlib
import warnings

import six
import typepy

import dataproperty as dp

from .error import InvalidDataError

class TableData(object):
    """
    Class to represent a table data structure.

    :param str table_name: Name of the table.
    :param list header_list: Table header names.
    :param list record_list: Table data records.
    """

    @property
    def table_name(self):
        """
        :return: Name of the table.
        :rtype: str
        """

        return self.__table_name

    @property
    def header_list(self):
        """
        :return: Table header names.
        :rtype: list
        """

        return self.__header_list

    @property
    def value_matrix(self):
        """
        :return: Table data records.
        :rtype: list
        """

        return self.__record_list

    @property
    def record_list(self):
        # alias property of value_matrix. this method will be deleted in the
        # future

        return self.value_matrix

    def __init__(
            self, table_name, header_list, record_list, is_strip_quote=False):

        self.__dp_extractor = dp.DataPropertyExtractor()
        self.__dp_extractor.strip_str_header = '"'
        if is_strip_quote:
            self.__dp_extractor.strip_str_value = '"'

        self.__table_name = table_name
        self.__header_list = header_list
        self.__record_list = self.__to_record_list(record_list)

    def __repr__(self):
        return "table_name={}, header_list={}, record_list={}".format(
            self.table_name, self.header_list, self.value_matrix)

    def __eq__(self, other):
        return all([
            self.table_name == other.table_name,
            self.header_list == other.header_list,
            all([
                all([
                    self.__compare_helper(lhs, rhs)
                    for lhs, rhs in zip(lhs_list, rhs_list)
                ])
                for lhs_list, rhs_list
                in zip(self.value_matrix, other.value_matrix)
            ]),
        ])

    def __ne__(self, other):
        return any([
            self.table_name != other.table_name,
            self.header_list != other.header_list,
            any([
                any([
                    not self.__compare_helper(lhs, rhs)
                    for lhs, rhs in zip(lhs_list, rhs_list)
                ])
                for lhs_list, rhs_list
                in zip(self.value_matrix, other.value_matrix)
            ]),
        ])

    def __hash__(self):
        body = (
            self.table_name +
            six.text_type(self.header_list) +
            six.text_type(self.value_matrix)
        )
        return hashlib.sha1(body.encode("utf-8")).hexdigest()

    def is_empty_header(self):
        """
        :return: |True| if the data :py:attr:`.header_list` is empty.
        :rtype: bool
        """

        return typepy.is_empty_sequence(self.header_list)

    def is_empty_record(self):
        """
        :return: |True| if the data :py:attr:`.value_matrix` is empty.
        :rtype: bool
        """

        return typepy.is_empty_sequence(self.value_matrix)

    def is_empty(self):
        """
        :return:
            |True| if the data :py:attr:`.header_list` or
            :py:attr:`.value_matrix` is empty.
        :rtype: bool
        """

        return any([self.is_empty_header(), self.is_empty_record()])

    def as_dict(self):
        """
        :return: Table data as a |dict| instance.
        :rtype: dict

        :Examples:

            .. code:: python

                from pytablereader import TableData

                TableData(
                    table_name="sample",
                    header_list=["a", "b"],
                    record_list=[[1, 2], [3.3, 4.4]]
                ).as_dict()

            .. code:: json

                {'sample': [{'a': 1, 'b': 2}, {'a': 3.3, 'b': 4.4}]}
        """

        self.__dp_extractor.float_type = float

        dict_body = []
        for value_list in self.value_matrix:
            if typepy.is_empty_sequence(value_list):
                continue

            dict_record = [
                (header, self.__dp_extractor.to_dataproperty(value).data)
                for header, value in zip(self.header_list, value_list)
                if value is not None
            ]

            if typepy.is_empty_sequence(dict_record):
                continue

            dict_body.append(dict(dict_record))

        return {self.table_name: dict_body}

    def asdict(self):
        warnings.warn(
            "asdict() will be deleted in the future, use as_dict instead.",
            DeprecationWarning
        )

        return self.as_dict()

    def as_dataframe(self):
        """
        :return: Table data as a ``pandas.DataFrame`` instance.
        :rtype: pandas.DataFrame

        :Examples:

            :ref:`example-as-dataframe`

        .. note::
            ``pandas`` package required to execute this method.
        """

        import pandas

        dataframe = pandas.DataFrame(self.value_matrix)
        if not self.is_empty_header():
            dataframe.columns = self.header_list

        return dataframe

    @staticmethod
    def from_dataframe(dataframe, table_name=""):
        """
        Initialize TableData instance from a pandas.DataFrame instance.

        :param pandas.DataFrame dataframe:
        :param str table_name: Table name to create.
        """

        return TableData(
            table_name=table_name,
            header_list=list(dataframe.columns.values),
            record_list=dataframe.values.tolist())

    def __compare_helper(self, lhs, rhs):
        from typepy.type import Nan

        if Nan(lhs).is_type() and Nan(rhs).is_type():
            return True

        return lhs == rhs

    def __to_record(self, values):
        """
        Convert values to a record.

        :param values: Value to be converted.
        :type values: |dict|/|namedtuple|/|list|/|tuple|
        :raises ValueError: If the ``value`` is invalid.
        """

        try:
            # dictionary to list
            return [
                self.__dp_extractor.to_dataproperty(values.get(header)).data
                for header in self.header_list
            ]
        except AttributeError:
            pass

        try:
            # namedtuple to list
            dict_value = values._asdict()
            return [
                self.__dp_extractor.to_dataproperty(
                    dict_value.get(header)).data
                for header in self.header_list
            ]
        except AttributeError:
            pass

        try:
            return [
                self.__dp_extractor.to_dataproperty(value).data
                for value in values
            ]
        except TypeError:
            raise InvalidDataError(
                "record must be a list or tuple: actual={}".format(values))

    def __to_record_list(self, record_list):
        """
        Convert matrix to records
        """

        self.__dp_extractor.float_type = Decimal

        if typepy.is_empty_sequence(self.header_list):
            return record_list

        return [
            self.__to_record(record)
            for record in record_list
        ]
<|endoftext|>


<commit_before>from ctypes import *
import os

__all__ = ['Pytelemetry']

# pytelemetry interface
class Pytelemetry:
    """
        Main class for the pytelemetry library.

        If you intend to implement a new transport, it needs to implement these
        four methods:

        >>> read(bytesAmount)
        Expects: Amount of bytes to read. Will return less bytes if
        bytes amount is bigger than immediately available bytes amount
        Returns: A bytes instance containing read bytes

        >>> readable()
        Returns: Amount of immediately available bytes

        >>> write(data)
        Expects: a bytes instance containing the data to write
        Returns: True

        >>> writeable()
        Returns: True if transport is available for writing,
                 False otherwise

    """
    def __init__(self, transport):
        """
            Creates a new instance of the Pytelemetry class.

            :param transport: A transport-compliant class. See Pytelemetry class
            documentation for more information
        """
        self.transport = transport
        self.callbacks = dict()
        self.default_callback = None

        if telemetry_use_c_api:
            self.api = TelemetryCBinding(self._on_frame)
        else:
            self.api = Telemetry()

    def publish(self, topic, data, datatype):
        """

        """
        self.api.publish(topic,data,datatype)

    # subscribe a callback to topic
    # Subscribing to None will call that function for any unsubscribed topic
    def subscribe(self, topic, cb):
        if topic:
            self.callbacks[topic] = cb
        else:
            self.default_callback = cb

    def update(self):
        self.api.update()

    def _on_frame(self, topic, payload)
        cb = None
        # Search if topic has a registered callback
        if topic in self.callbacks:
            cb = self.callbacks[topic]
        # else pick default callback
        else:
            cb = self.default_callback
        # check callback is valid
        if cb:
            cb(topic,payload)
<commit_msg>Typos + bugfixes<commit_after>from .telemetry.telemetry import Telemetry
from .telemetry.c_binding import TelemetryCBinding
__all__ = ['Pytelemetry']

_telemetry_use_c_api = True

# pytelemetry interface
class Pytelemetry:
    """
        Main class for the pytelemetry library.

        If you intend to implement a new transport, it needs to implement these
        four methods:

        >>> read(bytesAmount)
        Expects: Amount of bytes to read. Will return less bytes if
        bytes amount is bigger than immediately available bytes amount
        Returns: A bytes instance containing read bytes

        >>> readable()
        Returns: Amount of immediately available bytes

        >>> write(data)
        Expects: a bytes instance containing the data to write
        Returns: True

        >>> writeable()
        Returns: True if transport is available for writing,
                 False otherwise

    """
    def __init__(self, transport):
        """
            Creates a new instance of the Pytelemetry class.

            :param transport: A transport-compliant class. See Pytelemetry class
            documentation for more information
        """

        self.callbacks = dict()
        self.default_callback = None

        if _telemetry_use_c_api:
            self.api = TelemetryCBinding(transport,self._on_frame)
        else:
            self.api = Telemetry()

    def publish(self, topic, data, datatype):
        """

        """
        self.api.publish(topic,data,datatype)

    # subscribe a callback to topic
    # Subscribing to None will call that function for any unsubscribed topic
    def subscribe(self, topic, cb):
        if topic:
            self.callbacks[topic] = cb
        else:
            self.default_callback = cb

    def update(self):
        self.api.update()

    def _on_frame(self, topic, payload):
        cb = None
        # Search if topic has a registered callback
        if topic in self.callbacks:
            cb = self.callbacks[topic]
        # else pick default callback
        else:
            cb = self.default_callback
        # check callback is valid
        if cb:
            cb(topic,payload)
<|endoftext|>