<commit_before>'''
Created on Apr 24, 2013

@author: shroffk
'''
import os.path
import re
import traceback
from xml.dom import minidom
import sys
from optparse import OptionParser
from ConfigParser import SafeConfigParser
from tempfile import NamedTemporaryFile, mkstemp
'''
This Script

1. automates the process of creating the plugin.list and feature.list files needed by the build.

2. It tries to evaluate the dependencies as defined in the MANIFEST.MF files for the plugins and checks
   if these dependencies are resolved/included in the features of the product
'''

def createDependencyMap(repoDir):
    '''
    Read the manifest file create the dependency tree
    It will get all the Manifest files in the repoDir and create tree
    of the dependencies of each plugins.

    {id: {name:'',
         file:'complete file path',
         dependencies:[,,]
         }
    }
    '''
    dependencyStack = {}
    for dirpath, dirnames, filenames in os.walk(os.path.normpath(repoDir)):
        for completefilename in [ os.path.join(dirpath, f) for f in filenames if f.endswith(".MF")]:
            name = ''
            dependencies = []
            fileString = open(completefilename, 'r').read().replace(',\n', ',')
            for line in fileString.split('\n'):
                m = re.search('Bundle-SymbolicName: (.*)', line)
                if m:
                    name = m.group(1).split(';')[0].strip()
                m2 = re.match('Require-Bundle: (.*)', line)
                if m2:
                    for dependency in m2.group(1).split(','):
                        dependencyDetails = dependency.split(';')
                        details = {}
                        for detail in dependencyDetails:
                            matchVersion = re.match('bundle-version="(.*)"', detail.strip())
                            if matchVersion:
                                details['version'] = matchVersion.group(1)
                            matchResolution = re.match('resolution:=(.*)', detail.strip())
                            if matchResolution:
                                details['resolution'] = matchResolution.group(1)
                        '''
                        Ignore optional dependencies, if resolution is not specified it is assumed to be required
                        '''
                        if 'resolution' not in details.keys() or details['resolution'] != 'optional':
                            dependencies.append(dependency.split(';')[0].strip())
            dependencyStack[name] = {'name':name, 'file':completefilename, 'dependencies':dependencies}
    return dependencyStack

def createDependantPluginMap(dependencyMap):
    '''
    {id:[, ,]}
    pluginId and list of dependent plugins
    This is sort of like a reserve map of the manifest map
    '''
    d = {}
    for id in dependencyMap.keys():
        d[id] = [id2 for id2 in dependencyMap.keys() if id in dependencyMap[id2]['dependencies']]
    return d

def getAllDependencies(pluginId, dependencyMap):
    '''
    Given a pluginId, return a complete list of plugin dependencies
    '''
    l = []
    if pluginId in dependencyMap.keys():
        l.append(pluginId)
        if dependencyMap[pluginId]['dependencies']:
            for id in dependencyMap[pluginId]['dependencies']:
                l = l + getAllDependencies(id)
    return l

def readFeatures(repoDir):
    '''
    Read the all feature.xml
    {id:
        {id:'feature.id',
        file:'complete file path'
        plugins:['plugin.ids']
        includes:['included.features']
        }
    }
    '''
    features = {}
    for dirpath, dirnames, filenames in os.walk(os.path.normpath(repoDir)):
        for completefilename in [ os.path.join(dirpath, f) for f in filenames if f.endswith("feature.xml")]:
            xmldoc = minidom.parse(completefilename)
            id = ''
            for feature in xmldoc.getElementsByTagName('feature'):
                id = feature._attrs[u'id'].value
                plugins = []
                for node in feature.getElementsByTagName('plugin'):
                    plugins.append(node._attrs[u'id'].value)
                includes = []
                for node in feature.getElementsByTagName('includes'):
                    includes.append(node._attrs[u'id'].value)
            features[id] = {'id':id, 'file':completefilename, 'plugins':plugins, 'includes':includes}
    return features

def getFeatureList(featureId, featureMap):
    '''
    Return a list with the featureId and the Id of all the included features
    '''
    l = []
    if featureId in featureMap.keys():
        l.append(featureId)
        if featureMap[featureId]['includes']:
            for includedFeatureId in featureMap[featureId]['includes']:
                l = l + getFeatureList(featureId=includedFeatureId, featureMap=featureMap)
    return l

def getPluginList(featureId, featureMap):
    '''
    Returns a list of all the plugins that are part of this feature and all the included features
    '''
    l = []
    if featureId in featureMap.keys():
        l = l + featureMap[featureId]['plugins']
        if featureMap[featureId]['includes']:
            for includedFeatureId in featureMap[featureId]['includes']:
                l = l + getPluginList(featureId=includedFeatureId, featureMap=featureMap)
    return l

def ifNoneReturnDefault(object, default):
    '''
    if the object is None or empty string then this function returns the default value
    '''
    if object == None and object != '':
        return default
    else:
        return object

class FakeSecHead(object):
    def __init__(self, fp):
        self.fp = fp
        self.sechead = '[FakeSection]\n'
    def readline(self):
        if self.sechead:
            try: return self.sechead
            finally: self.sechead = None
        else: return self.fp.readline()

if __name__ == '__main__':
    repoDir = 'C:\git\cs-studio'

    usage = 'usage: %prog -r C:\git\cs-studio -p C:\git\cs-studio\products\NSLS2 -b C:\git\cs-studio\build'
    parser = OptionParser(usage=usage)
    parser.add_option('-r', '--repoDir', \
                      action='store', type='string', dest='repoDir', \
                      help='the repoDir')
    parser.add_option('-b', '--buildDir', \
                      action='store', type='string', dest='buildDir', \
                      help='the buildProperties')
    parser.add_option('-p', '--productDir', \
                      action='store', type='string', dest='productDir', \
                      help='the productDir')
    parser.add_option('-c', '--confBuildDir', \
                      action='store', type='string', dest='confBuildDir', \
                      help='the build directory containing build.properties')
    opts, args = parser.parse_args()
    repoDir = ifNoneReturnDefault(opts.repoDir, repoDir)
    productDir = ifNoneReturnDefault(opts.productDir, 'C:\git\cs-studio\products\NSLS2')
    buildDir = ifNoneReturnDefault(opts.buildDir, 'C:\git\cs-studio\build')
    confBuildDir = ifNoneReturnDefault(opts.confBuildDir, productDir)

    '''
    find the build.properties
    '''
    cf = SafeConfigParser()
    cf.readfp(FakeSecHead(open(os.path.join(confBuildDir, 'build.properties'), 'r')))
    topLevelElementType = cf.get('FakeSection', 'topLevelElementType')
    topLevelElementId = cf.get('FakeSection', 'topLevelElementId')
    productFileName = os.path.split(os.path.normpath(cf.get('FakeSection', 'product')))[1]
    if not str(productFileName).endswith('.product'):
        print 'Invalid product specified in build.properties'
        sys.exit(-1)
    '''
    Find the product file for the product specified in the build.properties
    '''
    for dirpath, dirnames, filenames in os.walk(os.path.normpath(productDir)):
        l = [ os.path.join(dirpath, f) for f in filenames if f == productFileName]
        if len(l) == 1:
           productFilePath = l[0]
        elif len(l) > 1:
            print 'Found more than one copy of ', productFileName, 'at:'
            for f in l:
               print f
    print 'Building:', productFilePath

    '''
    TODO: need to include the plugin on the product and explicitly include the dependencies defined there.
    TODO: handle optional dependencies
    '''
    try:
        dependencyMap = createDependencyMap(repoDir=repoDir)
        featureMap = readFeatures(repoDir=repoDir)

        '''
        Get started from a project, get the direct dependencies from the manifest
        META-INF/MANIFEST.MF

        Get the product file and get the features needs from
        *.product
        '''
        reqFeatures = []
        reqPlugins = []
        if productFilePath:
            xmldoc = minidom.parse(productFilePath)
            for features in xmldoc.getElementsByTagName('features'):
                for feature in features.getElementsByTagName('feature'):
                    reqFeatures.append(feature._attrs[u'id'].value)
                    '''Add the top level element if it is a feature'''
            for plugins in xmldoc.getElementsByTagName('plugins'):
                for plugin in plugins.getElementsByTagName('plugin'):
                    reqPlugins.append(plugin._attrs[u'id'].value)

        if topLevelElementType == 'feature':
            reqFeatures.append(topLevelElementId)

        '''
        Since our products are feature based products we will use the features to make the two lists
        1. feature.list : all the features to be packaged with this product
        2. plugin.list : all the plugins to be packaged with this product
        '''
        featureList = []
        for featureId in reqFeatures:
            featureList = featureList + getFeatureList(featureId=featureId, featureMap=featureMap)
        featureList = list(set(featureList))
        featureList.sort()
        '''this is feature.list'''
        print featureList
        featureListFile = open(os.path.join(confBuildDir, 'features.list'), 'w')
        for feature in featureList:
            '''
            This additional step is to check id feature folder name is different from feature id
            '''
            featureDir = os.path.split(os.path.normpath(featureMap.get(feature)['file']))[0]
            featureDirName = os.path.split(featureDir)[1]
            if(feature != featureDirName):
                print 'Consider revising feature folder name', featureDirName, 'to match featureId:', feature
            featureListFile.write("%s\n" % featureDirName)
        featureListFile.close()
        '''
        The plugin list represents the plugins that are part of/included in the features
        TODO [optional] We do need to add the additional plugin defined in the product.xml
        '''
        pluginList = []
        for featureId in reqFeatures:
            pluginList = pluginList + getPluginList(featureId=featureId, featureMap=featureMap)
        pluginList = list(set(pluginList))
        pluginList.sort()
        '''
        This is the exhaustive plugin.list
        It includes all the plugins included in any of the features(and included features) used by the product'''
#        print pluginList
        '''
        sourcePluginList:
        This is the plugin.list consisting of source plugins from the cs-studio repo
        '''
        sourcePluginList = [ plugin for plugin in pluginList if plugin in dependencyMap.keys() ]
        pluginListFile = open(os.path.join(confBuildDir, 'plugins.list'), 'w')
        for plugin in sourcePluginList:
            '''
            This additional step is to check id plugin folder name is different from plugin id
            '''
            pluginDirName = (os.path.normpath(dependencyMap.get(plugin)['file'])).split(os.sep)[-3]
            if(plugin != pluginDirName):
                print 'Consider revising plugin folder name', pluginDirName, 'to match pluginId:', plugin
            pluginListFile.write("%s\n" % pluginDirName)
        pluginListFile.close()
        print len(sourcePluginList), sourcePluginList

        '''
        Now we try to resolve the plugins required by those listed in the pluginList
        using the dependencies defined in the META-INF/MANIFEST.MF

        We shall output the missing dependencies and also a list of plugins that need them.
        '''
        manifestDependencyList = []
        for plugin in pluginList:
            if plugin in  dependencyMap.keys():
                manifestDependencyList.append(plugin)
                manifestDependencyList = manifestDependencyList + dependencyMap[plugin]['dependencies']
        manifestDependencyList = list(set(manifestDependencyList))
        manifestDependencyList.sort()
        '''
        Now we remove the plugins that from eclipse/deltapack
        '''
        reducedManifestDependencyList = [ plugin for plugin in manifestDependencyList if plugin in dependencyMap.keys() ]
        print len(reducedManifestDependencyList), reducedManifestDependencyList

        dependentPlugins = createDependantPluginMap(dependencyMap=dependencyMap)
        for missingDependency in [ plugin for plugin in reducedManifestDependencyList if plugin not in sourcePluginList ]:
            print 'Missing dependency: ' + missingDependency
            if missingDependency in dependentPlugins.keys():
                print 'Required by: '
                print str(dependentPlugins.get(missingDependency)).strip('[]')
    except:
        print 'Exception'
        traceback.print_exc()
        sys.exit(-1)
    pass
<commit_msg>correction on scan dependency<commit_after>'''
Created on Apr 24, 2013

@author: shroffk
'''
import os.path
import re
import traceback
from xml.dom import minidom
import sys
from optparse import OptionParser
from ConfigParser import SafeConfigParser
from tempfile import NamedTemporaryFile, mkstemp
import mmap
'''
This Script

1. automates the process of creating the plugin.list and feature.list files needed by the build.

2. It tries to evaluate the dependencies as defined in the MANIFEST.MF files for the plugins and checks
   if these dependencies are resolved/included in the features of the product
'''

def createDependencyMap(repoDir):
    '''
    Read the manifest file create the dependency tree
    It will get all the Manifest files in the repoDir and create tree
    of the dependencies of each plugins.

    {id: {name:'',
         file:'complete file path',
         dependencies:[,,]
         }
    }
    '''
    dependencyStack = {}
    for dirpath, dirnames, filenames in os.walk(os.path.normpath(repoDir)):
        for completefilename in [ os.path.join(dirpath, f) for f in filenames if f.endswith(".MF")]:
            name = ''
            dependencies = []
            fileString = open(completefilename, 'r').read().replace(',\n', ',')
            for line in fileString.split('\n'):
                m = re.search('Bundle-SymbolicName: (.*)', line)
                if m:
                    name = m.group(1).split(';')[0].strip()
                m2 = re.match('Require-Bundle: (.*)', line)
                if m2:
                    for dependency in m2.group(1).split(','):
                        dependencyDetails = dependency.split(';')
                        details = {}
                        for detail in dependencyDetails:
                            matchVersion = re.match('bundle-version="(.*)"', detail.strip())
                            if matchVersion:
                                details['version'] = matchVersion.group(1)
                            matchResolution = re.match('resolution:=(.*)', detail.strip())
                            if matchResolution:
                                details['resolution'] = matchResolution.group(1)
                        '''
                        Ignore optional dependencies, if resolution is not specified it is assumed to be required
                        '''
                        if 'resolution' not in details.keys() or details['resolution'] != 'optional':
                            dependencies.append(dependency.split(';')[0].strip())
            dependencyStack[name] = {'name':name, 'file':completefilename, 'dependencies':dependencies}
    return dependencyStack

def createDependantPluginMap(dependencyMap):
    '''
    {id:[, ,]}
    pluginId and list of dependent plugins
    This is sort of like a reserve map of the manifest map
    '''
    d = {}
    for id in dependencyMap.keys():
        d[id] = [id2 for id2 in dependencyMap.keys() if id in dependencyMap[id2]['dependencies']]
    return d

def getAllDependencies(pluginId, dependencyMap):
    '''
    Given a pluginId, return a complete list of plugin dependencies
    '''
    l = []
    if pluginId in dependencyMap.keys():
        l.append(pluginId)
        if dependencyMap[pluginId]['dependencies']:
            for id in dependencyMap[pluginId]['dependencies']:
                l = l + getAllDependencies(id)
    return l

def readFeatures(repoDir):
    '''
    Read the all feature.xml
    {id:
        {id:'feature.id',
        file:'complete file path'
        plugins:['plugin.ids']
        includes:['included.features']
        }
    }
    '''
    features = {}
    for dirpath, dirnames, filenames in os.walk(os.path.normpath(repoDir)):
        for completefilename in [ os.path.join(dirpath, f) for f in filenames if f.endswith("feature.xml")]:
            xmldoc = minidom.parse(completefilename)
            id = ''
            for feature in xmldoc.getElementsByTagName('feature'):
                id = feature._attrs[u'id'].value
                plugins = []
                for node in feature.getElementsByTagName('plugin'):
                    plugins.append(node._attrs[u'id'].value)
                includes = []
                for node in feature.getElementsByTagName('includes'):
                    includes.append(node._attrs[u'id'].value)
            features[id] = {'id':id, 'file':completefilename, 'plugins':plugins, 'includes':includes}
    return features

def getFeatureList(featureId, featureMap):
    '''
    Return a list with the featureId and the Id of all the included features
    '''
    l = []
    if featureId in featureMap.keys():
        l.append(featureId)
        if featureMap[featureId]['includes']:
            for includedFeatureId in featureMap[featureId]['includes']:
                l = l + getFeatureList(featureId=includedFeatureId, featureMap=featureMap)
    return l

def getPluginList(featureId, featureMap):
    '''
    Returns a list of all the plugins that are part of this feature and all the included features
    '''
    l = []
    if featureId in featureMap.keys():
        l = l + featureMap[featureId]['plugins']
        if featureMap[featureId]['includes']:
            for includedFeatureId in featureMap[featureId]['includes']:
                l = l + getPluginList(featureId=includedFeatureId, featureMap=featureMap)
    return l

def ifNoneReturnDefault(object, default):
    '''
    if the object is None or empty string then this function returns the default value
    '''
    if object == None and object != '':
        return default
    else:
        return object

class FakeSecHead(object):
    def __init__(self, fp):
        self.fp = fp
        self.sechead = '[FakeSection]\n'
    def readline(self):
        if self.sechead:
            try: return self.sechead
            finally: self.sechead = None
        else:
            return self.fp.readline()

def check(filetmp, search):
        datafile = file(filetmp)
        found = False #this isn't really necessary
        for line in datafile:
            if search in line:
                #found = True #not necessary
                return line
        return '' #because you finished the search without finding anything

if __name__ == '__main__':
    repoDir = 'C:\git\cs-studio'

    usage = 'usage: %prog -r C:\git\cs-studio -p C:\git\cs-studio\products\NSLS2 -b C:\git\cs-studio\build'
    parser = OptionParser(usage=usage)
    parser.add_option('-r', '--repoDir', \
                      action='store', type='string', dest='repoDir', \
                      help='the repoDir')
    parser.add_option('-b', '--buildDir', \
                      action='store', type='string', dest='buildDir', \
                      help='the buildProperties')
    parser.add_option('-p', '--productDir', \
                      action='store', type='string', dest='productDir', \
                      help='the productDir')
    parser.add_option('-c', '--confBuildDir', \
                      action='store', type='string', dest='confBuildDir', \
                      help='the build directory containing build.properties')
    opts, args = parser.parse_args()
    repoDir = ifNoneReturnDefault(opts.repoDir, repoDir)
    productDir = ifNoneReturnDefault(opts.productDir, 'C:\git\cs-studio\products\NSLS2')
    buildDir = ifNoneReturnDefault(opts.buildDir, 'C:\git\cs-studio\build')
    confBuildDir = ifNoneReturnDefault(opts.confBuildDir, productDir)

    '''
    find the build.properties
    '''
    cf = SafeConfigParser()
    cf.readfp(FakeSecHead(open(os.path.join(confBuildDir, 'build.properties'), 'r')))

    topLevelElementType =  ''
    topLevelElementId =  ''
    if check(confBuildDir + '/build.properties', 'feature.xml') :
        topLevelElementType = 'feature'
        topLevelElementId = 'feature.xml'

    line = check(confBuildDir + '/build.properties', '.product')
    productFileName = ''
    if line :
        line = line.strip()
        if ',' in line :
            line = line[:line.index(',')]
        productFileName = line

    if not str(productFileName).endswith('.product'):
        print 'Invalid product specified in build.properties'
        sys.exit(-1)
    '''
    Find the product file for the product specified in the build.properties
    '''
    for dirpath, dirnames, filenames in os.walk(os.path.normpath(productDir)):
        l = [ os.path.join(dirpath, f) for f in filenames if f == productFileName]
        if len(l) == 1:
           productFilePath = l[0]
        elif len(l) > 1:
            print 'Found more than one copy of ', productFileName, 'at:'
            for f in l:
               print f
    print 'Building:', productFilePath

    '''
    TODO: need to include the plugin on the product and explicitly include the dependencies defined there.
    TODO: handle optional dependencies
    '''
    try:
        dependencyMap = createDependencyMap(repoDir=repoDir)
        featureMap = readFeatures(repoDir=repoDir)

        '''
        Get started from a project, get the direct dependencies from the manifest
        META-INF/MANIFEST.MF

        Get the product file and get the features needs from
        *.product
        '''
        reqFeatures = []
        reqPlugins = []
        if productFilePath:
            xmldoc = minidom.parse(productFilePath)
            for features in xmldoc.getElementsByTagName('features'):
                for feature in features.getElementsByTagName('feature'):
                    reqFeatures.append(feature._attrs[u'id'].value)
                    '''Add the top level element if it is a feature'''
            for plugins in xmldoc.getElementsByTagName('plugins'):
                for plugin in plugins.getElementsByTagName('plugin'):
                    reqPlugins.append(plugin._attrs[u'id'].value)

        if topLevelElementType == 'feature':
            reqFeatures.append(topLevelElementId)

        '''
        Since our products are feature based products we will use the features to make the two lists
        1. feature.list : all the features to be packaged with this product
        2. plugin.list : all the plugins to be packaged with this product
        '''
        featureList = []
        for featureId in reqFeatures:
            featureList = featureList + getFeatureList(featureId=featureId, featureMap=featureMap)
        featureList = list(set(featureList))
        featureList.sort()
        '''this is feature.list'''
        print featureList
        featureListFile = open(os.path.join(confBuildDir, 'features.list'), 'w')
        for feature in featureList:
            '''
            This additional step is to check id feature folder name is different from feature id
            '''
            featureDir = os.path.split(os.path.normpath(featureMap.get(feature)['file']))[0]
            featureDirName = os.path.split(featureDir)[1]
            if(feature != featureDirName):
                print 'Consider revising feature folder name', featureDirName, 'to match featureId:', feature
            featureListFile.write("%s\n" % featureDirName)
        featureListFile.close()
        '''
        The plugin list represents the plugins that are part of/included in the features
        TODO [optional] We do need to add the additional plugin defined in the product.xml
        '''
        pluginList = []
        for featureId in reqFeatures:
            pluginList = pluginList + getPluginList(featureId=featureId, featureMap=featureMap)
        pluginList = list(set(pluginList))
        pluginList.sort()
        '''
        This is the exhaustive plugin.list
        It includes all the plugins included in any of the features(and included features) used by the product'''
#        print pluginList
        '''
        sourcePluginList:
        This is the plugin.list consisting of source plugins from the cs-studio repo
        '''
        sourcePluginList = [ plugin for plugin in pluginList if plugin in dependencyMap.keys() ]
        pluginListFile = open(os.path.join(confBuildDir, 'plugins.list'), 'w')
        for plugin in sourcePluginList:
            '''
            This additional step is to check id plugin folder name is different from plugin id
            '''
            pluginDirName = (os.path.normpath(dependencyMap.get(plugin)['file'])).split(os.sep)[-3]
            if(plugin != pluginDirName):
                print 'Consider revising plugin folder name', pluginDirName, 'to match pluginId:', plugin
            pluginListFile.write("%s\n" % pluginDirName)
        pluginListFile.close()
        print len(sourcePluginList), sourcePluginList

        '''
        Now we try to resolve the plugins required by those listed in the pluginList
        using the dependencies defined in the META-INF/MANIFEST.MF

        We shall output the missing dependencies and also a list of plugins that need them.
        '''
        manifestDependencyList = []
        for plugin in pluginList:
            if plugin in  dependencyMap.keys():
                manifestDependencyList.append(plugin)
                manifestDependencyList = manifestDependencyList + dependencyMap[plugin]['dependencies']
        manifestDependencyList = list(set(manifestDependencyList))
        manifestDependencyList.sort()
        '''
        Now we remove the plugins that from eclipse/deltapack
        '''
        reducedManifestDependencyList = [ plugin for plugin in manifestDependencyList if plugin in dependencyMap.keys() ]
        print len(reducedManifestDependencyList), reducedManifestDependencyList

        dependentPlugins = createDependantPluginMap(dependencyMap=dependencyMap)
        for missingDependency in [ plugin for plugin in reducedManifestDependencyList if plugin not in sourcePluginList ]:
            print 'Missing dependency: ' + missingDependency
            if missingDependency in dependentPlugins.keys():
                print 'Required by: '
                print str(dependentPlugins.get(missingDependency)).strip('[]')
    except:
        print 'Exception'
        traceback.print_exc()
        sys.exit(-1)
    pass
<|endoftext|>


<commit_before>"""
Date of Creation: 18th June 2015.

Description:   main dispatch for pyokit scripts

Copyright (C) 2010-2015
Philip J. Uren,

Authors: Philip J. Uren

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

# standard python imports
import sys

# check for rpy2 -- we need to disable some scripts if it is missing
try:
  from rpy2.robjects import r
except ImportError, e:
  have_functioning_rpy2 = False

# pyokit imports -- exceptions
from pyokit.io.ioError import PyokitIOError
from pyokit.common.pyokitError import PyokitError

# pyokit imports -- scripts
if have_functioning_rpy2:
  from pyokit.scripts import fdr
from pyokit.scripts import index
from pyokit.scripts import conservationProfile
from pyokit.scripts import join
from pyokit.scripts import regionCollapse

def main():
  """
  Parse the command line and dispatch the appropriate script.

  This function just performs dispatch on the command line that a user
  provided. Basically, look at the first argument, which specifies the
  function the user wants Ribocop to perform, then dispatch to the appropriate
  module.
  """
  try:
    if sys.argv[1] == "fdr":
      if have_functioning_rpy2:
        fdr.main(sys.argv[2:])
      else:
        sys.stderr.write("The fdr scripts is disabled; it needs rpy2 to " +
                         "run and that package either isn't installed or is " +
                         "not working properly.\n")
    elif sys.argv[1] == "index":
      index.main(sys.argv[2:], "pyokit index")
    elif sys.argv[1] == "consprofile":
      conservationProfile.main(sys.argv[2:], "pyokit consprofile")
    elif sys.argv[1] == "join":
      join._main(sys.argv[2:], "join")
    elif sys.argv[1] == "regionCollapse":
      regionCollapse.main(sys.argv[2:], "pyokit regionCollapse")
    else:
      sys.stderr.write("Pyokit: I don't recognise the option '" + sys.argv[1] +
                       "'.\n")
  except (IOError, PyokitIOError) as e:
    sys.stderr.write("Pyokit - Fatal IOError: " + str(e) + "\n")
    exit(1)
  except PyokitError as e:
    sys.stderr.write("Pyokit - Fatal Error: " + str(e) + "\n")
    exit(1)
<commit_msg>fixed minor bug where rpy2 was not recognised when it was present<commit_after>"""
Date of Creation: 18th June 2015.

Description:   main dispatch for pyokit scripts

Copyright (C) 2010-2015
Philip J. Uren,

Authors: Philip J. Uren

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

# standard python imports
import sys

# check for rpy2 -- we need to disable some scripts if it is missing
try:
  from rpy2.robjects import r
  have_functioning_rpy2 = True
except ImportError, e:
  have_functioning_rpy2 = False

# pyokit imports -- exceptions
from pyokit.io.ioError import PyokitIOError
from pyokit.common.pyokitError import PyokitError

# pyokit imports -- scripts
if have_functioning_rpy2:
  from pyokit.scripts import fdr
from pyokit.scripts import index
from pyokit.scripts import conservationProfile
from pyokit.scripts import join
from pyokit.scripts import regionCollapse

def main():
  """
  Parse the command line and dispatch the appropriate script.

  This function just performs dispatch on the command line that a user
  provided. Basically, look at the first argument, which specifies the
  function the user wants Ribocop to perform, then dispatch to the appropriate
  module.
  """
  try:
    if sys.argv[1] == "fdr":
      if have_functioning_rpy2:
        fdr.main(sys.argv[2:])
      else:
        sys.stderr.write("The fdr scripts is disabled; it needs rpy2 to " +
                         "run and that package either isn't installed or is " +
                         "not working properly.\n")
    elif sys.argv[1] == "index":
      index.main(sys.argv[2:], "pyokit index")
    elif sys.argv[1] == "consprofile":
      conservationProfile.main(sys.argv[2:], "pyokit consprofile")
    elif sys.argv[1] == "join":
      join._main(sys.argv[2:], "join")
    elif sys.argv[1] == "regionCollapse":
      regionCollapse.main(sys.argv[2:], "pyokit regionCollapse")
    else:
      sys.stderr.write("Pyokit: I don't recognise the option '" + sys.argv[1] +
                       "'.\n")
  except (IOError, PyokitIOError) as e:
    sys.stderr.write("Pyokit - Fatal IOError: " + str(e) + "\n")
    exit(1)
  except PyokitError as e:
    sys.stderr.write("Pyokit - Fatal Error: " + str(e) + "\n")
    exit(1)
<|endoftext|>


<commit_before># libavg - Media Playback Engine.
# Copyright (C) 2003-2008 Ulrich von Zadow
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# Current versions can be found at www.libavg.de
#
# Original author of this file is Martin Heistermann <mh at sponc dot de>
#

from libavg import Point2D
from eventList import EventList, Cursor
from mathutil import getDistSquared

MAX_ITERATIONS=50

class Centroid (Cursor):
    def __init__(self):
        super(Centroid, self).__init__()
        self.__members = []

    def isBroken(self):
        return len(self.__members)==0

    def reposition (self):
        pointSum = Point2D(0,0)
        for point in self.__members:
            pointSum += point.getPos()
        center = pointSum / len(self.__members)
        oldPos = self._pos
        self.setPos(center)
        return oldPos != self._pos

    def removeMember (self, member):
        self.__members.remove(member)
        self.reposition()
        self.resetMotion()

    def addMember (self, member):
        self.__members.append(member)
        self.reposition()
        self.resetMotion()

    def hasMember (self, member):
        return member in self.__members

    def __len__(self):
        return len(self.__members)

    def __repr__(self):
        return "centroid at %s, startpos %s,  members %s" % (self._pos, self._startPos, self.__members)

class ClusteredEventList:
    """ implements a variant of k-means.
    same API as EventList, with the difference that ClusteredEventList
    will simulate a maximum of 2 cursors (if more actual cursors are
    on the node, they are clustered.
    In contrast to EventList, the callbacks provide no EventCursors.
    """
    def __init__(self,
            node,
            source,
            onDown = lambda x: None,
            onUp = lambda x: None,
            onMotion = lambda x: None,
            resetMotion = lambda: None,
            captureEvents = True):
        self.__centroids = []
        self.__centroidByEvent = {}
        self.__doNewMotion = False

        self.__callback = {
                'onDown': onDown,
                'onUp': onUp,
                'onMotion': onMotion,
                'resetMotion': resetMotion,
        }
        self.__eventList = EventList(
                node = node,
                source = source,
                onDown = self.__onDown,
                onUp = self.__onUp,
                onMotion = self.__onMotion,
                resetMotion = self.__resetMotion,
                captureEvents = captureEvents)

    def handleInitialDown(self,event):
        self.__eventList.handleInitialDown(event)

    def __onDown(self, eventCursor):
        #self.__callback['onDown'](eventCursor)
        centroids = list(self.__centroids) # copy
        self.calcClusters()
        self.__resetMotion()
        if len(centroids) != len(self.__centroids):
            if len(centroids) and self.__centroids[0] == centroids[0]:
                newCentroid = self.__centroids[1]
            else:
                newCentroid = self.__centroids[0]
            self.__callback['onDown']()

    def __onUp(self, eventCursor):
        if eventCursor in self.__centroidByEvent:
            centroid = self.__centroidByEvent[eventCursor]
            centroid.removeMember(eventCursor)
            if len(centroid) == 0:
                self.__callback['onUp']()
            del self.__centroidByEvent[eventCursor]
        else:
            print "eventCursor %s not found" % eventCursor

        self.calcClusters()
        self.__resetMotion()

    def __onMotion(self, eventCursor):
        oldPositions = {}
        for centroid in self.__centroids:
            oldPositions[centroid] = centroid.getPos()
        self.calcClusters()
        self.__callback['onMotion']()

    def __resetMotion(self):
        for centroid in self.__centroids:
            centroid.resetMotion()
        self.__callback['resetMotion']()

    def delete(self):
        self.__centroids = []
        self.__centroidByEvent = {}
        self.__eventList.delete()

    def __calcMemberships (self):
        """ returns True if a membership changed, else False."""
        changed = False
        if not len(self.__centroids):
            return changed
        for point in self.__eventList.getCursors():
            closestCentroid = self.__centroids[0]
            minDist = getDistSquared(point.getPos(), closestCentroid.getPos())
            for centroid in self.__centroids:
                distance = getDistSquared (point.getPos(), centroid.getPos())
                if distance < minDist:
                    minDist = distance
                    closestCentroid = centroid
            if not closestCentroid.hasMember(point):
                self.__doNewMotion = True
                if point in self.__centroidByEvent:
                    self.__centroidByEvent[point].removeMember(point)
                self.__centroidByEvent[point] = closestCentroid
                closestCentroid.addMember(point)
                changed = True
        return changed

    def __tryCalcClusters (self):
        def __calcInitialCentroids():
            self.__centroidByEvent = {}
            self.__centroids = []
            def createCentroid(point):
                centroid = Centroid()
                centroid.addMember(point)
                self.__centroidByEvent[point] = centroid
                self.__centroids.append(centroid)

            maxDist = 0
            points = None
            if len(self.__eventList)>1:
                cursors = self.__eventList.getCursors()
                for p in cursors:
                    for q in cursors:
                        dist = getDistSquared(p.getPos(),q.getPos())
                        if dist >= maxDist:
                            points = p,q
                            maxDist = dist

                assert(points)
                assert(points[0] != points[1])
                for point in points:
                    createCentroid(point)
            elif len(self.__eventList) == 1:
                createCentroid(self.__eventList.getCursors()[0])

        def __setCentroids():
            changed = False
            for centroid in self.__centroids:
                if centroid.reposition():
                    changed = True
            return changed

        if not len(self.__centroids):
            __calcInitialCentroids()
        self.__calcMemberships()

        changed = True
        iterations = 0
        while changed:
            changed = False
            if __setCentroids():
                changed = True
            if self.__calcMemberships():
                changed = True
            iterations+=1
            if iterations>MAX_ITERATIONS:
                #print "too many iterations(%u), aborting" % iterations
                __setCentroids()
                break

    def calcClusters (self):
        def __hasBrokenCentroids():
            if len(self.__eventList)>1 and len(self.__centroids)!=2:
                return True
            for centroid in self.__centroids:
                if centroid.isBroken():
                    return True
            if len(self.__centroids)==2:
                if self.__centroids[0].getPos() == self.__centroids[1].getPos():
                    return True
            return False

        self.__tryCalcClusters()

        if __hasBrokenCentroids():
            #print "i have broken centroids"
            self.__centroids=[]
            self.__tryCalcClusters()
            if __hasBrokenCentroids():
                print "ignoring fatal error: cannot fix broken centroids: %s" % self.__centroids
                print "self.__eventList =%s " % self.__eventList
        if self.__doNewMotion:
            self.__doNewMotion = False
            self.__resetMotion()

    def getCursors (self):
        return self.__centroids

    def __len__(self):
        return min(len(self.__eventList), len(self.__centroids))

<commit_msg>clusteredEventList: avoid bug when 2 cursors have the same position<commit_after># libavg - Media Playback Engine.
# Copyright (C) 2003-2008 Ulrich von Zadow
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# Current versions can be found at www.libavg.de
#
# Original author of this file is Martin Heistermann <mh at sponc dot de>
#

from libavg import Point2D
from eventList import EventList, Cursor
from mathutil import getDistSquared

MAX_ITERATIONS=50

class Centroid (Cursor):
    def __init__(self):
        super(Centroid, self).__init__()
        self.__members = []

    def isBroken(self):
        return len(self.__members)==0

    def reposition (self):
        pointSum = Point2D(0,0)
        for point in self.__members:
            pointSum += point.getPos()
        center = pointSum / len(self.__members)
        oldPos = self._pos
        self.setPos(center)
        return oldPos != self._pos

    def removeMember (self, member):
        self.__members.remove(member)
        self.reposition()
        self.resetMotion()

    def addMember (self, member):
        self.__members.append(member)
        self.reposition()
        self.resetMotion()

    def hasMember (self, member):
        return member in self.__members

    def __len__(self):
        return len(self.__members)

    def __repr__(self):
        return "centroid at %s, startpos %s,  members %s" % (self._pos, self._startPos, self.__members)

class ClusteredEventList:
    """ implements a variant of k-means.
    same API as EventList, with the difference that ClusteredEventList
    will simulate a maximum of 2 cursors (if more actual cursors are
    on the node, they are clustered.
    In contrast to EventList, the callbacks provide no EventCursors.
    """
    def __init__(self,
            node,
            source,
            onDown = lambda x: None,
            onUp = lambda x: None,
            onMotion = lambda x: None,
            resetMotion = lambda: None,
            captureEvents = True):
        self.__centroids = []
        self.__centroidByEvent = {}
        self.__doNewMotion = False

        self.__callback = {
                'onDown': onDown,
                'onUp': onUp,
                'onMotion': onMotion,
                'resetMotion': resetMotion,
        }
        self.__eventList = EventList(
                node = node,
                source = source,
                onDown = self.__onDown,
                onUp = self.__onUp,
                onMotion = self.__onMotion,
                resetMotion = self.__resetMotion,
                captureEvents = captureEvents)

    def handleInitialDown(self,event):
        self.__eventList.handleInitialDown(event)

    def __onDown(self, eventCursor):
        #self.__callback['onDown'](eventCursor)
        centroids = list(self.__centroids) # copy
        self.calcClusters()
        self.__resetMotion()
        if len(centroids) != len(self.__centroids):
            if len(centroids) and self.__centroids[0] == centroids[0]:
                newCentroid = self.__centroids[1]
            else:
                newCentroid = self.__centroids[0]
            self.__callback['onDown']()

    def __onUp(self, eventCursor):
        if eventCursor in self.__centroidByEvent:
            centroid = self.__centroidByEvent[eventCursor]
            centroid.removeMember(eventCursor)
            if len(centroid) == 0:
                self.__callback['onUp']()
            del self.__centroidByEvent[eventCursor]
        else:
            print "eventCursor %s not found" % eventCursor

        self.calcClusters()
        self.__resetMotion()

    def __onMotion(self, eventCursor):
        oldPositions = {}
        for centroid in self.__centroids:
            oldPositions[centroid] = centroid.getPos()
        self.calcClusters()
        self.__callback['onMotion']()

    def __resetMotion(self):
        for centroid in self.__centroids:
            centroid.resetMotion()
        self.__callback['resetMotion']()

    def delete(self):
        self.__centroids = []
        self.__centroidByEvent = {}
        self.__eventList.delete()

    def __calcMemberships (self):
        """ returns True if a membership changed, else False."""
        changed = False
        if not len(self.__centroids):
            return changed
        for point in self.__eventList.getCursors():
            closestCentroid = self.__centroids[0]
            minDist = getDistSquared(point.getPos(), closestCentroid.getPos())
            for centroid in self.__centroids:
                distance = getDistSquared (point.getPos(), centroid.getPos())
                if distance < minDist:
                    minDist = distance
                    closestCentroid = centroid
            if not closestCentroid.hasMember(point):
                self.__doNewMotion = True
                if point in self.__centroidByEvent:
                    self.__centroidByEvent[point].removeMember(point)
                self.__centroidByEvent[point] = closestCentroid
                closestCentroid.addMember(point)
                changed = True
        return changed

    def __tryCalcClusters (self):
        def __calcInitialCentroids():
            self.__centroidByEvent = {}
            self.__centroids = []
            def createCentroid(point):
                centroid = Centroid()
                centroid.addMember(point)
                self.__centroidByEvent[point] = centroid
                self.__centroids.append(centroid)

            maxDist = 0
            points = None
            if len(self.__eventList)>1:
                cursors = self.__eventList.getCursors()
                for p in cursors:
                    for q in cursors:
                        dist = getDistSquared(p.getPos(),q.getPos())
                        if dist >= maxDist and p != q:
                            points = p,q
                            maxDist = dist

                assert(points)
                for point in points:
                    createCentroid(point)
            elif len(self.__eventList) == 1:
                createCentroid(self.__eventList.getCursors()[0])

        def __setCentroids():
            changed = False
            for centroid in self.__centroids:
                if centroid.reposition():
                    changed = True
            return changed

        if not len(self.__centroids):
            __calcInitialCentroids()
        self.__calcMemberships()

        changed = True
        iterations = 0
        while changed:
            changed = False
            if __setCentroids():
                changed = True
            if self.__calcMemberships():
                changed = True
            iterations+=1
            if iterations>MAX_ITERATIONS:
                #print "too many iterations(%u), aborting" % iterations
                __setCentroids()
                break

    def calcClusters (self):
        def __hasBrokenCentroids():
            if len(self.__eventList)>1 and len(self.__centroids)!=2:
                return True
            for centroid in self.__centroids:
                if centroid.isBroken():
                    return True
            if len(self.__centroids)==2:
                if self.__centroids[0].getPos() == self.__centroids[1].getPos():
                    return True
            return False

        self.__tryCalcClusters()

        if __hasBrokenCentroids():
            #print "i have broken centroids"
            self.__centroids=[]
            self.__tryCalcClusters()
            if __hasBrokenCentroids():
                print "ignoring fatal error: cannot fix broken centroids: %s" % self.__centroids
                print "self.__eventList =%s " % self.__eventList
        if self.__doNewMotion:
            self.__doNewMotion = False
            self.__resetMotion()

    def getCursors (self):
        return self.__centroids

    def __len__(self):
        return min(len(self.__eventList), len(self.__centroids))

<|endoftext|>


<commit_before>import color_sensor_ISL29125
from time import time

cs = color_sensor_ISL29125.color_senser(1)

if cs.valid_init:
	print "Valid color sensor"
else :
	print "Color Sensor invalid"

cs.setUpperThreshold(0x5)

print "UT : " + str( cs.readUpperThreshold())

for x in range(5):
	t0 = time()
	print "\nStatus: " + cs.readStatus()
	print "Red  : " + str( cs.readRed())
	print "Green: " + str( cs.readGreen())
	print "Blue : " + str( cs.readBlue())
	t1 = time()
	print "Sample time: " + str( int((t1-t0)*10000)/10.0) + "ms"

<commit_msg>use color guesser<commit_after>import color_sensor_ISL29125
from time import time

cs = color_sensor_ISL29125.color_senser(1)

if cs.valid_init:
	print "Valid color sensor"
else :
	print "Color Sensor invalid"

cs.setUpperThreshold(0x5)

print "UT : " + str( cs.readUpperThreshold())

for x in range(5):
	t0 = time()
	print "\nStatus: " + cs.readStatus()
	print "Red  : " + str( cs.readRed())
	print "Green: " + str( cs.readGreen())
	print "Blue : " + str( cs.readBlue())
	t1 = time()
	print "Guessed Color is " + cs.roadColor()
	print "Sample time: " + str( int((t1-t0)*10000)/10.0) + "ms"

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2016-2021 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

import os
import re
import argparse
from shutil import copyfile

SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'src'))
DEFAULT_PLATFORM_TOOLSET = R'v142'

libs = [
    'libbitcoin_cli',
    'libbitcoin_common',
    'libbitcoin_crypto',
    'libbitcoin_node',
    'libbitcoin_util',
    'libbitcoin_wallet_tool',
    'libbitcoin_wallet',
    'libbitcoin_zmq',
    'bench_bitcoin',
    'libtest_util',
]

ignore_list = [
]

lib_sources = {}

def parse_makefile(makefile):
    with open(makefile, 'r', encoding='utf-8') as file:
        current_lib = ''
        for line in file.read().splitlines():
            if current_lib:
                source = line.split()[0]
                if source.endswith('.cpp') and not source.startswith('$') and source not in ignore_list:
                    source_filename = source.replace('/', '\\')
                    object_filename = source.replace('/', '_')[:-4] + ".obj"
                    lib_sources[current_lib].append((source_filename, object_filename))
                if not line.endswith('\\'):
                    current_lib = ''
                continue
            for lib in libs:
                _lib = lib.replace('-', '_')
                if re.search(_lib + '.*_SOURCES \\= \\\\', line):
                    current_lib = lib
                    lib_sources[current_lib] = []
                    break

def set_common_properties(toolset):
    with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'r', encoding='utf-8') as rfile:
        s = rfile.read()
        s = re.sub('<PlatformToolset>.*?</PlatformToolset>', '<PlatformToolset>'+toolset+'</PlatformToolset>', s)
    with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'w', encoding='utf-8',newline='\n') as wfile:
        wfile.write(s)

def parse_config_into_btc_config():
    def find_between( s, first, last ):
        try:
            start = s.index( first ) + len( first )
            end = s.index( last, start )
            return s[start:end]
        except ValueError:
            return ""

    config_info = []
    with open(os.path.join(SOURCE_DIR,'../configure.ac'), encoding="utf8") as f:
        for line in f:
            if line.startswith("define"):
                config_info.append(find_between(line, "(_", ")"))

    config_info = [c for c in config_info if not c.startswith("COPYRIGHT_HOLDERS")]

    config_dict = dict(item.split(", ") for item in config_info)
    config_dict["PACKAGE_VERSION"] = f"\"{config_dict['CLIENT_VERSION_MAJOR']}.{config_dict['CLIENT_VERSION_MINOR']}.{config_dict['CLIENT_VERSION_BUILD']}\""
    version = config_dict["PACKAGE_VERSION"].strip('"')
    config_dict["PACKAGE_STRING"] = f"\"Bitcoin Core {version}\""

    with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h.in'), "r", encoding="utf8") as template_file:
        template = template_file.readlines()

    for index, line in enumerate(template):
        header = ""
        if line.startswith("#define"):
            header = line.split(" ")[1]
        if header in config_dict:
            template[index] = line.replace("$", f"{config_dict[header]}")

    with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), "w", encoding="utf8") as btc_config:
        btc_config.writelines(template)

def main():
    parser = argparse.ArgumentParser(description='Bitcoin-core msbuild configuration initialiser.')
    parser.add_argument('-toolset', nargs='?',help='Optionally sets the msbuild platform toolset, e.g. v142 for Visual Studio 2019.'
         ' default is %s.'%DEFAULT_PLATFORM_TOOLSET)
    args = parser.parse_args()
    if args.toolset:
        set_common_properties(args.toolset)

    for makefile_name in os.listdir(SOURCE_DIR):
        if 'Makefile' in makefile_name:
            parse_makefile(os.path.join(SOURCE_DIR, makefile_name))
    for key, value in lib_sources.items():
        vcxproj_filename = os.path.abspath(os.path.join(os.path.dirname(__file__), key, key + '.vcxproj'))
        content = ''
        for source_filename, object_filename in value:
            content += '    <ClCompile Include="..\\..\\src\\' + source_filename + '">\n'
            content += '      <ObjectFileName>$(IntDir)' + object_filename + '</ObjectFileName>\n'
            content += '    </ClCompile>\n'
        with open(vcxproj_filename + '.in', 'r', encoding='utf-8') as vcxproj_in_file:
            with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file:
                vcxproj_file.write(vcxproj_in_file.read().replace(
                    '@SOURCE_FILES@\n', content))
    parse_config_into_btc_config()
    copyfile(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), os.path.join(SOURCE_DIR, 'config/bitcoin-config.h'))
    copyfile(os.path.join(SOURCE_DIR,'../build_msvc/libsecp256k1_config.h'), os.path.join(SOURCE_DIR, 'secp256k1/src/libsecp256k1-config.h'))

if __name__ == '__main__':
    main()
<commit_msg>build, refactor: Add set_properties() to msvc-autogen.py<commit_after>#!/usr/bin/env python3
# Copyright (c) 2016-2021 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

import os
import re
import argparse
from shutil import copyfile

SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'src'))
DEFAULT_PLATFORM_TOOLSET = R'v142'

libs = [
    'libbitcoin_cli',
    'libbitcoin_common',
    'libbitcoin_crypto',
    'libbitcoin_node',
    'libbitcoin_util',
    'libbitcoin_wallet_tool',
    'libbitcoin_wallet',
    'libbitcoin_zmq',
    'bench_bitcoin',
    'libtest_util',
]

ignore_list = [
]

lib_sources = {}

def parse_makefile(makefile):
    with open(makefile, 'r', encoding='utf-8') as file:
        current_lib = ''
        for line in file.read().splitlines():
            if current_lib:
                source = line.split()[0]
                if source.endswith('.cpp') and not source.startswith('$') and source not in ignore_list:
                    source_filename = source.replace('/', '\\')
                    object_filename = source.replace('/', '_')[:-4] + ".obj"
                    lib_sources[current_lib].append((source_filename, object_filename))
                if not line.endswith('\\'):
                    current_lib = ''
                continue
            for lib in libs:
                _lib = lib.replace('-', '_')
                if re.search(_lib + '.*_SOURCES \\= \\\\', line):
                    current_lib = lib
                    lib_sources[current_lib] = []
                    break

def set_common_properties(toolset):
    with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'r', encoding='utf-8') as rfile:
        s = rfile.read()
        s = re.sub('<PlatformToolset>.*?</PlatformToolset>', '<PlatformToolset>'+toolset+'</PlatformToolset>', s)
    with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'w', encoding='utf-8',newline='\n') as wfile:
        wfile.write(s)

def parse_config_into_btc_config():
    def find_between( s, first, last ):
        try:
            start = s.index( first ) + len( first )
            end = s.index( last, start )
            return s[start:end]
        except ValueError:
            return ""

    config_info = []
    with open(os.path.join(SOURCE_DIR,'../configure.ac'), encoding="utf8") as f:
        for line in f:
            if line.startswith("define"):
                config_info.append(find_between(line, "(_", ")"))

    config_info = [c for c in config_info if not c.startswith("COPYRIGHT_HOLDERS")]

    config_dict = dict(item.split(", ") for item in config_info)
    config_dict["PACKAGE_VERSION"] = f"\"{config_dict['CLIENT_VERSION_MAJOR']}.{config_dict['CLIENT_VERSION_MINOR']}.{config_dict['CLIENT_VERSION_BUILD']}\""
    version = config_dict["PACKAGE_VERSION"].strip('"')
    config_dict["PACKAGE_STRING"] = f"\"Bitcoin Core {version}\""

    with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h.in'), "r", encoding="utf8") as template_file:
        template = template_file.readlines()

    for index, line in enumerate(template):
        header = ""
        if line.startswith("#define"):
            header = line.split(" ")[1]
        if header in config_dict:
            template[index] = line.replace("$", f"{config_dict[header]}")

    with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), "w", encoding="utf8") as btc_config:
        btc_config.writelines(template)

def set_properties(vcxproj_filename, placeholder, content):
    with open(vcxproj_filename + '.in', 'r', encoding='utf-8') as vcxproj_in_file:
        with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file:
            vcxproj_file.write(vcxproj_in_file.read().replace(placeholder, content))

def main():
    parser = argparse.ArgumentParser(description='Bitcoin-core msbuild configuration initialiser.')
    parser.add_argument('-toolset', nargs='?',help='Optionally sets the msbuild platform toolset, e.g. v142 for Visual Studio 2019.'
         ' default is %s.'%DEFAULT_PLATFORM_TOOLSET)
    args = parser.parse_args()
    if args.toolset:
        set_common_properties(args.toolset)

    for makefile_name in os.listdir(SOURCE_DIR):
        if 'Makefile' in makefile_name:
            parse_makefile(os.path.join(SOURCE_DIR, makefile_name))
    for key, value in lib_sources.items():
        vcxproj_filename = os.path.abspath(os.path.join(os.path.dirname(__file__), key, key + '.vcxproj'))
        content = ''
        for source_filename, object_filename in value:
            content += '    <ClCompile Include="..\\..\\src\\' + source_filename + '">\n'
            content += '      <ObjectFileName>$(IntDir)' + object_filename + '</ObjectFileName>\n'
            content += '    </ClCompile>\n'
        set_properties(vcxproj_filename, '@SOURCE_FILES@\n', content):
    parse_config_into_btc_config()
    copyfile(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), os.path.join(SOURCE_DIR, 'config/bitcoin-config.h'))
    copyfile(os.path.join(SOURCE_DIR,'../build_msvc/libsecp256k1_config.h'), os.path.join(SOURCE_DIR, 'secp256k1/src/libsecp256k1-config.h'))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""Script to plot a dump of predictions."""

import sys
from typing.io import BinaryIO

import acton.proto.io
from acton.proto.predictors_pb2 import Predictions
import acton.proto.wrappers
import click
import matplotlib.pyplot as plt
import sklearn.metrics

@click.command()
@click.argument('predictions',
                type=click.File('rb'),
                nargs=-1,
                required=True)
def plot(predictions: BinaryIO):
    """Plots predictions from a file.

    Parameters
    ----------
    predictions
        Predictions file.
    """

    if len(predictions) < 1:
        raise ValueError('Must have at least 1 set of predictions.')

    for predictions_ in predictions:
        # Read in the first protobuf to get the database file.
        protobuf = next(acton.proto.io.read_protos(predictions_, Predictions))
        protobuf = acton.proto.wrappers.PredictorOutput(protobuf)
        with protobuf.DB() as db:
            accuracies = []
            for protobuf in acton.proto.io.read_protos(
                    predictions_, Predictions):
                protobuf = acton.proto.wrappers.PredictorOutput(protobuf)
                ids = protobuf.ids
                predictions_ = protobuf.predictions
                assert predictions_.shape[0] == 1
                predictions_ = predictions_[0]
                labels = db.read_labels([b'0'], ids).ravel()
                predicted_labels = predictions_.round().ravel()
                accuracies.append(sklearn.metrics.accuracy_score(
                    labels, predicted_labels))

            plt.plot(accuracies, label=protobuf.proto.predictor)

    plt.xlabel('Number of additional labels')
    plt.ylabel('Accuracy score')
    plt.legend()
    plt.show()

if __name__ == '__main__':
    sys.exit(plot())
<commit_msg>Made plot function separate from Click<commit_after>"""Script to plot a dump of predictions."""

import sys
from typing import Iterable
from typing.io import BinaryIO

import acton.proto.io
from acton.proto.predictors_pb2 import Predictions
import acton.proto.wrappers
import click
import matplotlib.pyplot as plt
import sklearn.metrics

def plot(predictions: Iterable[BinaryIO]):
    """Plots predictions from a file.

    Parameters
    ----------
    predictions
        Files containing predictions.
    """
    if len(predictions) < 1:
        raise ValueError('Must have at least 1 set of predictions.')

    for predictions_ in predictions:
        # Read in the first protobuf to get the database file.
        protobuf = next(acton.proto.io.read_protos(predictions_, Predictions))
        protobuf = acton.proto.wrappers.PredictorOutput(protobuf)
        with protobuf.DB() as db:
            accuracies = []
            for protobuf in acton.proto.io.read_protos(
                    predictions_, Predictions):
                protobuf = acton.proto.wrappers.PredictorOutput(protobuf)
                ids = [id_.encode('ascii') for id_ in protobuf.ids]
                predictions_ = protobuf.predictions
                assert predictions_.shape[0] == 1
                predictions_ = predictions_[0]
                labels = db.read_labels([b'0'], ids).ravel()
                predicted_labels = predictions_.round().ravel()
                accuracies.append(sklearn.metrics.accuracy_score(
                    labels, predicted_labels))

            plt.plot(accuracies, label=protobuf.proto.predictor)

    plt.xlabel('Number of additional labels')
    plt.ylabel('Accuracy score')
    plt.legend()
    plt.show()

@click.command()
@click.argument('predictions',
                type=click.File('rb'),
                nargs=-1,
                required=True)
def _plot(predictions: Iterable[BinaryIO]):
    """Plots predictions from a file.

    Parameters
    ----------
    predictions
        Files containing predictions.
    """
    return plot(predictions)

if __name__ == '__main__':
    sys.exit(_plot())
<|endoftext|>


<commit_before>from adam.core import Asset, supports_mime_types
import PIL.Image, PIL.ExifTags

@supports_mime_types('image/jpeg')
def read_jpeg(jpeg_file):
    asset = Asset()
    asset.mime_type = 'image/jpeg'
    image = PIL.Image.open(jpeg_file)
    asset.width,asset.height = image.info['jfif_density']
    asset.essence = jpeg_file
    return asset
<commit_msg>image.read_jpeg(jpeg_file) reads the image dimension from the image objects instead of the metadata.<commit_after>from adam.core import Asset, supports_mime_types
import PIL.Image, PIL.ExifTags

@supports_mime_types('image/jpeg')
def read_jpeg(jpeg_file):
    asset = Asset()
    asset.mime_type = 'image/jpeg'
    image = PIL.Image.open(jpeg_file)
    asset.width = image.width
    asset.height = image.height
    asset.essence = jpeg_file
    return asset
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Slack integration."""

import requests

class SlackMessenger(object):
    """A Slack messenger."""

    def __init__(self, app=None):
        """Initialize messenger."""
        self.enabled = False
        if app:
            self.app = app
            self.init_app(app)

    def init_app(self, app):
        """Initialize Slack Messenger."""
        self.app = app
        self.enabled = app.config.get('SLACK_NOTIFICATIONS_ENABLED')
        if self.enabled:
            self.url = app.config.get('SLACK_WEBHOOK')
            self.channel = app.config.get('SLACK_CHANNEL')
            self.username = app.config.get('SLACK_USERNAME')
            self.icon_emoji = app.config.get('SLACK_ICON_EMOJI')
        app.extensions['messenger'] = self

    def notify_slack(self, msg):
        """Send a simple notification to Slack."""
        if self.enabled:
            payload = {'username': self.username,
                       'icon_emoji': self.icon_emoji,
                       'channel': self.channel,
                       'text': msg}
            requests.post(self.url, json=payload)
<commit_msg>Add debug logging<commit_after># -*- coding: utf-8 -*-
"""Slack integration."""

import requests

class SlackMessenger(object):
    """A Slack messenger."""

    def __init__(self, app=None):
        """Initialize messenger."""
        self.enabled = False
        if app:
            self.app = app
            self.init_app(app)

    def init_app(self, app):
        """Initialize Slack Messenger."""
        self.app = app
        self.enabled = app.config.get('SLACK_NOTIFICATIONS_ENABLED')
        if self.enabled:
            self.url = app.config.get('SLACK_WEBHOOK')
            self.channel = app.config.get('SLACK_CHANNEL')
            self.username = app.config.get('SLACK_USERNAME')
            self.icon_emoji = app.config.get('SLACK_ICON_EMOJI')
        app.extensions['messenger'] = self

    def notify_slack(self, msg):
        """Send a simple notification to Slack."""
        if self.enabled:
            payload = {'username': self.username,
                       'icon_emoji': self.icon_emoji,
                       'channel': self.channel,
                       'text': msg}
            requests.post(self.url, json=payload)
            self.app.logger.debug('Sent {} to {}'.format(msg, self.channel))
        else:
            self.app.logger.debug('Ignoring message request')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('cms', '0003_auto_20140926_2347'),
    ]

    operations = [
        migrations.AddField(
            model_name='cmsplugin',
            name='depth',
            field=models.PositiveIntegerField(default=0),
            preserve_default=False,
        ),
        migrations.AddField(
            model_name='cmsplugin',
            name='numchild',
            field=models.PositiveIntegerField(default=0),
            preserve_default=True,
        ),
        migrations.AddField(
            model_name='cmsplugin',
            name='path',
            field=models.CharField(default='', unique=True, max_length=255),
            preserve_default=False,
        ),
        migrations.AddField(
            model_name='page',
            name='depth',
            field=models.PositiveIntegerField(default=0),
            preserve_default=False,
        ),
        migrations.AddField(
            model_name='page',
            name='numchild',
            field=models.PositiveIntegerField(default=0),
            preserve_default=True,
        ),
        migrations.AddField(
            model_name='page',
            name='path',
            field=models.CharField(default='', unique=True, max_length=255),
            preserve_default=False,
        ),
    ]
<commit_msg>Fix migrations for MP_Tree<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('cms', '0003_auto_20140926_2347'),
    ]

    operations = [
        migrations.AddField(
            model_name='cmsplugin',
            name='depth',
            field=models.PositiveIntegerField(default=0),
            preserve_default=False,
        ),
        migrations.AddField(
            model_name='cmsplugin',
            name='numchild',
            field=models.PositiveIntegerField(default=0),
            preserve_default=True,
        ),
        migrations.AddField(
            model_name='cmsplugin',
            name='path',
            field=models.CharField(default='', max_length=255),
            preserve_default=False,
        ),
        migrations.AddField(
            model_name='page',
            name='depth',
            field=models.PositiveIntegerField(default=0),
            preserve_default=False,
        ),
        migrations.AddField(
            model_name='page',
            name='numchild',
            field=models.PositiveIntegerField(default=0),
            preserve_default=True,
        ),
        migrations.AddField(
            model_name='page',
            name='path',
            field=models.CharField(default='', max_length=255),
            preserve_default=False,
        ),
    ]
<|endoftext|>


<commit_before>from tsundiary import User, Post, db
from datetime import date, datetime, timedelta

def delete_user(sid):
    Post.query.filter_by(user_sid=sid).delete()
    User.query.filter_by(sid=sid).delete()
    db.session.commit()

def user_from_sid(sid):
    return User.query.filter_by(sid=sid).first()

def graph_all_posts():
    posts = Post.query.all()
    d = min(p.posted_date for p in posts)
    while d < date.today():
        print("%s, %d" % (d, len([p for p in posts if p.posted_date == d])))
        d += timedelta(days=1)

def stalk(sid, depth=1):
    entries = user_from_sid(sid).posts.order_by(Post.posted_date.desc()).limit(depth)
    for e in entries:
        print(e.posted_date)
        print(e.content)
        print "---"
<commit_msg>add more admin stuff<commit_after>from tsundiary import User, Post, db
from datetime import date, datetime, timedelta

def delete_user(sid):
    Post.query.filter_by(user_sid=sid).delete()
    User.query.filter_by(sid=sid).delete()
    db.session.commit()

def user_from_sid(sid):
    return User.query.filter_by(sid=sid).first()

def graph_all_posts():
    posts = Post.query.all()
    d = min(p.posted_date for p in posts)
    while d < date.today():
        print("%s, %d" % (d, len([p for p in posts if p.posted_date == d])))
        d += timedelta(days=1)

def get_active_users():
    users = {}
    for p in Post.query.filter(Post.posted_date >= date.today() - timedelta(days=6)).all():
        if p.user_sid not in users:
            users[p.user_sid] = 0
        users[p.user_sid] += 1

    upairs = users.items()
    upairs.sort()
    upairs.sort(key=lambda x:x[1], reverse=True)
    print("users by # posts in last 7 days:")
    for sid,count in upairs:
        print("%3d: %s" % (count, sid))

def stalk(sid, depth=1):
    entries = user_from_sid(sid).posts.order_by(Post.posted_date.desc()).limit(depth)
    for e in entries:
        print(e.posted_date)
        print(e.content)
        print "---"
<|endoftext|>


<commit_before># pylint: disable=C0111,R0903

"""Displays information about the current song in Google Play music player.

Requires the following executable:
    * gpmdp-remote
"""

import string

import bumblebee.util
import bumblebee.input
import bumblebee.output
import bumblebee.engine

class Module(bumblebee.engine.Module):
    def __init__(self, engine, config):
        widgets = [
            bumblebee.output.Widget(name="gpmdp.prev"),
            bumblebee.output.Widget(name="gpmdp.main", full_text=self.description),
            bumblebee.output.Widget(name="gpmdp.next"),
        ]
        super(Module, self).__init__(engine, config, widgets)

        engine.input.register_callback(widgets[0], button=bumblebee.input.LEFT_MOUSE,
            cmd="playerctl previous")
        engine.input.register_callback(widgets[1], button=bumblebee.input.LEFT_MOUSE,
             cmd="playerctl play-pause")
        engine.input.register_callback(widgets[2], button=bumblebee.input.LEFT_MOUSE,
             cmd="playerctl next")

        self._status = None
        self._tags = None

    def description(self, widget):
        return self._tags if self._tags else "n/a"

    def update(self, widgets):
        self._load_song()

    def state(self, widget):
        if widget.name == "gpmdp.prev":
            return "prev"
        if widget.name == "gpmdp.next":
            return "next"
        return self._status

    def _load_song(self):
        info = ""
        try:
            info = bumblebee.util.execute("gpmdp-remote current")
            status = bumblebee.util.execute("gpmdp-remote status")
        except RuntimeError:
            pass
        self._status = status.decode('utf-8').split("\n")[0].lower()
        self._tags = info.decode('utf-8').split("\n")[0]

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<commit_msg>fix should work for Python 2 and 3<commit_after># pylint: disable=C0111,R0903

"""Displays information about the current song in Google Play music player.

Requires the following executable:
    * gpmdp-remote
"""

import string

import bumblebee.util
import bumblebee.input
import bumblebee.output
import bumblebee.engine

class Module(bumblebee.engine.Module):
    def __init__(self, engine, config):
        widgets = [
            bumblebee.output.Widget(name="gpmdp.prev"),
            bumblebee.output.Widget(name="gpmdp.main", full_text=self.description),
            bumblebee.output.Widget(name="gpmdp.next"),
        ]
        super(Module, self).__init__(engine, config, widgets)

        engine.input.register_callback(widgets[0], button=bumblebee.input.LEFT_MOUSE,
            cmd="playerctl previous")
        engine.input.register_callback(widgets[1], button=bumblebee.input.LEFT_MOUSE,
             cmd="playerctl play-pause")
        engine.input.register_callback(widgets[2], button=bumblebee.input.LEFT_MOUSE,
             cmd="playerctl next")

        self._status = None
        self._tags = None

    def description(self, widget):
        return self._tags if self._tags else "n/a"

    def update(self, widgets):
        self._load_song()

    def state(self, widget):
        if widget.name == "gpmdp.prev":
            return "prev"
        if widget.name == "gpmdp.next":
            return "next"
        return self._status

    def _load_song(self):
        info = ""
        try:
            info = bumblebee.util.execute("gpmdp-remote current")
            status = bumblebee.util.execute("gpmdp-remote status")
        except RuntimeError:
            pass
        try:
            unicode_status = status.decode('utf-8')
            unicode_info = info.decode('utf-8')
        except AttributeError:
            unicode_status = status
            unicode_info = info
        self._status = unicode_status.split("\n")[0].lower()
        self._tags = unicode_info.split("\n")[0]

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before><commit_msg>Fix course export issue<commit_after><|endoftext|>


<commit_before>from base64 import b64encode
import json

from pyramid.httpexceptions import HTTPBadRequest
from pyramid.response import Response
from pyramid.view import view_config

def json_base64_encode_chunks(filename, file, chunk_size=65536):
    """
    Generate a JSON-wrapped base64-encoded string.
    See http://en.wikipedia.org/wiki/Base64
    """
    yield '{"filename":%s,"data":"' % (json.dumps(filename),)
    while True:
        line = file.read(chunk_size)
        if not line:
            break
        yield b64encode(line)
    yield '","success":true}'

@view_config(route_name='echo')
def echo(request):
    """
    Echo an uploaded file back to the client as an text/html document so it can
    be handled by Ext.

    The response is JSON-wrapped and base64-encoded to ensure that there are no
    special HTML characters or charset problems and so that braindead ext
    doesn't barf on it.

    We use an iterator to avoid loading the whole file into memory.  See
    http://docs.sencha.com/ext-js/3-4/#!/api/Ext.form.BasicForm-cfg-fileUpload
    """
    if request.method != 'POST':
        return HTTPBadRequest()
    try:
        file = request.POST['file']
    except KeyError:
        return HTTPBadRequest()
    response = Response()
    response.app_iter = json_base64_encode_chunks(file.filename, file.file)
    response.content_type = 'text/html'
    return response
<commit_msg>Don't attempt to chunk echoed files (it breaks the base64 encoding)<commit_after>from base64 import b64encode
import json

from pyramid.httpexceptions import HTTPBadRequest
from pyramid.response import Response
from pyramid.view import view_config

def json_base64_encode(filename, file):
    """
    Generate a JSON-wrapped base64-encoded string.
    See http://en.wikipedia.org/wiki/Base64
    """
    yield '{"filename":%s,"data":"' % (json.dumps(filename),)
    yield b64encode(file.read())
    yield '","success":true}'

@view_config(route_name='echo')
def echo(request):
    """
    Echo an uploaded file back to the client as an text/html document so it can
    be handled by Ext.

    The response is JSON-wrapped and base64-encoded to ensure that there are no
    special HTML characters or charset problems and so that braindead ext
    doesn't barf on it.

    See:
    http://docs.sencha.com/ext-js/3-4/#!/api/Ext.form.BasicForm-cfg-fileUpload
    """
    if request.method != 'POST':
        return HTTPBadRequest()
    try:
        file = request.POST['file']
    except KeyError:
        return HTTPBadRequest()
    response = Response()
    response.app_iter = json_base64_encode(file.filename, file.file)
    response.content_type = 'text/html'
    return response
<|endoftext|>


<commit_before><commit_msg>Delete foo.py<commit_after><|endoftext|>