<commit_before>#!/usr/bin/env python

import sys, os, re, collections, subprocess, sniper_lib, sniper_config

def ex_ret(cmd):
  return subprocess.Popen(cmd, stdout = subprocess.PIPE).communicate()[0]
def cppfilt(name):
  return ex_ret([ 'c++filt', name ])

class Function:
  def __init__(self, eip, name, location):
    self.eip = eip
    self.name = cppfilt(name).strip()
    self.location = location.split(':')
    self.imgname = self.location[0]
    self.sourcefile = self.location[2]
    self.sourceline = long(self.location[3])
    if self.sourceline:
      self.locationshort = '%s:%d' % (os.path.basename(self.sourcefile), self.sourceline)
    else:
      self.locationshort = os.path.basename(self.imgname)
  def __str__(self):
    return '[%12s]  %-20s %s' % (self.eip, self.name, self.locationshort)

def dirSum(a, b):
  for k, v in b.items():
    a[k] = a.get(k, 0) + v
  return a

class AllocationSite:
  def __init__(self, stack, numallocations, totalallocated, hitwhereload, hitwherestore, evictedby):
    self.stack = stack
    self.numallocations = numallocations
    self.totalallocated = totalallocated
    self.totalloads = sum(hitwhereload.values())
    self.hitwhereload = hitwhereload
    self.totalstores = sum(hitwherestore.values())
    self.hitwherestore = hitwherestore
    self.evictedby = evictedby
  def update(self, numallocations, totalallocated, hitwhereload, hitwherestore, evictedby):
    self.numallocations += numallocations
    self.totalallocated += totalallocated
    self.totalloads += sum(hitwhereload.values())
    self.hitwhereload = dirSum(self.hitwhereload, hitwhereload)
    self.totalstores += sum(hitwherestore.values())
    self.hitwherestore = dirSum(self.hitwherestore, hitwherestore)
    self.evictedby = dirSum(self.evictedby, evictedby)

def format_abs_ratio(val, tot):
  if tot:
    return '%12d  (%5.1f%%)' % (val, (100. * val) / tot)
  else:
    return '%12d          ' % val

class MemoryTracker:
  def __init__(self, resultsdir = '.'):
    filename = os.path.join(resultsdir, 'sim.memorytracker')
    if not os.path.exists(filename):
      raise IOError('Cannot find output file %s' % filename)

    results = sniper_lib.get_results(resultsdir = resultsdir)
    config = results['config']
    stats = results['results']

    self.hitwhere_load_global = dict([ (k.split('-', 3)[3], sum(v)) for k, v in stats.items() if k.startswith('L1-D.loads-where-') ])
    self.hitwhere_load_unknown = self.hitwhere_load_global.copy()
    self.hitwhere_store_global = dict([ (k.split('-', 3)[3], sum(v)) for k, v in stats.items() if k.startswith('L1-D.stores-where-') ])
    self.hitwhere_store_unknown = self.hitwhere_store_global.copy()

    llc_level = int(sniper_config.get_config(config, 'perf_model/cache/levels'))
    self.evicts_global = sum([ sum(v) for k, v in stats.items() if re.match('L%d.evict-.$' % llc_level, k) ])
    self.evicts_unknown = self.evicts_global

    self.functions = {}
    self.sites = {}
    self.siteids = {}

    fp = open(filename)
    for line in fp:
      if line.startswith('W\t'):
        self.hitwheres = line.strip().split('\t')[1].strip(',').split(',')
      elif line.startswith('F\t'):
        _, eip, name, location = line.strip().split('\t')
        self.functions[eip] = Function(eip, name, location)
      elif line.startswith('S\t'):
        line = line.strip().split('\t')
        siteid = line[1]
        stack = line[2].strip(':').split(':')
        stack = self.collapseStack(stack)
        results = { 'numallocations': 0, 'totalallocated': 0, 'hitwhereload': {}, 'hitwherestore': {}, 'evictedby': {} }
        for data in line[3:]:
          key, value = data.split('=')
          if key == 'num-allocations':
            results['numallocations'] = long(value)
          if key == 'total-allocated':
            results['totalallocated'] = long(value)
          elif key == 'hit-where':
            entries = map(lambda s: s.split(':'), value.strip(',').split(','))
            results['hitwhereload'] = dict([ (s[1:], long(v)) for s, v in entries if s.startswith('L') ])
            for k, v in results['hitwhereload'].items():
              self.hitwhere_load_unknown[k] -= v
            results['hitwherestore'] = dict([ (s[1:], long(v)) for s, v in entries if s.startswith('S') ])
            for k, v in results['hitwherestore'].items():
              self.hitwhere_store_unknown[k] -= v
          elif key == 'evicted-by':
            results['evictedby'] = dict(map(lambda (s, v): (s, long(v)), map(lambda s: s.split(':'), value.strip(',').split(','))))
            self.evicts_unknown -= sum(results['evictedby'].values())
        self.siteids[siteid] = stack
        if stack in self.sites:
          self.sites[stack].update(**results)
        else:
          self.sites[stack] = AllocationSite(stack, **results)
      else:
        raise ValueError('Invalid format %s' % line)
    #print ', '.join([ '%s:%d' % (k, v) for k, v in hitwhere_global.items() if v ])
    #print ', '.join([ '%s:%d' % (k, v) for k, v in hitwhere_unknown.items() if v ])
    #print evicts_global, evicts_unknown

  def collapseStack(self, stack):
    _stack = []
    for eip in stack:
      if eip == '0':
        continue
      if self.functions[eip].name in ('.plt',):
        continue
      if self.functions[eip].name.startswith('_dl_'):
        continue
      _stack.append(eip)
    return tuple(_stack)

  def write(self, obj):
    sites_sorted = sorted(self.sites.items(), key = lambda (k, v): v.totalloads + v.totalstores, reverse = True)
    site_names = dict([ (stack, '#%d' % (idx+1)) for idx, (stack, site) in enumerate(sites_sorted) ])
    totalloads = sum(self.hitwhere_load_global.values())
    totalstores = sum(self.hitwhere_store_global.values())

    for stack, site in sites_sorted:
      print >> obj, 'Site %s:' % site_names[stack]
      print >> obj, '\tCall stack:'
      for eip in site.stack:
        print >> obj, '\t\t%s' % self.functions[eip]
      print >> obj, '\tAllocations: %d' % site.numallocations
      print >> obj, '\tTotal allocated: %s (%s average)' % (sniper_lib.format_size(site.totalallocated), sniper_lib.format_size(site.totalallocated / site.numallocations))
      print >> obj, '\tHit-where:'
      print >> obj, '\t\t%-15s: %s' % ('Loads', format_abs_ratio(site.totalloads, totalloads)),
      print >> obj, '\t%-15s: %s' % ('Stores', format_abs_ratio(site.totalstores, totalstores))
      for hitwhere in self.hitwheres:
        if site.hitwhereload.get(hitwhere) or site.hitwherestore.get(hitwhere):
          cnt = site.hitwhereload[hitwhere]
          print >> obj, '\t\t  %-15s: %s' % (hitwhere, format_abs_ratio(cnt, site.totalloads)),
          cnt = site.hitwherestore[hitwhere]
          print >> obj, '\t  %-15s: %s' % (hitwhere, format_abs_ratio(cnt, site.totalstores))
      print >> obj, '\tEvicted-by:'
      evicts = {}
      for siteid, cnt in site.evictedby.items():
        _stack = self.siteids[siteid] if siteid != '0' else 'other'
        evicts[_stack] = evicts.get(_stack, 0) + cnt
      evicts = sorted(evicts.items(), key = lambda (stack, cnt): cnt, reverse = True)
      for _stack, cnt in evicts[:10]:
        name = site_names.get(_stack, 'other') if _stack != stack else 'self'
        print >> obj, '\t\t%-15s: %12d' % (name, cnt)
      print >> obj

    print >> obj, 'By hit-where:'
    for hitwhere in self.hitwheres:
      if self.hitwhere_load_global[hitwhere] + self.hitwhere_store_global[hitwhere]:
        totalloadhere = self.hitwhere_load_global[hitwhere]
        totalstorehere = self.hitwhere_store_global[hitwhere]
        print >> obj, '\t%s:' % hitwhere
        print >> obj, '\t\t%-15s: %s' % ('Loads', format_abs_ratio(totalloadhere, totalloads)),
        print >> obj, '\t%-15s: %s' % ('Stores', format_abs_ratio(totalstorehere, totalstores))
        for stack, site in sorted(self.sites.items(), key = lambda (k, v): v.hitwhereload.get(hitwhere, 0) + v.hitwherestore.get(hitwhere, 0), reverse = True):
          if site.hitwhereload.get(hitwhere) > .001 * totalloadhere or site.hitwherestore.get(hitwhere) > .001 * totalstorehere:
            print >> obj, '\t\t  %-15s: %s' % (site_names[stack], format_abs_ratio(site.hitwhereload.get(hitwhere), totalloadhere)),
            print >> obj, '\t  %-15s: %s' % (site_names[stack], format_abs_ratio(site.hitwherestore.get(hitwhere), totalstorehere))
        if self.hitwhere_load_unknown.get(hitwhere) > .001 * totalloadhere or self.hitwhere_store_unknown.get(hitwhere) > .001 * totalstorehere:
          print >> obj, '\t\t  %-15s: %s' % ('other', format_abs_ratio(self.hitwhere_load_unknown.get(hitwhere), totalloadhere)),
          print >> obj, '\t  %-15s: %s' % ('other', format_abs_ratio(self.hitwhere_store_unknown.get(hitwhere), totalstorehere))

if __name__ == '__main__':

  import getopt

  def usage():
    print '%s  [-d <resultsdir (.)> | -o <outputdir>]' % sys.argv[0]
    sys.exit(1)

  HOME = os.path.dirname(__file__)
  resultsdir = '.'
  outputdir = None

  try:
    opts, cmdline = getopt.getopt(sys.argv[1:], "hd:o:")
  except getopt.GetoptError, e:
    # print help information and exit:
    print >> sys.stderr, e
    usage()
  for o, a in opts:
    if o == '-h':
      usage()
      sys.exit()
    if o == '-d':
      resultsdir = a
    if o == '-o':
      outputdir = a

  result = MemoryTracker(resultsdir)
  result.write(file(os.path.join(outputdir, 'sim.memoryprofile'), 'w') if outputdir else sys.stdout)
<commit_msg>[memory_tracker] Print source code line contents when known<commit_after>#!/usr/bin/env python

import sys, os, re, collections, subprocess, sniper_lib, sniper_config

def ex_ret(cmd):
  return subprocess.Popen(cmd, stdout = subprocess.PIPE).communicate()[0]
def cppfilt(name):
  return ex_ret([ 'c++filt', name ])

def get_source_line(filename, linenr):
  for linenum, line in enumerate(file(filename)):
    if linenum+1 == linenr:
      return line.strip()
  return ''

class Function:
  def __init__(self, eip, name, location):
    self.eip = eip
    self.name = cppfilt(name).strip()
    self.location = location.split(':')
    self.imgname = self.location[0]
    self.sourcefile = self.location[2]
    self.sourceline = long(self.location[3])
    if self.sourceline:
      self.locationshort = '%s:%d' % (os.path.basename(self.sourcefile), self.sourceline)
      if os.path.exists(self.sourcefile):
        self.locationshort += '  { %s }' % get_source_line(self.sourcefile, self.sourceline)
    else:
      self.locationshort = os.path.basename(self.imgname)
  def __str__(self):
    return '[%12s]  %-20s %s' % (self.eip, self.name, self.locationshort)

def dirSum(a, b):
  for k, v in b.items():
    a[k] = a.get(k, 0) + v
  return a

class AllocationSite:
  def __init__(self, stack, numallocations, totalallocated, hitwhereload, hitwherestore, evictedby):
    self.stack = stack
    self.numallocations = numallocations
    self.totalallocated = totalallocated
    self.totalloads = sum(hitwhereload.values())
    self.hitwhereload = hitwhereload
    self.totalstores = sum(hitwherestore.values())
    self.hitwherestore = hitwherestore
    self.evictedby = evictedby
  def update(self, numallocations, totalallocated, hitwhereload, hitwherestore, evictedby):
    self.numallocations += numallocations
    self.totalallocated += totalallocated
    self.totalloads += sum(hitwhereload.values())
    self.hitwhereload = dirSum(self.hitwhereload, hitwhereload)
    self.totalstores += sum(hitwherestore.values())
    self.hitwherestore = dirSum(self.hitwherestore, hitwherestore)
    self.evictedby = dirSum(self.evictedby, evictedby)

def format_abs_ratio(val, tot):
  if tot:
    return '%12d  (%5.1f%%)' % (val, (100. * val) / tot)
  else:
    return '%12d          ' % val

class MemoryTracker:
  def __init__(self, resultsdir = '.'):
    filename = os.path.join(resultsdir, 'sim.memorytracker')
    if not os.path.exists(filename):
      raise IOError('Cannot find output file %s' % filename)

    results = sniper_lib.get_results(resultsdir = resultsdir)
    config = results['config']
    stats = results['results']

    self.hitwhere_load_global = dict([ (k.split('-', 3)[3], sum(v)) for k, v in stats.items() if k.startswith('L1-D.loads-where-') ])
    self.hitwhere_load_unknown = self.hitwhere_load_global.copy()
    self.hitwhere_store_global = dict([ (k.split('-', 3)[3], sum(v)) for k, v in stats.items() if k.startswith('L1-D.stores-where-') ])
    self.hitwhere_store_unknown = self.hitwhere_store_global.copy()

    llc_level = int(sniper_config.get_config(config, 'perf_model/cache/levels'))
    self.evicts_global = sum([ sum(v) for k, v in stats.items() if re.match('L%d.evict-.$' % llc_level, k) ])
    self.evicts_unknown = self.evicts_global

    self.functions = {}
    self.sites = {}
    self.siteids = {}

    fp = open(filename)
    for line in fp:
      if line.startswith('W\t'):
        self.hitwheres = line.strip().split('\t')[1].strip(',').split(',')
      elif line.startswith('F\t'):
        _, eip, name, location = line.strip().split('\t')
        self.functions[eip] = Function(eip, name, location)
      elif line.startswith('S\t'):
        line = line.strip().split('\t')
        siteid = line[1]
        stack = line[2].strip(':').split(':')
        stack = self.collapseStack(stack)
        results = { 'numallocations': 0, 'totalallocated': 0, 'hitwhereload': {}, 'hitwherestore': {}, 'evictedby': {} }
        for data in line[3:]:
          key, value = data.split('=')
          if key == 'num-allocations':
            results['numallocations'] = long(value)
          if key == 'total-allocated':
            results['totalallocated'] = long(value)
          elif key == 'hit-where':
            entries = map(lambda s: s.split(':'), value.strip(',').split(','))
            results['hitwhereload'] = dict([ (s[1:], long(v)) for s, v in entries if s.startswith('L') ])
            for k, v in results['hitwhereload'].items():
              self.hitwhere_load_unknown[k] -= v
            results['hitwherestore'] = dict([ (s[1:], long(v)) for s, v in entries if s.startswith('S') ])
            for k, v in results['hitwherestore'].items():
              self.hitwhere_store_unknown[k] -= v
          elif key == 'evicted-by':
            results['evictedby'] = dict(map(lambda (s, v): (s, long(v)), map(lambda s: s.split(':'), value.strip(',').split(','))))
            self.evicts_unknown -= sum(results['evictedby'].values())
        self.siteids[siteid] = stack
        if stack in self.sites:
          self.sites[stack].update(**results)
        else:
          self.sites[stack] = AllocationSite(stack, **results)
      else:
        raise ValueError('Invalid format %s' % line)
    #print ', '.join([ '%s:%d' % (k, v) for k, v in hitwhere_global.items() if v ])
    #print ', '.join([ '%s:%d' % (k, v) for k, v in hitwhere_unknown.items() if v ])
    #print evicts_global, evicts_unknown

  def collapseStack(self, stack):
    _stack = []
    for eip in stack:
      if eip == '0':
        continue
      name = self.functions[eip].name if eip in self.functions else '(unknown)'
      if name in ('.plt',):
        continue
      if name.startswith('_dl_'):
        continue
      _stack.append(eip)
    return tuple(_stack)

  def write(self, obj):
    sites_sorted = sorted(self.sites.items(), key = lambda (k, v): v.totalloads + v.totalstores, reverse = True)
    site_names = dict([ (stack, '#%d' % (idx+1)) for idx, (stack, site) in enumerate(sites_sorted) ])
    totalloads = sum(self.hitwhere_load_global.values())
    totalstores = sum(self.hitwhere_store_global.values())

    for stack, site in sites_sorted:
      print >> obj, 'Site %s:' % site_names[stack]
      print >> obj, '\tCall stack:'
      for eip in site.stack:
        print >> obj, '\t\t%s' % (self.functions[eip] if eip in self.functions else '(unknown)')
      print >> obj, '\tAllocations: %d' % site.numallocations
      print >> obj, '\tTotal allocated: %s (%s average)' % (sniper_lib.format_size(site.totalallocated), sniper_lib.format_size(site.totalallocated / site.numallocations))
      print >> obj, '\tHit-where:'
      print >> obj, '\t\t%-15s: %s' % ('Loads', format_abs_ratio(site.totalloads, totalloads)),
      print >> obj, '\t%-15s: %s' % ('Stores', format_abs_ratio(site.totalstores, totalstores))
      for hitwhere in self.hitwheres:
        if site.hitwhereload.get(hitwhere) or site.hitwherestore.get(hitwhere):
          cnt = site.hitwhereload[hitwhere]
          print >> obj, '\t\t  %-15s: %s' % (hitwhere, format_abs_ratio(cnt, site.totalloads)),
          cnt = site.hitwherestore[hitwhere]
          print >> obj, '\t  %-15s: %s' % (hitwhere, format_abs_ratio(cnt, site.totalstores))
      print >> obj, '\tEvicted-by:'
      evicts = {}
      for siteid, cnt in site.evictedby.items():
        _stack = self.siteids[siteid] if siteid != '0' else 'other'
        evicts[_stack] = evicts.get(_stack, 0) + cnt
      evicts = sorted(evicts.items(), key = lambda (stack, cnt): cnt, reverse = True)
      for _stack, cnt in evicts[:10]:
        name = site_names.get(_stack, 'other') if _stack != stack else 'self'
        print >> obj, '\t\t%-15s: %12d' % (name, cnt)
      print >> obj

    print >> obj, 'By hit-where:'
    for hitwhere in self.hitwheres:
      if self.hitwhere_load_global[hitwhere] + self.hitwhere_store_global[hitwhere]:
        totalloadhere = self.hitwhere_load_global[hitwhere]
        totalstorehere = self.hitwhere_store_global[hitwhere]
        print >> obj, '\t%s:' % hitwhere
        print >> obj, '\t\t%-15s: %s' % ('Loads', format_abs_ratio(totalloadhere, totalloads)),
        print >> obj, '\t%-15s: %s' % ('Stores', format_abs_ratio(totalstorehere, totalstores))
        for stack, site in sorted(self.sites.items(), key = lambda (k, v): v.hitwhereload.get(hitwhere, 0) + v.hitwherestore.get(hitwhere, 0), reverse = True):
          if site.hitwhereload.get(hitwhere) > .001 * totalloadhere or site.hitwherestore.get(hitwhere) > .001 * totalstorehere:
            print >> obj, '\t\t  %-15s: %s' % (site_names[stack], format_abs_ratio(site.hitwhereload.get(hitwhere), totalloadhere)),
            print >> obj, '\t  %-15s: %s' % (site_names[stack], format_abs_ratio(site.hitwherestore.get(hitwhere), totalstorehere))
        if self.hitwhere_load_unknown.get(hitwhere) > .001 * totalloadhere or self.hitwhere_store_unknown.get(hitwhere) > .001 * totalstorehere:
          print >> obj, '\t\t  %-15s: %s' % ('other', format_abs_ratio(self.hitwhere_load_unknown.get(hitwhere), totalloadhere)),
          print >> obj, '\t  %-15s: %s' % ('other', format_abs_ratio(self.hitwhere_store_unknown.get(hitwhere), totalstorehere))

if __name__ == '__main__':

  import getopt

  def usage():
    print '%s  [-d <resultsdir (.)> | -o <outputdir>]' % sys.argv[0]
    sys.exit(1)

  HOME = os.path.dirname(__file__)
  resultsdir = '.'
  outputdir = None

  try:
    opts, cmdline = getopt.getopt(sys.argv[1:], "hd:o:")
  except getopt.GetoptError, e:
    # print help information and exit:
    print >> sys.stderr, e
    usage()
  for o, a in opts:
    if o == '-h':
      usage()
      sys.exit()
    if o == '-d':
      resultsdir = a
    if o == '-o':
      outputdir = a

  result = MemoryTracker(resultsdir)
  result.write(file(os.path.join(outputdir, 'sim.memoryprofile'), 'w') if outputdir else sys.stdout)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import copy
import os
import sys

from saml2.metadata import entity_descriptor, metadata_tostring_fix
from saml2.metadata import entities_descriptor
from saml2.metadata import sign_entity_descriptor
from saml2.sigver import security_context
from saml2.validate import valid_instance
from saml2.config import Config

from satosa.backends.saml2 import SamlBackend
from satosa.frontends.saml2 import SamlFrontend
from satosa.image_converter import convert_to_base64
from satosa.plugin_loader import _load_plugins, backend_filter, frontend_filter
from satosa.satosa_config import SATOSAConfig

# =============================================================================
# Script that creates SAML2 metadata files from
# SATOSA Saml2 frontends and backends
# =============================================================================

parser = argparse.ArgumentParser()
parser.add_argument('-v', dest='valid', help="How long, in days, the metadata is valid from the time of creation")
parser.add_argument('-c', dest='cert', help='certificate')
parser.add_argument('-i', dest='id', help="The ID of the entities descriptor")
parser.add_argument('-k', dest='keyfile', help="A file with a key to sign the metadata with")
parser.add_argument('-n', dest='name', default="")
parser.add_argument('-s', dest='sign', action='store_true', help="sign the metadata")
parser.add_argument('-x', dest='xmlsec', help="xmlsec binaries to be used for the signing")
parser.add_argument('-f', dest="frontend", help='generate frontend metadata', action="store_true")
parser.add_argument('-b', dest="backend", help='generate backend metadata', action="store_true")
parser.add_argument('-o', dest="output", default=".", help='output path')
parser.add_argument(dest="config", nargs="+")
args = parser.parse_args()

generate_frontend = args.frontend
generate_backend = args.backend

# If no generate frontend/backend option, generate both
if not (args.frontend or args.backend):
    generate_frontend = True
    generate_backend = True

valid_for = 0
nspair = {"xs": "http://www.w3.org/2001/XMLSchema"}

if args.valid:
    # translate into hours
    valid_for = int(args.valid) * 24

def _make_metadata(config_dict):
    eds = []
    cnf = Config()

    config_dict = _convert_logo_images(config_dict)
    cnf.load(copy.deepcopy(config_dict), metadata_construction=True)

    if valid_for:
        cnf.valid_for = valid_for
    eds.append(entity_descriptor(cnf))

    conf = Config()
    conf.key_file = args.keyfile
    conf.cert_file = args.cert
    conf.debug = 1
    conf.xmlsec_binary = args.xmlsec
    secc = security_context(conf)

    if args.id:
        desc, xmldoc = entities_descriptor(eds, valid_for, args.name, args.id,
                                           args.sign, secc)
        valid_instance(desc)
        print(desc.to_string(nspair))
    else:
        for eid in eds:
            if args.sign:
                assert conf.key_file
                assert conf.cert_file
                eid, xmldoc = sign_entity_descriptor(eid, args.id, secc)
            else:
                xmldoc = None

            valid_instance(eid)
            xmldoc = metadata_tostring_fix(eid, nspair, xmldoc).decode()
            return xmldoc

def _convert_logo_images(config_dict):
    try:
        logo_list = config_dict['service']['idp']['ui_info']['logo']
        index = 0
        for logo in logo_list:
            try:
                logo['text'] = convert_to_base64(logo['text'])
                config_dict['service']['idp']['ui_info']['logo'][index] = logo
            except KeyError:
                pass
            index += 1
    except KeyError:
        pass

    return config_dict

for filespec in args.config:
    bas, fil = os.path.split(filespec)
    if bas != "":
        sys.path.insert(0, bas)

    config = SATOSAConfig(fil)
    metadata = {"backends": {}, "frontends": {}}
    backend_plugins = _load_plugins(config.PLUGIN_PATH, config.BACKEND_MODULES, backend_filter, config.BASE)
    frontend_plugins = _load_plugins(config.PLUGIN_PATH, config.FRONTEND_MODULES, frontend_filter, config.BASE)

    providers = []
    for plugin in backend_plugins:
        providers.append(plugin.name)
        if issubclass(plugin.module, SamlBackend) and generate_backend:
            metadata["backends"][plugin.name] = _make_metadata(plugin.config["config"])

    if generate_frontend:
        for plugin in frontend_plugins:
            if issubclass(plugin.module, SamlFrontend):
                module = plugin.module(None, None, plugin.config)
                module.register_endpoints(providers)
                metadata["frontends"][plugin.name] = _make_metadata(module.config)

    if generate_backend:
        for backend, data in metadata["backends"].items():
            file = open("%s/%s_backend_metadata.xml" % (args.output, backend), "w")
            file.write(data)
            file.close()
    if generate_frontend:
        for frontend, data in metadata["frontends"].items():
            file = open("%s/%s_frontend_metadata.xml" % (args.output, frontend), "w")
            file.write(data)
            file.close()
<commit_msg>Added some logging to make_saml_metadata<commit_after>#!/usr/bin/env python
import argparse
import copy
import logging
import os
import sys

from saml2.metadata import entity_descriptor, metadata_tostring_fix
from saml2.metadata import entities_descriptor
from saml2.metadata import sign_entity_descriptor
from saml2.sigver import security_context
from saml2.validate import valid_instance
from saml2.config import Config

from satosa.backends.saml2 import SamlBackend
from satosa.frontends.saml2 import SamlFrontend
from satosa.image_converter import convert_to_base64
from satosa.plugin_loader import _load_plugins, backend_filter, frontend_filter
from satosa.satosa_config import SATOSAConfig

# =============================================================================
# Script that creates SAML2 metadata files from
# SATOSA Saml2 frontends and backends
# =============================================================================

parser = argparse.ArgumentParser()
parser.add_argument('-v', dest='valid', help="How long, in days, the metadata is valid from the time of creation")
parser.add_argument('-c', dest='cert', help='certificate')
parser.add_argument('-i', dest='id', help="The ID of the entities descriptor")
parser.add_argument('-k', dest='keyfile', help="A file with a key to sign the metadata with")
parser.add_argument('-n', dest='name', default="")
parser.add_argument('-s', dest='sign', action='store_true', help="sign the metadata")
parser.add_argument('-x', dest='xmlsec', help="xmlsec binaries to be used for the signing")
parser.add_argument('-f', dest="frontend", help='generate frontend metadata', action="store_true")
parser.add_argument('-b', dest="backend", help='generate backend metadata', action="store_true")
parser.add_argument('-o', dest="output", default=".", help='output path')
parser.add_argument(dest="config", nargs="+")
args = parser.parse_args()

LOGGER = logging.getLogger("")
handler = logging.StreamHandler()
logFormatter = logging.Formatter("[%(name)-12.12s] [%(levelname)-5.5s]  %(message)s")
handler.setFormatter(logFormatter)
LOGGER.addHandler(handler)
LOGGER.setLevel(logging.INFO)

generate_frontend = args.frontend
generate_backend = args.backend

# If no generate frontend/backend option, generate both
if not (args.frontend or args.backend):
    generate_frontend = True
    generate_backend = True

LOGGER.info("Generating: frontends: %s, backends: %s" % (generate_frontend, generate_backend))

valid_for = 0
nspair = {"xs": "http://www.w3.org/2001/XMLSchema"}

if args.valid:
    # translate into hours
    valid_for = int(args.valid) * 24

def _make_metadata(config_dict):
    eds = []
    cnf = Config()

    config_dict = _convert_logo_images(config_dict)
    cnf.load(copy.deepcopy(config_dict), metadata_construction=True)

    if valid_for:
        cnf.valid_for = valid_for
    eds.append(entity_descriptor(cnf))

    conf = Config()
    conf.key_file = args.keyfile
    conf.cert_file = args.cert
    conf.debug = 1
    conf.xmlsec_binary = args.xmlsec
    secc = security_context(conf)

    if args.id:
        desc, xmldoc = entities_descriptor(eds, valid_for, args.name, args.id,
                                           args.sign, secc)
        valid_instance(desc)
        print(desc.to_string(nspair))
    else:
        for eid in eds:
            if args.sign:
                assert conf.key_file
                assert conf.cert_file
                eid, xmldoc = sign_entity_descriptor(eid, args.id, secc)
            else:
                xmldoc = None

            valid_instance(eid)
            xmldoc = metadata_tostring_fix(eid, nspair, xmldoc).decode()
            return xmldoc

def _convert_logo_images(config_dict):
    try:
        logo_list = config_dict['service']['idp']['ui_info']['logo']
        index = 0
        for logo in logo_list:
            try:
                logo['text'] = convert_to_base64(logo['text'])
                config_dict['service']['idp']['ui_info']['logo'][index] = logo
            except KeyError:
                pass
            index += 1
    except KeyError:
        pass

    return config_dict

for filespec in args.config:
    bas, fil = os.path.split(filespec)
    if bas != "":
        sys.path.insert(0, bas)

    config = SATOSAConfig(fil)
    metadata = {"backends": {}, "frontends": {}}
    backend_plugins = _load_plugins(config.PLUGIN_PATH, config.BACKEND_MODULES, backend_filter, config.BASE)
    frontend_plugins = _load_plugins(config.PLUGIN_PATH, config.FRONTEND_MODULES, frontend_filter, config.BASE)

    LOGGER.info("Loaded backend modules: %s" % backend_plugins)
    LOGGER.info("Loaded frontend modules: %s" % frontend_plugins)

    providers = []
    for plugin in backend_plugins:
        providers.append(plugin.name)
        if issubclass(plugin.module, SamlBackend) and generate_backend:
            LOGGER.info("Generating saml backend '%s' metadata..." % plugin.name)
            metadata["backends"][plugin.name] = _make_metadata(plugin.config["config"])

    LOGGER.info("Found backend modules: %s" % providers)

    if generate_frontend:
        for plugin in frontend_plugins:
            if issubclass(plugin.module, SamlFrontend):
                LOGGER.info("Generating saml frontend '%s' metadata..." % plugin.name)
                module = plugin.module(None, None, plugin.config)
                module.register_endpoints(providers)
                metadata["frontends"][plugin.name] = _make_metadata(module.config)

    if generate_backend:
        for backend, data in metadata["backends"].items():
            path = "%s/%s_backend_metadata.xml" % (args.output, backend)
            LOGGER.info("Writing backend '%s' metadata to '%s'" % (backend, path))
            file = open(path, "w")
            file.write(data)
            file.close()
    if generate_frontend:
        for frontend, data in metadata["frontends"].items():
            path = "%s/%s_frontend_metadata.xml" % (args.output, frontend)
            LOGGER.info("Writing frontend '%s' metadata to '%s'" % (frontend, path))
            file = open(path, "w")
            file.write(data)
            file.close()
<|endoftext|>


<commit_before># ------------------------------------------------------------------------------
# OCCAM
#
# Copyright (c) 2011-2012, SRI International
#
#  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of SRI International nor the names of its contributors may
#   be used to endorse or promote products derived from this software without
#   specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# ------------------------------------------------------------------------------

from . import config #LLVM, OCCAM_LIB, getStdTool
import subprocess, sys, os
import logging, tempfile
import shutil

def open_input(fn):
    if fn == '-':
        return os.fdopen(os.dup(0))
    else:
        return open(fn, 'r')

class ReturnCode (Exception):
    def __init__(self, value, cmd, proc):
        self._value = value
        self._proc = proc
        self._cmd = cmd

    def __str__(self):
        return "%s\nreturned %d" % (' '.join(self._cmd), self._value)

def runUnknown(tool,args):
    lenv = os.environ.copy()
    if lenv['OCCAM_PROTECT_PATH'] is not None:
        def which(name):
            for path in os.environ['OCCAM_PROTECTED_PATH'].split(os.pathsep):
                exe = os.path.join(path,name)
                if os.path.isfile(exe):
                    return exe
            return None
        tool = which(tool)
    cmdline = [tool] + args
    logging.getLogger().info('Calling unknown target: %s', ' '.join(cmdline))
    proc = subprocess.Popen(cmdline, env=lenv)
    retcode = proc.wait()
    logging.getLogger().info(' => %d', retcode)
    return retcode

def run(prog, args, quiet=False, inp=None,pipe=True, wd=None, resetPath=True):
    log = logging.getLogger()

    # 0 = stdin
    if inp is None:
        fd = os.fdopen(os.dup(0))
    else:
        fd = inp

    if quiet:
        err = subprocess.PIPE
    else:
        err = sys.stderr

    lenv = None
    if 'OCCAM_PROTECT_PATH' in os.environ:
        lenv = os.environ.copy()
        lenv['PATH'] = lenv['OCCAM_PROTECTED_PATH']
    elif resetPath:
        lenv = os.environ.copy()
        occam_home =  lenv["OCCAM_HOME"]
        if occam_home:
            pathelems = [e for e in lenv["PATH"].split(':') if os.path.abspath(occam_home) != os.path.abspath(e)]
        else:
            raise Exception("OCCAM_HOME not set properly in the environment")
        #pathelems = [e for e in lenv["PATH"].split(':') if e.find("occam") == -1]
        lenv["PATH"] = ':'.join(pathelems)

    info = ("\nPROG ", prog, "\nPATH ", lenv["PATH"], "\nresetPath ", str(resetPath), "\n")
    log.warn("run %s", ' '.join(info))

    if pipe:
        proc = subprocess.Popen([prog] + args,
                                stderr=err,
                                stdout=subprocess.PIPE,
                                stdin=fd,
                                cwd=wd,
                                env=lenv)
    else:
        proc = subprocess.Popen([prog] + args,
                                stderr=sys.stderr,
                                stdout=sys.stdout,
                                stdin=fd,
                                cwd=wd,
                                env=lenv)
    retcode = proc.wait()
    if inp is None:
        fd.close()
    if quiet:
        log.log(logging.INFO, 'EXECUTING: %(cmd)s => %(code)d\n%(err)s',
                {'cmd'  : ' '.join([prog] + args),
                 'code' : retcode,
                 'err'  : proc.stderr.read()})
    else:
        log.log(logging.INFO, 'EXECUTING: %(cmd)s => %(code)d',
                {'cmd'  : ' '.join([prog] + args),
                 'code' : retcode})

    if retcode != 0:
        ex = ReturnCode(retcode, [prog] + args, proc)
        logging.getLogger().error('ERROR: %s', ex)
        raise ex
    return retcode

def all_args(opt, args):
    result = []
    for x in args:
        result += [opt, x]
    return result

def previrt(fin, fout, args, **opts):
    args = ['-load=%s' % config.OCCAM_LIB,
            fin, '-o=%s' % fout] + args
    return run(config.LLVM['opt'], args, **opts)

def previrt_progress(fin, fout, args, output=None, **opts):
    args = [config.LLVM['opt'], '-load=%s' % config.OCCAM_LIB,
            fin, '-o=%s' % fout] + args
    proc = subprocess.Popen(args,
                            stderr=subprocess.PIPE,
                            stdout=subprocess.PIPE,
                            stdin=subprocess.PIPE)
    progress = proc.stderr.read()
    retcode = proc.wait()
    logging.getLogger().info('%(cmd)s => %(code)d\n%(progress)s',
                             {'cmd'  : ' '.join(args),
                              'code' : retcode,
                              'progress' : progress})
    if output != None:
        output[0] = progress
    return '...progress...' in progress

def isLLVM(f):
    proc = subprocess.Popen([config.getStdTool('file'), f],
                            stdout=subprocess.PIPE,
                            stdin=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    out = proc.stdout.read()
    proc.wait()
    return 'LLVM' in out

<commit_msg>Test equality of paths (to OCCAM_HOME) after normalizing and taking on the bin.<commit_after># ------------------------------------------------------------------------------
# OCCAM
#
# Copyright (c) 2011-2012, SRI International
#
#  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of SRI International nor the names of its contributors may
#   be used to endorse or promote products derived from this software without
#   specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# ------------------------------------------------------------------------------

from . import config #LLVM, OCCAM_LIB, getStdTool
import subprocess, sys, os
import logging, tempfile
import shutil

def open_input(fn):
    if fn == '-':
        return os.fdopen(os.dup(0))
    else:
        return open(fn, 'r')

class ReturnCode (Exception):
    def __init__(self, value, cmd, proc):
        self._value = value
        self._proc = proc
        self._cmd = cmd

    def __str__(self):
        return "%s\nreturned %d" % (' '.join(self._cmd), self._value)

def runUnknown(tool,args):
    lenv = os.environ.copy()
    if lenv['OCCAM_PROTECT_PATH'] is not None:
        def which(name):
            for path in os.environ['OCCAM_PROTECTED_PATH'].split(os.pathsep):
                exe = os.path.join(path,name)
                if os.path.isfile(exe):
                    return exe
            return None
        tool = which(tool)
    cmdline = [tool] + args
    logging.getLogger().info('Calling unknown target: %s', ' '.join(cmdline))
    proc = subprocess.Popen(cmdline, env=lenv)
    retcode = proc.wait()
    logging.getLogger().info(' => %d', retcode)
    return retcode

def run(prog, args, quiet=False, inp=None,pipe=True, wd=None, resetPath=True):
    log = logging.getLogger()

    # 0 = stdin
    if inp is None:
        fd = os.fdopen(os.dup(0))
    else:
        fd = inp

    if quiet:
        err = subprocess.PIPE
    else:
        err = sys.stderr

    lenv = None
    if 'OCCAM_PROTECT_PATH' in os.environ:
        lenv = os.environ.copy()
        lenv['PATH'] = lenv['OCCAM_PROTECTED_PATH']
    elif resetPath:
        lenv = os.environ.copy()
        occam_bin =  lenv["OCCAM_BIN"]
        if occam_bin:
            pathelems = [e for e in lenv["PATH"].split(':') if os.path.abspath(occam_bin) != os.path.abspath(e)]
        else:
            raise Exception("OCCAM_BIN not set properly in the environment")
        #pathelems = [e for e in lenv["PATH"].split(':') if e.find("occam") == -1]
        lenv["PATH"] = ':'.join(pathelems)

    info = ("\nPROG ", prog, "\nPATH ", lenv["PATH"], "\nresetPath ", str(resetPath), "\n")
    log.warn("run %s", ' '.join(info))

    if pipe:
        proc = subprocess.Popen([prog] + args,
                                stderr=err,
                                stdout=subprocess.PIPE,
                                stdin=fd,
                                cwd=wd,
                                env=lenv)
    else:
        proc = subprocess.Popen([prog] + args,
                                stderr=sys.stderr,
                                stdout=sys.stdout,
                                stdin=fd,
                                cwd=wd,
                                env=lenv)
    retcode = proc.wait()
    if inp is None:
        fd.close()
    if quiet:
        log.log(logging.INFO, 'EXECUTING: %(cmd)s => %(code)d\n%(err)s',
                {'cmd'  : ' '.join([prog] + args),
                 'code' : retcode,
                 'err'  : proc.stderr.read()})
    else:
        log.log(logging.INFO, 'EXECUTING: %(cmd)s => %(code)d',
                {'cmd'  : ' '.join([prog] + args),
                 'code' : retcode})

    if retcode != 0:
        ex = ReturnCode(retcode, [prog] + args, proc)
        logging.getLogger().error('ERROR: %s', ex)
        raise ex
    return retcode

def all_args(opt, args):
    result = []
    for x in args:
        result += [opt, x]
    return result

def previrt(fin, fout, args, **opts):
    args = ['-load=%s' % config.OCCAM_LIB,
            fin, '-o=%s' % fout] + args
    return run(config.LLVM['opt'], args, **opts)

def previrt_progress(fin, fout, args, output=None, **opts):
    args = [config.LLVM['opt'], '-load=%s' % config.OCCAM_LIB,
            fin, '-o=%s' % fout] + args
    proc = subprocess.Popen(args,
                            stderr=subprocess.PIPE,
                            stdout=subprocess.PIPE,
                            stdin=subprocess.PIPE)
    progress = proc.stderr.read()
    retcode = proc.wait()
    logging.getLogger().info('%(cmd)s => %(code)d\n%(progress)s',
                             {'cmd'  : ' '.join(args),
                              'code' : retcode,
                              'progress' : progress})
    if output != None:
        output[0] = progress
    return '...progress...' in progress

def isLLVM(f):
    proc = subprocess.Popen([config.getStdTool('file'), f],
                            stdout=subprocess.PIPE,
                            stdin=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    out = proc.stdout.read()
    proc.wait()
    return 'LLVM' in out

<|endoftext|>


<commit_before>"""Active learning by learning (ALBL)

This module includes two classes. ActiveLearningByLearning is the main
algorithm for ALBL and Exp4P is the multi-armed bandit algorithm which will be
used in ALBL.

"""
from libact.base.interfaces import QueryStrategy
import numpy as np
import copy

class ActiveLearningByLearning(QueryStrategy):
    """Active Learning By Learning (ALBL) query strategy.

    ALBL is an active learning algorithm that adaptively choose among existing
    query strategies to decide which data to make query. It utilizes
    Exp4.P, a multi-armed bandit algorithm to adaptively make such decision.
    More details of ALBL can refer to the work listed in the reference section.

    Parameters
    ----------
    models: list of libact.query_strategies.* object instance
        The active learning algorithms used in ALBL, which will be both the
        the arms in the multi-armed bandit algorithm Exp4.P.

    delta: float, optional (default=0.1)
        Parameter for Exp4.P.

    pmin: float, 0<pmin<len(n_active_algorithm), optional (default=0.05)
        Parameter for Exp4.P. The minimal probability for random selection of
        the arms (aka the underlying active learning algorithms).

    uniform_sampler: {True, False}, optional (default=Truee)
        Determining whether to include uniform random sample as one of arms.

    T: integer, optional (default=100)
        Query budget, the maximal number of queries to be made.

    clf: libact.model.* object instance
        The learning model used for the task.

    Attributes
    ----------
    models_: list of libact.query_strategies.* object instance

    exp4p_: instance of Exp4P object

    queried_hist_: list of integer
        A list of entry_id of the dataset which is queried in the past.

    Reference
    ---------

    .. [1] Wei-Ning Hsu, and Hsuan-Tien Lin. "Active Learning by Learning."
           Twenty-Ninth AAAI Conference on Artificial Intelligence. 2015.
    """

    def __init__(self, *args, **kwargs):
        print("XD")
        super(ActiveLearningByLearning, self).__init__(*args, **kwargs)
        self.models_ = kwargs.pop('models', None)
        if self.models_ is None:
            raise TypeError(
                "__init__() missing required keyword-only argument: 'models'"
                )
        elif not self.models_:
            raise ValueError("models list is empty")

        # parameters for Exp4.p
        self.delta = kwargs.pop('delta', 0.1)

        # query budget
        self.T = kwargs.pop('T', 100)

        self.unlabeled_entry_ids, X_pool = \
            zip(*self.dataset.get_unlabeled_entries())
        self.unlabeled_invert_id_idx = {}
        for i, entry in enumerate(self.dataset.get_unlabeled_entries()):
            self.unlabeled_invert_id_idx[entry[0]] = i

        self.uniform_sampler = kwargs.pop('uniform_sampler', True)
        if type(self.uniform_sampler) != type(True):
            raise ValueError("'uniform_sampler' should be {True, False}")

        self.pmin = kwargs.pop('pmin', None)
        if self.pmin and (self.pmin >= len(self.models_)+self.uniform_sampler
                          or self.pmin <= 0):
            raise ValueError("'pmin' should be 0 < pmin < "
                             "len(n_active_algorithm)")

        self.exp4p_ = Exp4P(
            query_models=self.models_,
            T=self.T,
            delta=self.delta,
            pmin=self.pmin,
            unlabeled_invert_id_idx=self.unlabeled_invert_id_idx,
            uniform_sampler=self.uniform_sampler
        )
        self.budget_used = 0

        # classifier instance
        self.clf = kwargs.pop('clf', None)
        if self.clf is None:
            raise TypeError(
                "__init__() missing required keyword-only argument: 'models'"
                )

        self.W = []
        self.queried_hist_ = []

    def calc_reward_fn(self):
        """Calculate the reward value"""
        clf = copy.copy(self.clf)
        clf.train(self.dataset)

        # reward function: Importance-Weighted-Accuracy (IW-ACC) (tau, f)
        reward = 0.
        for i in range(len(self.queried_hist_)):
            reward += self.W[i] *\
                (clf.predict(self.dataset.data[self.queried_hist_[i]][0])[0] ==
                 self.dataset.data[self.queried_hist_[i]][1])
        reward /= (self.dataset.len_labeled() + self.dataset.len_unlabeled())
        reward /= self.T
        return reward

    def calc_query(self):
        """Calculate which point to query"""
        dataset = self.dataset
        try:
            unlabeled_entry_ids, X_pool = zip(*dataset.get_unlabeled_entries())
        except ValueError:
            # might be no more unlabeled data left
            return

        while self.budget_used < self.T:
            q = self.exp4p_.next(
                self.calc_reward_fn(),
                self.queried_hist_[-1],
                self.dataset.data[self.queried_hist_[-1]][1]
            )
            ask_idx = np.random.choice(
                        np.arange(
                            len(self.unlabeled_invert_id_idx)), size=1, p=q
                    )[0]
            ask_id = self.unlabeled_entry_ids[ask_idx]
            self.W.append(1./q[ask_idx])
            self.queried_hist_.append(ask_id)

            if ask_id in unlabeled_entry_ids:
                self.budget_used += 1
                return

    def update(self, entry_id, label):
        """Calculate the next query after updating the question asked with an
        answer."""
        self.calc_query()

    def make_query(self):
        """Except for the initial query, it returns the id of the data albl
        wants to query."""
        if self.queried_hist_ == []:
            # initial query
            q = self.exp4p_.next(-1, None, None)
            ask_idx = np.random.choice(
                        np.arange(len(self.unlabeled_invert_id_idx)), size=1, p=q
                    )[0]
            ask_id = self.unlabeled_entry_ids[ask_idx]
            self.W.append(1./q[ask_idx])
            self.queried_hist_.append(ask_id)
        return self.queried_hist_[-1]

class Exp4P():
    """A multi-armed bandit algorithm Exp4.P.

    For the Exp4.P used in ALBL, the number of arms (actions) and number of
    experts are equal to the number of active learning algorithms wanted to
    use. The arms (actions) are the active learning algorithms, where is
    inputed from parameter 'query_models'. There is no need for the input of
    experts, the advice of the kth expert are always equal e_k, where e_k is
    the kth column of the identity matrix.

    Parameters
    ----------
    query_models: QueryStrategy instances
        The active learning algorithms wanted to use, it is equivalent to
        actions or arms in original Exp4.P.

    unlabeled_invert_id_idx: dictionary
        A look up table for the correspondance of entry_id to the index of the
        unlabeled data.

    delta: float, >0, optional (default=0.1)
        A parameter.

    pmin: float, 0<pmin<1/len(query_models), optional (default=:math:`\frac{√{log(N)}{KT}`)
        The minimal probability for random selection of the arms (aka the
        unlabeled data).

    T: integer, optional (default=100)
        The maximum number of rounds.

    uniform_sampler: {True, False}, optional (default=Truee)
        Determining whether to include uniform random sampler as one of the
        underlying active learning algorithms.

    Attributes
    ----------
    t: integer
        The current round this instance is at.

    N: integer
        The number of arms (actions) in this exp4.p instance.

    quert_models_: list of libact.query_strategies.* object instance
        The underlying active learning algorithm instances.

    Reference
    ---------
    .. [1] Beygelzimer, Alina, et al. "Contextual bandit algorithms with
           supervised learning guarantees." In Proceedings on the International
           Conference on Artificial Intelligence and Statistics (AISTATS),
           2011u.
    """

    def __init__(self, *args, **kwargs):
        """ """
        # QueryStrategy class object instances
        self.query_models_ = kwargs.pop('query_models', None)
        if self.query_models_ is None:
            raise TypeError(
                "__init__() missing required keyword-only argument: 'query_models'"
                )
        elif not self.query_models_:
            raise ValueError("query_models list is empty")

        # whether to include uniform random sampler as one of underlying active
        # learning algorithms
        self.uniform_sampler = kwargs.pop('uniform_sampler', True)

        # n_armss
        if self.uniform_sampler:
            self.N = len(self.query_models_) + 1
        else:
            self.N = len(self.query_models_)

        # weight vector to each query_models, shape = (N, )
        self.w = np.array([1. for i in range(self.N)])

        # max iters
        self.T = kwargs.pop('T', 100)

        # delta > 0
        self.delta = kwargs.pop('delta', 0.1)

        # n_arms = n_models (n_query_algorithms) in ALBL
        self.K = self.N

        # p_min in [0, 1/n_arms]
        self.pmin = kwargs.pop('pmin', None)
        if self.pmin == None:
            self.pmin = np.sqrt(np.log(self.N) / self.K / self.T)

        self.exp4p_gen = self.exp4p()

        self.unlabeled_invert_id_idx = kwargs.pop('unlabeled_invert_id_idx')
        if not self.unlabeled_invert_id_idx:
            raise TypeError(
                "__init__() missing required keyword-only argument:"
                "'unlabeled_invert_id_idx'"
                )

    def __next__(self, reward, ask_id, lbl):
        """For Python3 compatibility of generator."""
        return self.next(reward, ask_id, lbl)

    def next(self, reward, ask_id, lbl):
        """Taking the label and the reward value of last question and returns
        the next question to ask."""
        # first run don't have reward, TODO exception on reward == -1 only once
        if reward == -1:
            return next(self.exp4p_gen)
        else:
        # TODO exception on reward in [0, 1]
            return self.exp4p_gen.send((reward, ask_id, lbl))

    def update_query_models(self, qid, lbl):
        """Update model's dataset after making a query."""
        for model in self.query_models_:
            model.update(qid, lbl)

    def exp4p(self):
        """The generator which implements the main part of Exp4.P.

        Yield
        -------
        p: array-like, shape = [K]
            The query vector which tells ALBL what kind of distribution if
            should sample from the unlabeled pool.

        Send
        ----
        reward: float
            The reward value calculated from ALBL.

        ask_id: integer
            The entry_id of the sample point ALBL asked.

        lbl: integer
            The answer received from asking the entry_id ask_id.
        """
        while True:
            #TODO probabilistic active learning algorithm
            # len(self.unlabeled_invert_id_idx) is the number of unlabeled data
            query = np.zeros((self.N, len(self.unlabeled_invert_id_idx)))
            if self.uniform_sampler:
                query[-1, :] = 1. / len(self.unlabeled_invert_id_idx)
            for i, model in enumerate(self.query_models_):
                query[i][self.unlabeled_invert_id_idx[model.make_query()]] = 1

            # choice vector, shape = (self.K, )
            W = np.sum(self.w)
            p = (1 - self.K * self.pmin) * self.w / W + self.pmin

            # query vector, shape= = (self.n_unlabeled, )
            q = np.dot(p, query)

            reward, ask_id, lbl = yield q
            self.update_query_models(ask_id, lbl)
            ask_idx = self.unlabeled_invert_id_idx[ask_id]

            rhat = reward * query[:, ask_idx] / q[ask_idx]

            # The original advice vector in Exp4.P in ALBL is a identity matrix
            #yhat = np.dot(advice, rhat)
            yhat = rhat
            #vhat = np.sum(advice / np.tile(p, (self.N, 1)), axis=1)
            vhat = 1 / p
            self.w = self.w * np.exp(
                    self.pmin / 2 * (yhat + vhat*np.sqrt(
                        np.log(self.N/self.delta) / self.K / self.T)
                        )
                    )

        raise StopIteration
<commit_msg>fix albl's doc typo<commit_after>"""Active learning by learning (ALBL)

This module includes two classes. ActiveLearningByLearning is the main
algorithm for ALBL and Exp4P is the multi-armed bandit algorithm which will be
used in ALBL.

"""
from libact.base.interfaces import QueryStrategy
import numpy as np
import copy

class ActiveLearningByLearning(QueryStrategy):
    """Active Learning By Learning (ALBL) query strategy.

    ALBL is an active learning algorithm that adaptively choose among existing
    query strategies to decide which data to make query. It utilizes
    Exp4.P, a multi-armed bandit algorithm to adaptively make such decision.
    More details of ALBL can refer to the work listed in the reference section.

    Parameters
    ----------
    models: list of libact.query_strategies.* object instance
        The active learning algorithms used in ALBL, which will be both the
        the arms in the multi-armed bandit algorithm Exp4.P.

    delta: float, optional (default=0.1)
        Parameter for Exp4.P.

    pmin: float, 0<pmin<len(n_active_algorithm), optional (default=0.05)
        Parameter for Exp4.P. The minimal probability for random selection of
        the arms (aka the underlying active learning algorithms).

    uniform_sampler: {True, False}, optional (default=Truee)
        Determining whether to include uniform random sample as one of arms.

    T: integer, optional (default=100)
        Query budget, the maximal number of queries to be made.

    clf: libact.model.* object instance
        The learning model used for the task.

    Attributes
    ----------
    models_: list of libact.query_strategies.* object instance

    exp4p_: instance of Exp4P object

    queried_hist_: list of integer
        A list of entry_id of the dataset which is queried in the past.

    Reference
    ---------

    .. [1] Wei-Ning Hsu, and Hsuan-Tien Lin. "Active Learning by Learning."
           Twenty-Ninth AAAI Conference on Artificial Intelligence. 2015.
    """

    def __init__(self, *args, **kwargs):
        print("XD")
        super(ActiveLearningByLearning, self).__init__(*args, **kwargs)
        self.models_ = kwargs.pop('models', None)
        if self.models_ is None:
            raise TypeError(
                "__init__() missing required keyword-only argument: 'models'"
                )
        elif not self.models_:
            raise ValueError("models list is empty")

        # parameters for Exp4.p
        self.delta = kwargs.pop('delta', 0.1)

        # query budget
        self.T = kwargs.pop('T', 100)

        self.unlabeled_entry_ids, X_pool = \
            zip(*self.dataset.get_unlabeled_entries())
        self.unlabeled_invert_id_idx = {}
        for i, entry in enumerate(self.dataset.get_unlabeled_entries()):
            self.unlabeled_invert_id_idx[entry[0]] = i

        self.uniform_sampler = kwargs.pop('uniform_sampler', True)
        if type(self.uniform_sampler) != type(True):
            raise ValueError("'uniform_sampler' should be {True, False}")

        self.pmin = kwargs.pop('pmin', None)
        if self.pmin and (self.pmin >= len(self.models_)+self.uniform_sampler
                          or self.pmin <= 0):
            raise ValueError("'pmin' should be 0 < pmin < "
                             "len(n_active_algorithm)")

        self.exp4p_ = Exp4P(
            query_models=self.models_,
            T=self.T,
            delta=self.delta,
            pmin=self.pmin,
            unlabeled_invert_id_idx=self.unlabeled_invert_id_idx,
            uniform_sampler=self.uniform_sampler
        )
        self.budget_used = 0

        # classifier instance
        self.clf = kwargs.pop('clf', None)
        if self.clf is None:
            raise TypeError(
                "__init__() missing required keyword-only argument: 'models'"
                )

        self.W = []
        self.queried_hist_ = []

    def calc_reward_fn(self):
        """Calculate the reward value"""
        clf = copy.copy(self.clf)
        clf.train(self.dataset)

        # reward function: Importance-Weighted-Accuracy (IW-ACC) (tau, f)
        reward = 0.
        for i in range(len(self.queried_hist_)):
            reward += self.W[i] *\
                (clf.predict(self.dataset.data[self.queried_hist_[i]][0])[0] ==
                 self.dataset.data[self.queried_hist_[i]][1])
        reward /= (self.dataset.len_labeled() + self.dataset.len_unlabeled())
        reward /= self.T
        return reward

    def calc_query(self):
        """Calculate which point to query"""
        dataset = self.dataset
        try:
            unlabeled_entry_ids, X_pool = zip(*dataset.get_unlabeled_entries())
        except ValueError:
            # might be no more unlabeled data left
            return

        while self.budget_used < self.T:
            q = self.exp4p_.next(
                self.calc_reward_fn(),
                self.queried_hist_[-1],
                self.dataset.data[self.queried_hist_[-1]][1]
            )
            ask_idx = np.random.choice(
                        np.arange(
                            len(self.unlabeled_invert_id_idx)), size=1, p=q
                    )[0]
            ask_id = self.unlabeled_entry_ids[ask_idx]
            self.W.append(1./q[ask_idx])
            self.queried_hist_.append(ask_id)

            if ask_id in unlabeled_entry_ids:
                self.budget_used += 1
                return

    def update(self, entry_id, label):
        """Calculate the next query after updating the question asked with an
        answer."""
        self.calc_query()

    def make_query(self):
        """Except for the initial query, it returns the id of the data albl
        wants to query."""
        if self.queried_hist_ == []:
            # initial query
            q = self.exp4p_.next(-1, None, None)
            ask_idx = np.random.choice(
                        np.arange(len(self.unlabeled_invert_id_idx)), size=1, p=q
                    )[0]
            ask_id = self.unlabeled_entry_ids[ask_idx]
            self.W.append(1./q[ask_idx])
            self.queried_hist_.append(ask_id)
        return self.queried_hist_[-1]

class Exp4P():
    """A multi-armed bandit algorithm Exp4.P.

    For the Exp4.P used in ALBL, the number of arms (actions) and number of
    experts are equal to the number of active learning algorithms wanted to
    use. The arms (actions) are the active learning algorithms, where is
    inputed from parameter 'query_models'. There is no need for the input of
    experts, the advice of the kth expert are always equal e_k, where e_k is
    the kth column of the identity matrix.

    Parameters
    ----------
    query_models: QueryStrategy instances
        The active learning algorithms wanted to use, it is equivalent to
        actions or arms in original Exp4.P.

    unlabeled_invert_id_idx: dictionary
        A look up table for the correspondance of entry_id to the index of the
        unlabeled data.

    delta: float, >0, optional (default=0.1)
        A parameter.

    pmin: float, 0<pmin<1/len(query_models), optional (default=:math:`\frac{√{log(N)}{KT}`)
        The minimal probability for random selection of the arms (aka the
        unlabeled data).

    T: integer, optional (default=100)
        The maximum number of rounds.

    uniform_sampler: {True, False}, optional (default=Truee)
        Determining whether to include uniform random sampler as one of the
        underlying active learning algorithms.

    Attributes
    ----------
    t: integer
        The current round this instance is at.

    N: integer
        The number of arms (actions) in this exp4.p instance.

    quert_models_: list of libact.query_strategies.* object instance
        The underlying active learning algorithm instances.

    Reference
    ---------
    .. [1] Beygelzimer, Alina, et al. "Contextual bandit algorithms with
           supervised learning guarantees." In Proceedings on the International
           Conference on Artificial Intelligence and Statistics (AISTATS),
           2011u.
    """

    def __init__(self, *args, **kwargs):
        """ """
        # QueryStrategy class object instances
        self.query_models_ = kwargs.pop('query_models', None)
        if self.query_models_ is None:
            raise TypeError(
                "__init__() missing required keyword-only argument: 'query_models'"
                )
        elif not self.query_models_:
            raise ValueError("query_models list is empty")

        # whether to include uniform random sampler as one of underlying active
        # learning algorithms
        self.uniform_sampler = kwargs.pop('uniform_sampler', True)

        # n_armss
        if self.uniform_sampler:
            self.N = len(self.query_models_) + 1
        else:
            self.N = len(self.query_models_)

        # weight vector to each query_models, shape = (N, )
        self.w = np.array([1. for i in range(self.N)])

        # max iters
        self.T = kwargs.pop('T', 100)

        # delta > 0
        self.delta = kwargs.pop('delta', 0.1)

        # n_arms = n_models (n_query_algorithms) in ALBL
        self.K = self.N

        # p_min in [0, 1/n_arms]
        self.pmin = kwargs.pop('pmin', None)
        if self.pmin == None:
            self.pmin = np.sqrt(np.log(self.N) / self.K / self.T)

        self.exp4p_gen = self.exp4p()

        self.unlabeled_invert_id_idx = kwargs.pop('unlabeled_invert_id_idx')
        if not self.unlabeled_invert_id_idx:
            raise TypeError(
                "__init__() missing required keyword-only argument:"
                "'unlabeled_invert_id_idx'"
                )

    def __next__(self, reward, ask_id, lbl):
        """For Python3 compatibility of generator."""
        return self.next(reward, ask_id, lbl)

    def next(self, reward, ask_id, lbl):
        """Taking the label and the reward value of last question and returns
        the next question to ask."""
        # first run don't have reward, TODO exception on reward == -1 only once
        if reward == -1:
            return next(self.exp4p_gen)
        else:
        # TODO exception on reward in [0, 1]
            return self.exp4p_gen.send((reward, ask_id, lbl))

    def update_query_models(self, qid, lbl):
        """Update model's dataset after making a query."""
        for model in self.query_models_:
            model.update(qid, lbl)

    def exp4p(self):
        """The generator which implements the main part of Exp4.P.

        Yields
        ------
        p: array-like, shape = [K]
            The query vector which tells ALBL what kind of distribution if
            should sample from the unlabeled pool.

        Send
        ----
        reward: float
            The reward value calculated from ALBL.

        ask_id: integer
            The entry_id of the sample point ALBL asked.

        lbl: integer
            The answer received from asking the entry_id ask_id.
        """
        while True:
            #TODO probabilistic active learning algorithm
            # len(self.unlabeled_invert_id_idx) is the number of unlabeled data
            query = np.zeros((self.N, len(self.unlabeled_invert_id_idx)))
            if self.uniform_sampler:
                query[-1, :] = 1. / len(self.unlabeled_invert_id_idx)
            for i, model in enumerate(self.query_models_):
                query[i][self.unlabeled_invert_id_idx[model.make_query()]] = 1

            # choice vector, shape = (self.K, )
            W = np.sum(self.w)
            p = (1 - self.K * self.pmin) * self.w / W + self.pmin

            # query vector, shape= = (self.n_unlabeled, )
            q = np.dot(p, query)

            reward, ask_id, lbl = yield q
            self.update_query_models(ask_id, lbl)
            ask_idx = self.unlabeled_invert_id_idx[ask_id]

            rhat = reward * query[:, ask_idx] / q[ask_idx]

            # The original advice vector in Exp4.P in ALBL is a identity matrix
            #yhat = np.dot(advice, rhat)
            yhat = rhat
            #vhat = np.sum(advice / np.tile(p, (self.N, 1)), axis=1)
            vhat = 1 / p
            self.w = self.w * np.exp(
                    self.pmin / 2 * (yhat + vhat*np.sqrt(
                        np.log(self.N/self.delta) / self.K / self.T)
                        )
                    )

        raise StopIteration<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import os.path
import sys
import subprocess

from txclib import utils
from txclib.project import Project
try:
    from json import loads as parse_json
except ImportError:
    from simplejson import loads as parse_json

class UiLanguagesCheck(Project):
    def get_ui_langs(self):
        our_path = os.path.dirname(os.path.realpath(__file__))
        core_file = our_path + '/../app/Config/core.php'
        php_script = ('define("WWW_ROOT", "");'
                      'define("ROOT", "");'
                      'define("DS", "");'
                      'include("src/Lib/LanguagesLib.php");'
                      '$conf = include("config/app_local.php");'
                      'print json_encode(array_map('
                      '    function($v) {'
                      '        return \App\Lib\LanguagesLib::languageTag($v[0]);'
                      '    },'
                      '    $conf["UI"]["languages"]'
                      '));')
        php_cmd = "php -r '" + php_script + "'"
        proc = subprocess.Popen(php_cmd, shell=True, stdout=subprocess.PIPE)
        return parse_json(proc.stdout.read())

    def run(self, resource=None):
        if not resource:
            resource = self.get_resource_list()[0]
        try:
            project_slug, resource_slug = resource.split('.', 1)
        except ValueError:
            logger.error("Invalid resource name: {}".format(resource))
            sys.exit(1)

        lang_map = self.get_resource_lang_mapping(resource)
        ui_langs = self.get_ui_langs()

        self.url_info = {
            'host': self.get_resource_host(None),
            'project': project_slug,
            'resource': resource_slug
        }
        all_stats = self._get_stats_for_resource()

        stats_iter = sorted(all_stats.iteritems(), key=lambda (k,v): int(v['completed'][:-1]))
        print("{:3s}      [{}]".format('lang', 'is included in app_local.php'))
        for tx_code, lang_stats in stats_iter:
            try:
                our_code = lang_map[tx_code]
            except KeyError:
                our_code = tx_code
            available = our_code in ui_langs
            print("{:3s}: {:>4s} [{}]".format(our_code, lang_stats['completed'], 'X' if available else ' '))

def main(argv):
    path_to_tx = utils.find_dot_tx()

    check = UiLanguagesCheck(path_to_tx)
    check.run()

if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>Make that script work with both Python 2 and 3<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import os.path
import sys
import subprocess

from txclib import utils
from txclib.project import Project
try:
    from json import loads as parse_json
except ImportError:
    from simplejson import loads as parse_json

class UiLanguagesCheck(Project):
    def get_ui_langs(self):
        our_path = os.path.dirname(os.path.realpath(__file__))
        core_file = our_path + '/../app/Config/core.php'
        php_script = ('define("WWW_ROOT", "");'
                      'define("ROOT", "");'
                      'define("DS", "");'
                      'include("src/Lib/LanguagesLib.php");'
                      '$conf = include("config/app_local.php");'
                      'print json_encode(array_map('
                      '    function($v) {'
                      '        return \App\Lib\LanguagesLib::languageTag($v[0]);'
                      '    },'
                      '    $conf["UI"]["languages"]'
                      '));')
        php_cmd = "php -r '" + php_script + "'"
        proc = subprocess.Popen(php_cmd, shell=True, stdout=subprocess.PIPE)
        return parse_json(proc.stdout.read())

    def run(self, resource=None):
        if not resource:
            resource = self.get_resource_list()[0]
        try:
            project_slug, resource_slug = resource.split('.', 1)
        except ValueError:
            logger.error("Invalid resource name: {}".format(resource))
            sys.exit(1)

        lang_map = self.get_resource_lang_mapping(resource)
        ui_langs = self.get_ui_langs()

        self.url_info = {
            'host': self.get_resource_host(None),
            'project': project_slug,
            'resource': resource_slug
        }
        all_stats = self._get_stats_for_resource()

        stats_iter = sorted(all_stats.items(), key=lambda kv: int(kv[1]['completed'][:-1]))
        print("{:3s}      [{}]".format('lang', 'is included in app_local.php'))
        for tx_code, lang_stats in stats_iter:
            try:
                our_code = lang_map[tx_code]
            except KeyError:
                our_code = tx_code
            available = our_code in ui_langs
            print("{:3s}: {:>4s} [{}]".format(our_code, lang_stats['completed'], 'X' if available else ' '))

def main(argv):
    path_to_tx = utils.find_dot_tx()

    check = UiLanguagesCheck(path_to_tx)
    check.run()

if __name__ == "__main__":
    main(sys.argv[1:])
<|endoftext|>