<commit_before>'''
FeatherDuster - A wizard-like interface to cryptanalib
Author: Daniel Crowley

FeatherDuster is a tool for brushing away magical crypto fairy dust.
'''

import sys
import os
import readline
import completer
from ishell.console import Console
from ishell.command import Command

try:
   from IPython import embed
except ImportError:
   import code

   def embed():
      vars = globals()
      vars.update(locals())
      shell = code.InteractiveConsole(vars)
      shell.interact()

import feathermodules

feathermodules.samples = []
feathermodules.results = False
feathermodules.selected_attack_name = ''
feathermodules.current_options = {}

from feathermodules.stream import *
from feathermodules.block import *
from feathermodules.classical import *
from feathermodules.auxiliary import *
from feathermodules.custom import *
from feathermodules.pubkey import *

import cryptanalib as ca

# import
class ImportMultiFileCommand(Command):
   def run(self, line):
      ishellCompleter = readline.get_completer()
      readline.set_completer_delims(' \t\n;')
      readline.parse_and_bind("tab: complete")
      readline.set_completer(completer.complete)

      sample_file = raw_input('Please enter the filename you want to open: ')
      try:
         sample_fh = open(sample_file,'r')
         feathermodules.samples.extend([sample.strip() for sample in sample_fh.readlines()])
         sample_fh.close()
         feathermodules.samples = filter(lambda x: x != '' and x != None, feathermodules.samples)
      except:
         print 'Something went wrong. Sorry! Please try again.'
      finally:
         readline.set_completer(ishellCompleter)

class ImportSingleFileCommand(Command):
   def run(self, line):
      ishellCompleter = readline.get_completer()
      readline.set_completer_delims(' \t\n;')
      readline.parse_and_bind("tab: complete")
      readline.set_completer(completer.complete)

      sample_file = raw_input('Please enter the filename you want to open: ')
      try:
         sample_fh = open(sample_file,'r')
         feathermodules.samples.append(sample_fh.read())
         sample_fh.close()
         feathermodules.samples = filter(lambda x: x != '' and x != None, feathermodules.samples)
      except:
         print 'Something went wrong. Sorry! Please try again.'
      finally:
         readline.set_completer(ishellCompleter)

class ImportManualEntryCommand(Command):
   def run(self, line):
      feathermodules.samples.append(raw_input('Please enter your sample: ').strip())
      feathermodules.samples = filter(lambda x: x != '' and x != None, feathermodules.samples)

class ImportResultsCommand(Command):
   def run(self, line):
      if not feathermodules.results:
         print 'Last module failed to produce results.'
      elif feathermodules.results == True:
         print 'Last module succeeded, but did not return results.'
      else:
         print 'Last results (long values may be truncated):'
         print '-'*80
         for i in range(len(feathermodules.results)):
            print '{0:d}: {1:60s}'.format(i, repr(feathermodules.results[i]))

      selection = raw_input('\nPlease enter your selection by number, or \'all\' for all: ')
      try:
         if selection == 'all':
            feathermodules.samples.extend(feathermodules.results)
         elif int(selection) < len(feathermodules.results):
            feathermodules.samples.append(feathermodules.results[int(selection)])
         else:
            print 'Invalid entry, please try again.'
         feathermodules.samples = filter(lambda x: x != '' and x != None, feathermodules.samples)
      except ValueError:
         print 'Invalid entry, please try again.'

class ImportClearCommand(Command):
   def run(self, line):
      feathermodules.samples = []

class ImportCommand(Command):
   def args(self):
      return ['multifile', 'singlefile', 'manualentry', 'results', 'clear']

import_sample = ImportCommand('import', help='Import samples for analysis', dynamic_args=True)

import_multifile = ImportMultiFileCommand(
   'multifile',
   help='Import multiple newline-separated samples from one file',
   dynamic_args=True)
import_singlefile = ImportSingleFileCommand(
   'singlefile',
   help='Import a single file as a raw sample',
   dynamic_args=True)
import_manualentry = ImportManualEntryCommand(
   'manualentry',
   help='Manually enter a single sample',
   dynamic_args=True)
import_results = ImportResultsCommand(
   'results',
   help='Import last module\'s results as samples',
   dynamic_args=True)
import_clear = ImportClearCommand(
   'clear',
   help='Clear current sample set',
   dynamic_args=True)

import_sample.addChild(import_multifile)
import_sample.addChild(import_singlefile)
import_sample.addChild(import_manualentry)
import_sample.addChild(import_results)
import_sample.addChild(import_clear)

# console
class ConsoleCommand(Command):
   def run(self, line):
      ishellCompleter = readline.get_completer()
      embed()
      readline.set_completer(ishellCompleter)

console = ConsoleCommand('console', help='Opens an interactive prompt', dynamic_args=True)

# export to file
class ExportCommand(Command):
   def run(self, line):
      def _formatOutput(res):
         if isinstance(res, str):
            return res
         else:
             try:
                 return "\n".join(_formatOutput(r) for r in res)
             except TypeError:
                 return str(res)

      ishellCompleter = readline.get_completer()
      readline.set_completer_delims(' \t\n;')
      readline.parse_and_bind("tab: complete")
      readline.set_completer(completer.complete)

      filePath =  raw_input("Please specify a path to the output file: ").strip()

      readline.set_completer(ishellCompleter)
      if os.path.isfile(filePath):
         confirm = raw_input("File already exists and will be overwritten, confirm? [y/N] ")
         if confirm is "" or confirm[0] not in ("y", "Y"):
            print "Canceled."
            return

      with open(filePath, "w+") as handle:
        handle.write(_formatOutput(feathermodules.results))

export = ExportCommand('export', help='Export current results to file', dynamic_args=True)

# use
class UseCommand(Command):
   def args(self):
      return feathermodules.module_list.keys()
   def run(self, line):
      if line.split()[-1] not in feathermodules.module_list.keys():
         print 'Invalid module selection. Please try again.'
      else:
         feathermodules.selected_attack = feathermodules.module_list[ line.split()[-1] ]
         feathermodules.selected_attack_name = line.split()[-1]
         feathermodules.current_options = feathermodules.selected_attack['options']

use = UseCommand('use', help='Select the module to use', dynamic_args=True)

# analyze
class AnalyzeCommand(Command):
   def run(self, line):
      if len(feathermodules.samples) == 0:
         print 'No loaded samples. Please use the \'import\' command.'
         return False
      print '[+] Analyzing samples...'
      analysis_results = ca.analyze_ciphertext(feathermodules.samples, verbose=True)
      if analysis_results['decoded_ciphertexts'] != feathermodules.samples:
         feathermodules.samples = analysis_results['decoded_ciphertexts']
      print ''
      print '[+] Suggested modules:'
      for attack in feathermodules.module_list.keys():
         if len(set(feathermodules.module_list[attack]['keywords']) & set(analysis_results['keywords'])) > 0:
            print '   {0:<20} - {1:<57}'.format(attack, feathermodules.module_list[attack]['description'])

analyze = AnalyzeCommand('analyze', help='Analyze/decode samples', dynamic_args=True)

# autopwn
class AutopwnCommand(Command):
   def run(self, line):
      if len(feathermodules.samples) == 0:
         print 'No loaded samples. Please use the \'import\' command.'
         return False
      print '[+] Analyzing samples...'
      analysis_results = ca.analyze_ciphertext(feathermodules.samples, verbose=True)
      if analysis_results['decoded_ciphertexts'] != feathermodules.samples:
         feathermodules.samples = analysis_results['decoded_ciphertexts']
      for attack in feathermodules.module_list.keys():
         if len(set(feathermodules.module_list[attack]['keywords']) & set(analysis_results['keywords'])) > 0:
            print 'Running module: %s' % attack
            feathermodules.current_options = feathermodules.module_list[attack]['options']
            print feathermodules.module_list[attack]['attack_function'](feathermodules.samples)

autopwn = AutopwnCommand('autopwn', help='Analyze samples and run all attacks', dynamic_args=True)

# search
class SearchCommand(Command):
   def run(self, line):
      matching_modules = []
      search_param = line.split()[-1].lower()
      for attack in feathermodules.module_list.keys():
         if attack.lower().find(search_param) != -1:
            matching_modules.append(attack)
         elif feathermodules.module_list[attack]['description'].lower().find(search_param) != -1:
            matching_modules.append(attack)
         elif search_param in feathermodules.module_list[attack]['keywords']:
            matching_modules.append(attack)
      for module in matching_modules:
         print "%s - %s" % (module, feathermodules.module_list[module]['description'])

search = SearchCommand('search', help='Search module names and descriptions by keyword')

# samples
class SamplesCommand(Command):
   def run(self, line):
      print '-' * 60
      for sample in feathermodules.samples:
         print repr(sample)
      print '-' * 60

samples = SamplesCommand('samples', help='Show samples')

# modules
class ModulesCommand(Command):
   def run(self, line):
      for attack in feathermodules.module_list.keys():
         print "%s - %s" % (attack, feathermodules.module_list[attack]['description'])

modules = ModulesCommand('modules', help='Show all available modules')

# run
class RunCommand(Command):
   def run(self, line):
      if len(feathermodules.samples) == 0:
         print 'No loaded samples. Please use the \'import\' command.'
         return False
      elif feathermodules.selected_attack_name not in feathermodules.module_list.keys():
         print 'Invalid module selection. Please use the \'use\' command.'
         return False
      feathermodules.results = feathermodules.selected_attack['attack_function'](feathermodules.samples)

run = RunCommand('run', help='Run the currently selected module')

# options
class OptionsCommand(Command):
   def run(self, line):
      if feathermodules.selected_attack_name not in feathermodules.module_list.keys():
         print 'Please select a valid module first.'
         return False
      else:
         print ''
         print '{0:^60}'.format('Currently selected module: ' + feathermodules.selected_attack_name)
         print '-' * 60
         if len(feathermodules.selected_attack['options'].items()) == 0:
            print 'No options to configure.'
         else:
            for option, default in feathermodules.selected_attack['options'].items():
               try:
                  print '{0:<20}{1:>40}'.format(option, feathermodules.current_options[option])
               except:
                  print '{0:<20}{1:>40}'.format(option, default)

# set
class SetCommand(Command):
   def run(self, line):
      line_split = line.split()
      # set option_name value
      if not (len(line_split) == 2 and '=' in line_split[1]):
         print 'Usage: set <option>=<value>'
         return False
      option = line_split[1].split('=')[0]
      first_eq = line_split[1].find('=')
      value = line_split[1][first_eq+1:]
      feathermodules.current_options[option] = value
   def args(self):
      return feathermodules.selected_attack['options'].keys()

# unset
class UnsetCommand(Command):
   def run(self, line):
      line_split = line.split()
      # unset option_name
      if len(line_split) != 2:
         print 'Usage: unset <option>'
         return False
      option = line_split[1]
      feathermodules.current_options[option] = feathermodules.selected_attack['options'][option]
   def args(self):
      return feathermodules.selected_attack['options'].keys()

# results
class ResultsCommand(Command):
   def run(self, line):
      if not feathermodules.results:
         print 'Last module failed to produce results.'
      elif feathermodules.results == True:
         print 'Last module succeeded, but did not return results.'
      else:
         print 'Last results (long values may be truncated):'
         print '-'*80
         for i in range(len(feathermodules.results)):
            print '{0:d}: {1:60s}'.format(i, repr(feathermodules.results[i]))

set_command = SetCommand('set', help='Set an option (i.e., "set num_answers=3"', dynamic_args=True)
unset = UnsetCommand('unset', help='Revert an option to its default value', dynamic_args=True)
options = OptionsCommand('options', help='Show the current option values', dynamic_args=True)
results = ResultsCommand('results', help='Show the results from the last module run')

# Build the console
fd_console = Console(prompt='\nFeatherDuster', prompt_delim='>')

fd_console.addChild(import_sample)
fd_console.addChild(console)
fd_console.addChild(export)
fd_console.addChild(use)
fd_console.addChild(analyze)
fd_console.addChild(autopwn)
fd_console.addChild(search)
fd_console.addChild(samples)
fd_console.addChild(modules)
fd_console.addChild(run)
fd_console.addChild(options)
fd_console.addChild(set_command)
fd_console.addChild(unset)
fd_console.addChild(results)

#--------
# Main menu
#

for filename in sys.argv[1:]:
   if filename in ['-h', '--help']:
      print 'Usage: python featherduster.py [ciphertext file 1] ... [ciphertext file n]'
      exit()
   try:
      sample_fh = open(filename,'r')
      feathermodules.samples.append(sample_fh.read())
      sample_fh.close()
   except:
      continue

print """Welcome to FeatherDuster!

To get started, use 'import' to load samples.
Then, use 'analyze' to analyze/decode samples and get attack recommendations.
Next, run the 'use' command to select an attack module.
Finally, use 'run' to run the attack and see its output.

For a command reference, press Enter on a blank line.
"""

fd_console.loop()

print '\nThank you for using FeatherDuster!'
<commit_msg>Wrap module execution in try/except so that bugs in modules do not crash FeatherDuster entirely<commit_after>'''
FeatherDuster - A wizard-like interface to cryptanalib
Author: Daniel Crowley

FeatherDuster is a tool for brushing away magical crypto fairy dust.
'''

import sys
import os
import readline
import completer
from ishell.console import Console
from ishell.command import Command

try:
   from IPython import embed
except ImportError:
   import code

   def embed():
      vars = globals()
      vars.update(locals())
      shell = code.InteractiveConsole(vars)
      shell.interact()

import feathermodules

feathermodules.samples = []
feathermodules.results = False
feathermodules.selected_attack_name = ''
feathermodules.current_options = {}

from feathermodules.stream import *
from feathermodules.block import *
from feathermodules.classical import *
from feathermodules.auxiliary import *
from feathermodules.custom import *
from feathermodules.pubkey import *

import cryptanalib as ca

# import
class ImportMultiFileCommand(Command):
   def run(self, line):
      ishellCompleter = readline.get_completer()
      readline.set_completer_delims(' \t\n;')
      readline.parse_and_bind("tab: complete")
      readline.set_completer(completer.complete)

      sample_file = raw_input('Please enter the filename you want to open: ')
      try:
         sample_fh = open(sample_file,'r')
         feathermodules.samples.extend([sample.strip() for sample in sample_fh.readlines()])
         sample_fh.close()
         feathermodules.samples = filter(lambda x: x != '' and x != None, feathermodules.samples)
      except:
         print 'Something went wrong. Sorry! Please try again.'
      finally:
         readline.set_completer(ishellCompleter)

class ImportSingleFileCommand(Command):
   def run(self, line):
      ishellCompleter = readline.get_completer()
      readline.set_completer_delims(' \t\n;')
      readline.parse_and_bind("tab: complete")
      readline.set_completer(completer.complete)

      sample_file = raw_input('Please enter the filename you want to open: ')
      try:
         sample_fh = open(sample_file,'r')
         feathermodules.samples.append(sample_fh.read())
         sample_fh.close()
         feathermodules.samples = filter(lambda x: x != '' and x != None, feathermodules.samples)
      except:
         print 'Something went wrong. Sorry! Please try again.'
      finally:
         readline.set_completer(ishellCompleter)

class ImportManualEntryCommand(Command):
   def run(self, line):
      feathermodules.samples.append(raw_input('Please enter your sample: ').strip())
      feathermodules.samples = filter(lambda x: x != '' and x != None, feathermodules.samples)

class ImportResultsCommand(Command):
   def run(self, line):
      if not feathermodules.results:
         print 'Last module failed to produce results.'
      elif feathermodules.results == True:
         print 'Last module succeeded, but did not return results.'
      else:
         print 'Last results (long values may be truncated):'
         print '-'*80
         for i in range(len(feathermodules.results)):
            print '{0:d}: {1:60s}'.format(i, repr(feathermodules.results[i]))

      selection = raw_input('\nPlease enter your selection by number, or \'all\' for all: ')
      try:
         if selection == 'all':
            feathermodules.samples.extend(feathermodules.results)
         elif int(selection) < len(feathermodules.results):
            feathermodules.samples.append(feathermodules.results[int(selection)])
         else:
            print 'Invalid entry, please try again.'
         feathermodules.samples = filter(lambda x: x != '' and x != None, feathermodules.samples)
      except ValueError:
         print 'Invalid entry, please try again.'

class ImportClearCommand(Command):
   def run(self, line):
      feathermodules.samples = []

class ImportCommand(Command):
   def args(self):
      return ['multifile', 'singlefile', 'manualentry', 'results', 'clear']

import_sample = ImportCommand('import', help='Import samples for analysis', dynamic_args=True)

import_multifile = ImportMultiFileCommand(
   'multifile',
   help='Import multiple newline-separated samples from one file',
   dynamic_args=True)
import_singlefile = ImportSingleFileCommand(
   'singlefile',
   help='Import a single file as a raw sample',
   dynamic_args=True)
import_manualentry = ImportManualEntryCommand(
   'manualentry',
   help='Manually enter a single sample',
   dynamic_args=True)
import_results = ImportResultsCommand(
   'results',
   help='Import last module\'s results as samples',
   dynamic_args=True)
import_clear = ImportClearCommand(
   'clear',
   help='Clear current sample set',
   dynamic_args=True)

import_sample.addChild(import_multifile)
import_sample.addChild(import_singlefile)
import_sample.addChild(import_manualentry)
import_sample.addChild(import_results)
import_sample.addChild(import_clear)

# console
class ConsoleCommand(Command):
   def run(self, line):
      ishellCompleter = readline.get_completer()
      embed()
      readline.set_completer(ishellCompleter)

console = ConsoleCommand('console', help='Opens an interactive prompt', dynamic_args=True)

# export to file
class ExportCommand(Command):
   def run(self, line):
      def _formatOutput(res):
         if isinstance(res, str):
            return res
         else:
             try:
                 return "\n".join(_formatOutput(r) for r in res)
             except TypeError:
                 return str(res)

      ishellCompleter = readline.get_completer()
      readline.set_completer_delims(' \t\n;')
      readline.parse_and_bind("tab: complete")
      readline.set_completer(completer.complete)

      filePath =  raw_input("Please specify a path to the output file: ").strip()

      readline.set_completer(ishellCompleter)
      if os.path.isfile(filePath):
         confirm = raw_input("File already exists and will be overwritten, confirm? [y/N] ")
         if confirm is "" or confirm[0] not in ("y", "Y"):
            print "Canceled."
            return

      with open(filePath, "w+") as handle:
        handle.write(_formatOutput(feathermodules.results))

export = ExportCommand('export', help='Export current results to file', dynamic_args=True)

# use
class UseCommand(Command):
   def args(self):
      return feathermodules.module_list.keys()
   def run(self, line):
      if line.split()[-1] not in feathermodules.module_list.keys():
         print 'Invalid module selection. Please try again.'
      else:
         feathermodules.selected_attack = feathermodules.module_list[ line.split()[-1] ]
         feathermodules.selected_attack_name = line.split()[-1]
         feathermodules.current_options = feathermodules.selected_attack['options']

use = UseCommand('use', help='Select the module to use', dynamic_args=True)

# analyze
class AnalyzeCommand(Command):
   def run(self, line):
      if len(feathermodules.samples) == 0:
         print 'No loaded samples. Please use the \'import\' command.'
         return False
      print '[+] Analyzing samples...'
      analysis_results = ca.analyze_ciphertext(feathermodules.samples, verbose=True)
      if analysis_results['decoded_ciphertexts'] != feathermodules.samples:
         feathermodules.samples = analysis_results['decoded_ciphertexts']
      print ''
      print '[+] Suggested modules:'
      for attack in feathermodules.module_list.keys():
         if len(set(feathermodules.module_list[attack]['keywords']) & set(analysis_results['keywords'])) > 0:
            print '   {0:<20} - {1:<57}'.format(attack, feathermodules.module_list[attack]['description'])

analyze = AnalyzeCommand('analyze', help='Analyze/decode samples', dynamic_args=True)

# autopwn
class AutopwnCommand(Command):
   def run(self, line):
      if len(feathermodules.samples) == 0:
         print 'No loaded samples. Please use the \'import\' command.'
         return False
      print '[+] Analyzing samples...'
      analysis_results = ca.analyze_ciphertext(feathermodules.samples, verbose=True)
      if analysis_results['decoded_ciphertexts'] != feathermodules.samples:
         feathermodules.samples = analysis_results['decoded_ciphertexts']
      for attack in feathermodules.module_list.keys():
         if len(set(feathermodules.module_list[attack]['keywords']) & set(analysis_results['keywords'])) > 0:
            print 'Running module: %s' % attack
            feathermodules.current_options = feathermodules.module_list[attack]['options']
            try:
               print feathermodules.module_list[attack]['attack_function'](feathermodules.samples)
            except:
               print '[*] Module execution failed, please report this issue at https://github.com/nccgroup/featherduster/issues'

autopwn = AutopwnCommand('autopwn', help='Analyze samples and run all attacks', dynamic_args=True)

# search
class SearchCommand(Command):
   def run(self, line):
      matching_modules = []
      search_param = line.split()[-1].lower()
      for attack in feathermodules.module_list.keys():
         if attack.lower().find(search_param) != -1:
            matching_modules.append(attack)
         elif feathermodules.module_list[attack]['description'].lower().find(search_param) != -1:
            matching_modules.append(attack)
         elif search_param in feathermodules.module_list[attack]['keywords']:
            matching_modules.append(attack)
      for module in matching_modules:
         print "%s - %s" % (module, feathermodules.module_list[module]['description'])

search = SearchCommand('search', help='Search module names and descriptions by keyword')

# samples
class SamplesCommand(Command):
   def run(self, line):
      print '-' * 60
      for sample in feathermodules.samples:
         print repr(sample)
      print '-' * 60

samples = SamplesCommand('samples', help='Show samples')

# modules
class ModulesCommand(Command):
   def run(self, line):
      for attack in feathermodules.module_list.keys():
         print "%s - %s" % (attack, feathermodules.module_list[attack]['description'])

modules = ModulesCommand('modules', help='Show all available modules')

# run
class RunCommand(Command):
   def run(self, line):
      if len(feathermodules.samples) == 0:
         print 'No loaded samples. Please use the \'import\' command.'
         return False
      elif feathermodules.selected_attack_name not in feathermodules.module_list.keys():
         print 'Invalid module selection. Please use the \'use\' command.'
         return False
      try:
         feathermodules.results = feathermodules.selected_attack['attack_function'](feathermodules.samples)
      except:
         print '[*] Module execution failed, please report this issue at https://github.com/nccgroup/featherduster/issues'

run = RunCommand('run', help='Run the currently selected module')

# options
class OptionsCommand(Command):
   def run(self, line):
      if feathermodules.selected_attack_name not in feathermodules.module_list.keys():
         print 'Please select a valid module first.'
         return False
      else:
         print ''
         print '{0:^60}'.format('Currently selected module: ' + feathermodules.selected_attack_name)
         print '-' * 60
         if len(feathermodules.selected_attack['options'].items()) == 0:
            print 'No options to configure.'
         else:
            for option, default in feathermodules.selected_attack['options'].items():
               try:
                  print '{0:<20}{1:>40}'.format(option, feathermodules.current_options[option])
               except:
                  print '{0:<20}{1:>40}'.format(option, default)

# set
class SetCommand(Command):
   def run(self, line):
      line_split = line.split()
      # set option_name value
      if not (len(line_split) == 2 and '=' in line_split[1]):
         print 'Usage: set <option>=<value>'
         return False
      option = line_split[1].split('=')[0]
      first_eq = line_split[1].find('=')
      value = line_split[1][first_eq+1:]
      feathermodules.current_options[option] = value
   def args(self):
      return feathermodules.selected_attack['options'].keys()

# unset
class UnsetCommand(Command):
   def run(self, line):
      line_split = line.split()
      # unset option_name
      if len(line_split) != 2:
         print 'Usage: unset <option>'
         return False
      option = line_split[1]
      feathermodules.current_options[option] = feathermodules.selected_attack['options'][option]
   def args(self):
      return feathermodules.selected_attack['options'].keys()

# results
class ResultsCommand(Command):
   def run(self, line):
      if not feathermodules.results:
         print 'Last module failed to produce results.'
      elif feathermodules.results == True:
         print 'Last module succeeded, but did not return results.'
      else:
         print 'Last results (long values may be truncated):'
         print '-'*80
         for i in range(len(feathermodules.results)):
            print '{0:d}: {1:60s}'.format(i, repr(feathermodules.results[i]))

set_command = SetCommand('set', help='Set an option (i.e., "set num_answers=3"', dynamic_args=True)
unset = UnsetCommand('unset', help='Revert an option to its default value', dynamic_args=True)
options = OptionsCommand('options', help='Show the current option values', dynamic_args=True)
results = ResultsCommand('results', help='Show the results from the last module run')

# Build the console
fd_console = Console(prompt='\nFeatherDuster', prompt_delim='>')

fd_console.addChild(import_sample)
fd_console.addChild(console)
fd_console.addChild(export)
fd_console.addChild(use)
fd_console.addChild(analyze)
fd_console.addChild(autopwn)
fd_console.addChild(search)
fd_console.addChild(samples)
fd_console.addChild(modules)
fd_console.addChild(run)
fd_console.addChild(options)
fd_console.addChild(set_command)
fd_console.addChild(unset)
fd_console.addChild(results)

#--------
# Main menu
#

for filename in sys.argv[1:]:
   if filename in ['-h', '--help']:
      print 'Usage: python featherduster.py [ciphertext file 1] ... [ciphertext file n]'
      exit()
   try:
      sample_fh = open(filename,'r')
      feathermodules.samples.append(sample_fh.read())
      sample_fh.close()
   except:
      continue

print """Welcome to FeatherDuster!

To get started, use 'import' to load samples.
Then, use 'analyze' to analyze/decode samples and get attack recommendations.
Next, run the 'use' command to select an attack module.
Finally, use 'run' to run the attack and see its output.

For a command reference, press Enter on a blank line.
"""

fd_console.loop()

print '\nThank you for using FeatherDuster!'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from distutils import version

__version__ = '1.2.2'
version_info = version.StrictVersion(__version__).version

default_app_config = 'debreach.apps.DebreachConfig'
<commit_msg>Bump version number<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from distutils import version

__version__ = '1.3.0'
version_info = version.StrictVersion(__version__).version

default_app_config = 'debreach.apps.DebreachConfig'
<|endoftext|>


<commit_before>__author__ = 'lqrz'

import cPickle as pickle
import gensim
import logging
import pdb
from nltk.corpus import PlaintextCorpusReader
import sys
import multiprocessing as mp
import codecs
from annoy import AnnoyIndex
from sklearn.metrics.pairwise import cosine_similarity

# logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger('')
hdlr = logging.FileHandler('decompound2.log')
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.DEBUG)

def decompound((inputCompound, nAccuracy)):

    global annoy_tree
    global vectors
    global pickledIndexes
    global pickledVectors

    if len(inputCompound) == 0:
        return []

    # get all matching prefixes
    logger.info('Getting all matching prefixes')
    prefixes = set()
    for prefix in vectors.keys():
        found = inputCompound.find(prefix)
        if found == 0 and len(vectors[prefix]) > 0 and len(inputCompound[len(prefix):])>0:
            prefixes.add(prefix)

    logger.debug('Possible prefixes')
    logger.debug(prefixes)

    # get all possible splits
    logger.info('Getting possible splits')

    splits = set()
    splitsWithNoRep = set()

    for prefix in prefixes:
        fugenlaute = ['', 'e', 'es']
        for fug in fugenlaute:
            if fug == '' or inputCompound[len(prefix):].find(fug) == 0:
                if not debug:
                    try:
                        # look for the uppercased rest representation
                        tail = inputCompound[len(prefix) + len(fug):].title()
                        logger.debug('Tail: '+tail)
                        tailRepresentationIndex = pickledIndexes[tail]
                        logger.debug('Tail index: '+str(tailRepresentationIndex))
                        splits.add((prefix, tail, tailRepresentationIndex))
                        msg = ' '.join(['Considering split', inputCompound, prefix, tail])
                        logger.debug(msg)
                    except KeyError:
                        # if i dont have a vector rep for the rest, i discard it
                        splitsWithNoRep.add((prefix, tail))
                        msg = ' '.join(['Discarding split', inputCompound, prefix, tail])
                        logger.debug(msg)
                    try:
                        # look for the lowercased rest representation
                        tail = inputCompound[len(prefix) + len(fug):]
                        logger.debug('Tail: '+tail)
                        tailRepresentationIndex = pickledIndexes[tail]
                        logger.debug('Tail index: '+str(tailRepresentationIndex))
                        splits.add((prefix, tail, tailRepresentationIndex))
                        msg = ' '.join(['Considering split', inputCompound, prefix, tail])
                        logger.debug(msg)
                    except KeyError:
                        # if i dont have a vector rep for the rest, i discard it
                        msg = ' '.join(['Discarding split', inputCompound, prefix, tail])
                        logger.debug(msg)
                        splitsWithNoRep.add((prefix, tail))
                        continue

    if len(splits) == 0:
        logger.error('Cannot decompound '+inputCompound)
        # exit()
        return [(inputCompound, '', '')]

    # apply direction vectors to splits
    logger.info('Applying direction vectors to possible splits')
    representations = set()
    bestSimilarity = 0.46 # so we do not split "Bahnhof" = ["Bahn", "Hof"]
    best = None
    maxEvidence = 0
    bestEvidence = None
    for prefix, tail, tailRepresentationIndex in splits:
        msg = ' '.join(['Applying', str(len(vectors[prefix])), 'direction vectors to split', prefix, tail])
        logger.debug(msg)
        for origin, evidence in vectors[prefix]:
            logger.debug('Prefix '+prefix+' by indexes '+str(origin[0])+' and '+str(origin[1]))
            try:
                dirVectorCompoundRepresentation = pickledVectors[origin[0]]
                logger.debug('Found key in vector dict for index '+str(origin[0]))
            except KeyError:
                logger.debug('ERROR COULDNT FIND KEY '+str(origin[0])+' IN VECTOR DICT')
                continue

            try:
                dirVectorTailRepresentation = pickledVectors[origin[1]]
                logger.debug('Found key in vector dict for index '+str(origin[1]))
            except KeyError:
                logger.debug('ERROR COULDNT FIND KEY '+str(origin[1])+' IN VECTOR DICT')
                continue

            dirVectorDifference = dirVectorCompoundRepresentation - dirVectorTailRepresentation

            try:
                logger.debug('Looking up tail index '+str(tailRepresentationIndex))
                predictionRepresentation = pickledVectors[tailRepresentationIndex] + dirVectorDifference
                logger.debug('Found key in vector dict for index '+str(tailRepresentationIndex))
            except KeyError:
                logger.debug('ERROR COULDNT FIND KEY '+str(tailRepresentationIndex)+' IN VECTOR DICT')
                continue

            # accuracy
            # neighbours = sorted(model.most_similar(positive=[predictionRepresentation], negative=[], topn=nAccuracy), \
            #                     key=lambda x: x[1], reverse=True)

            logger.debug('Getting Annoy KNN')
            try:
                neighbours = annoy_tree.get_nns_by_vector(list(predictionRepresentation), nAccuracy)
                logger.debug(neighbours)
            except:
                logger.error('Problem found when retrieving KNN for prediction representation')
                logger.error(list(predictionRepresentation))
                exit()

            try:
                logger.debug('Looking up word '+inputCompound+' in index dict')
                inputCompoundIndex = pickledIndexes[inputCompound]
                logger.debug('Found key in index dict for word '+inputCompound)
            except KeyError:
                logger.debug('ERROR COULDNT FIND KEY '+inputCompound+' IN INDEX VECTOR')
                continue

            try:
                logger.debug('Looking up index '+str(inputCompoundIndex))
                inputCompoundRep = pickledVectors[inputCompoundIndex]
                logger.debug('Found key in vector dict for index '+str(inputCompoundIndex))
            except KeyError:
                logger.debug('ERROR COULDNT FIND KEY '+str(inputCompoundIndex)+' IN VECTOR DICT')
                continue

            try:
                rank = [i for i, nei in enumerate(neighbours) if nei == inputCompoundIndex][0]
                logger.debug(str(inputCompoundIndex)+' found in neighbours. Rank: '+str(rank))
                similarity = cosine_similarity(predictionRepresentation, inputCompoundRep)[0][0]
                logger.debug('Computed cosine similarity: '+str(similarity))
                res = (prefix, tail, origin[0], origin[1], rank, similarity)
                representations.add(res)
                if similarity > bestSimilarity: # compare cosine similarity
                    logger.debug('Found new best similarity score. Old: '+str(bestSimilarity)+' New: '+str(similarity))
                    bestSimilarity = similarity
                    best = res
            except IndexError:
                logger.debug(str(inputCompoundIndex)+' not found in neighbours. NO RANK. WONT SPLIT')
                continue
            # except IndexError:
            #     splitsWithNoRep.add((prefix, tail))
            #     res = (prefix, tail, origin[0], origin[1])
            #     if len(evidence) > maxEvidence:
            #         maxEvidence = len(evidence)
            #         bestEvidence = res
            #
            #     continue

    logger.debug('Choosing best direction vector')
    chosenSplit = None

    if best:
        chosenSplit = best
        msg = ' '.join(['Splitting',inputCompound,'as', chosenSplit[0], chosenSplit[1], str(chosenSplit[2]), \
                        str(chosenSplit[3]), 'rank', str(chosenSplit[4]), 'similarity', str(chosenSplit[5])])
        logger.debug(msg)
        logger.debug('Decompounding '+chosenSplit[1])
    else:
        # nobody got the original representation within the KNN
        # chosenSplit = bestEvidence
        chosenSplit = (inputCompound, '') # not split at all
        logger.debug('Not splitting compound '+inputCompound)

    # logging.debug('Found prefix '+chosenSplit[0])
    # logging.debug('Decompounding '+chosenSplit[1])

    # return [(inputCompound, chosenSplit[0], chosenSplit[1])] + decompound((chosenSplit[1], nAccuracy))
    return [(inputCompound, chosenSplit[0], chosenSplit[1])] # do not apply recursion

if __name__ == '__main__':
    # resultsPath = 'results/dir_vecs_4_100.p'
    # annoyTreeFile = 'tree.ann'
    # pickledIndexes = pickle.load(open('decompoundIndexes.p','rb'))
    # pickledVectors = pickle.load(open('decompoundVectors.p','rb'))
    # corpusPath = './prueba.txt'
    # outPath = 'splits.txt'

    # multiprocessed = True
    # nWorkers = 4

    if len(sys.argv) == 9:
        resultsPath = sys.argv[1]
        # w2vPath = sys.argv[2]
        annoyTreeFile = sys.argv[2]
        corpusPath = sys.argv[3]
        pickledIndexesPath = sys.argv[4]
        pickledVectorsPath = sys.argv[5]
        multiprocessed = sys.argv[6]
        nWorkers = sys.argv[7]
        outPath = sys.argv[8]

    elif len(sys.argv)>1:
        print 'Error in params'
        exit()

    idx = corpusPath.rfind('/') + 1
    folder = corpusPath[0:idx]
    filename = corpusPath[idx:]

    logger.debug('Corpus folder: '+folder)
    logger.debug('Corpus filename: '+filename)

    corpus = PlaintextCorpusReader(folder, filename, encoding='utf-8')
    inputCompounds = corpus.words()

    logger.debug('Words in corpus')
    logger.debug(inputCompounds)

    #TODO: define threshold
    nAccuracy= 100

    debug = False

    logger.info('Getting pickled direction vectors file')
    vectors = pickle.load(open(resultsPath, 'rb'))

    logger.info('Getting pickled indexes')
    pickledIndexes = pickle.load(open(pickledIndexesPath,'rb'))
    pickledVectors = pickle.load(open(pickledVectorsPath,'rb'))

    logger.info('Getting annoy tree')
    # model = gensim.models.Word2Vec.load_word2vec_format(w2vPath, binary=True)
    annoy_tree = AnnoyIndex(500)
    annoy_tree.load(annoyTreeFile)

    if multiprocessed:
        logger.info('Instantiating pool with '+str(nWorkers))
        pool = mp.Pool(processes=int(nWorkers))
        results = pool.map(decompound, zip(inputCompounds, [nAccuracy]*len(inputCompounds)))
    else:
        results = []
        for inputCompound in inputCompounds:
            try:
                inputCompoundIndex = pickledIndexes[inputCompound]
                compoundRepresentation = pickledVectors[inputCompoundIndex]
                results.append(decompound((inputCompound, nAccuracy)))
            except KeyError:
                logger.error('No word2vec representation for input compound'+inputCompound)
                # exit()
                results.append(inputCompound)

    print results

    fout = codecs.open(outPath, 'w', encoding='utf-8')

    for i, split in enumerate(results):
        for comp, decomp1, decomp2 in split:
            fout.write(comp + '\t' + decomp1 + '\t' + decomp2 + '\n')

    fout.close()

    logger.info('End')<commit_msg>whitespace tokenizer<commit_after>__author__ = 'lqrz'

import cPickle as pickle
import gensim
import logging
import pdb
from nltk.corpus import PlaintextCorpusReader
from nltk.tokenize import WhitespaceTokenizer
import sys
import multiprocessing as mp
import codecs
from annoy import AnnoyIndex
from sklearn.metrics.pairwise import cosine_similarity

# logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger('')
hdlr = logging.FileHandler('decompound2.log')
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.DEBUG)

def decompound((inputCompound, nAccuracy)):

    global annoy_tree
    global vectors
    global pickledIndexes
    global pickledVectors

    if len(inputCompound) == 0:
        return []

    # get all matching prefixes
    logger.info('Getting all matching prefixes')
    prefixes = set()
    for prefix in vectors.keys():
        found = inputCompound.find(prefix)
        if found == 0 and len(vectors[prefix]) > 0 and len(inputCompound[len(prefix):])>0:
            prefixes.add(prefix)

    logger.debug('Possible prefixes')
    logger.debug(prefixes)

    # get all possible splits
    logger.info('Getting possible splits')

    splits = set()
    splitsWithNoRep = set()

    for prefix in prefixes:
        fugenlaute = ['', 'e', 'es']
        for fug in fugenlaute:
            if fug == '' or inputCompound[len(prefix):].find(fug) == 0:
                if not debug:
                    try:
                        # look for the uppercased rest representation
                        tail = inputCompound[len(prefix) + len(fug):].title()
                        logger.debug('Tail: '+tail)
                        tailRepresentationIndex = pickledIndexes[tail]
                        logger.debug('Tail index: '+str(tailRepresentationIndex))
                        splits.add((prefix, tail, tailRepresentationIndex))
                        msg = ' '.join(['Considering split', inputCompound, prefix, tail])
                        logger.debug(msg)
                    except KeyError:
                        # if i dont have a vector rep for the rest, i discard it
                        splitsWithNoRep.add((prefix, tail))
                        msg = ' '.join(['Discarding split', inputCompound, prefix, tail])
                        logger.debug(msg)
                    try:
                        # look for the lowercased rest representation
                        tail = inputCompound[len(prefix) + len(fug):]
                        logger.debug('Tail: '+tail)
                        tailRepresentationIndex = pickledIndexes[tail]
                        logger.debug('Tail index: '+str(tailRepresentationIndex))
                        splits.add((prefix, tail, tailRepresentationIndex))
                        msg = ' '.join(['Considering split', inputCompound, prefix, tail])
                        logger.debug(msg)
                    except KeyError:
                        # if i dont have a vector rep for the rest, i discard it
                        msg = ' '.join(['Discarding split', inputCompound, prefix, tail])
                        logger.debug(msg)
                        splitsWithNoRep.add((prefix, tail))
                        continue

    if len(splits) == 0:
        logger.error('Cannot decompound '+inputCompound)
        # exit()
        return [(inputCompound, '', '')]

    # apply direction vectors to splits
    logger.info('Applying direction vectors to possible splits')
    representations = set()
    bestSimilarity = 0.46 # so we do not split "Bahnhof" = ["Bahn", "Hof"]
    best = None
    maxEvidence = 0
    bestEvidence = None
    for prefix, tail, tailRepresentationIndex in splits:
        msg = ' '.join(['Applying', str(len(vectors[prefix])), 'direction vectors to split', prefix, tail])
        logger.debug(msg)
        for origin, evidence in vectors[prefix]:
            logger.debug('Prefix '+prefix+' by indexes '+str(origin[0])+' and '+str(origin[1]))
            try:
                dirVectorCompoundRepresentation = pickledVectors[origin[0]]
                logger.debug('Found key in vector dict for index '+str(origin[0]))
            except KeyError:
                logger.debug('ERROR COULDNT FIND KEY '+str(origin[0])+' IN VECTOR DICT')
                continue

            try:
                dirVectorTailRepresentation = pickledVectors[origin[1]]
                logger.debug('Found key in vector dict for index '+str(origin[1]))
            except KeyError:
                logger.debug('ERROR COULDNT FIND KEY '+str(origin[1])+' IN VECTOR DICT')
                continue

            dirVectorDifference = dirVectorCompoundRepresentation - dirVectorTailRepresentation

            try:
                logger.debug('Looking up tail index '+str(tailRepresentationIndex))
                predictionRepresentation = pickledVectors[tailRepresentationIndex] + dirVectorDifference
                logger.debug('Found key in vector dict for index '+str(tailRepresentationIndex))
            except KeyError:
                logger.debug('ERROR COULDNT FIND KEY '+str(tailRepresentationIndex)+' IN VECTOR DICT')
                continue

            # accuracy
            # neighbours = sorted(model.most_similar(positive=[predictionRepresentation], negative=[], topn=nAccuracy), \
            #                     key=lambda x: x[1], reverse=True)

            logger.debug('Getting Annoy KNN')
            try:
                neighbours = annoy_tree.get_nns_by_vector(list(predictionRepresentation), nAccuracy)
                logger.debug(neighbours)
            except:
                logger.error('Problem found when retrieving KNN for prediction representation')
                logger.error(list(predictionRepresentation))
                exit()

            try:
                logger.debug('Looking up word '+inputCompound+' in index dict')
                inputCompoundIndex = pickledIndexes[inputCompound]
                logger.debug('Found key in index dict for word '+inputCompound)
            except KeyError:
                logger.debug('ERROR COULDNT FIND KEY '+inputCompound+' IN INDEX VECTOR')
                continue

            try:
                logger.debug('Looking up index '+str(inputCompoundIndex))
                inputCompoundRep = pickledVectors[inputCompoundIndex]
                logger.debug('Found key in vector dict for index '+str(inputCompoundIndex))
            except KeyError:
                logger.debug('ERROR COULDNT FIND KEY '+str(inputCompoundIndex)+' IN VECTOR DICT')
                continue

            try:
                rank = [i for i, nei in enumerate(neighbours) if nei == inputCompoundIndex][0]
                logger.debug(str(inputCompoundIndex)+' found in neighbours. Rank: '+str(rank))
                similarity = cosine_similarity(predictionRepresentation, inputCompoundRep)[0][0]
                logger.debug('Computed cosine similarity: '+str(similarity))
                res = (prefix, tail, origin[0], origin[1], rank, similarity)
                representations.add(res)
                if similarity > bestSimilarity: # compare cosine similarity
                    logger.debug('Found new best similarity score. Old: '+str(bestSimilarity)+' New: '+str(similarity))
                    bestSimilarity = similarity
                    best = res
            except IndexError:
                logger.debug(str(inputCompoundIndex)+' not found in neighbours. NO RANK. WONT SPLIT')
                continue
            # except IndexError:
            #     splitsWithNoRep.add((prefix, tail))
            #     res = (prefix, tail, origin[0], origin[1])
            #     if len(evidence) > maxEvidence:
            #         maxEvidence = len(evidence)
            #         bestEvidence = res
            #
            #     continue

    logger.debug('Choosing best direction vector')
    chosenSplit = None

    if best:
        chosenSplit = best
        msg = ' '.join(['Splitting',inputCompound,'as', chosenSplit[0], chosenSplit[1], str(chosenSplit[2]), \
                        str(chosenSplit[3]), 'rank', str(chosenSplit[4]), 'similarity', str(chosenSplit[5])])
        logger.debug(msg)
        logger.debug('Decompounding '+chosenSplit[1])
    else:
        # nobody got the original representation within the KNN
        # chosenSplit = bestEvidence
        chosenSplit = (inputCompound, '') # not split at all
        logger.debug('Not splitting compound '+inputCompound)

    # logging.debug('Found prefix '+chosenSplit[0])
    # logging.debug('Decompounding '+chosenSplit[1])

    # return [(inputCompound, chosenSplit[0], chosenSplit[1])] + decompound((chosenSplit[1], nAccuracy))
    return [(inputCompound, chosenSplit[0], chosenSplit[1])] # do not apply recursion

if __name__ == '__main__':
    # resultsPath = 'results/dir_vecs_4_100.p'
    # annoyTreeFile = 'tree.ann'
    # pickledIndexes = pickle.load(open('decompoundIndexes.p','rb'))
    # pickledVectors = pickle.load(open('decompoundVectors.p','rb'))
    # corpusPath = './prueba.txt'
    # outPath = 'splits.txt'

    # multiprocessed = True
    # nWorkers = 4

    if len(sys.argv) == 9:
        resultsPath = sys.argv[1]
        # w2vPath = sys.argv[2]
        annoyTreeFile = sys.argv[2]
        corpusPath = sys.argv[3]
        pickledIndexesPath = sys.argv[4]
        pickledVectorsPath = sys.argv[5]
        multiprocessed = sys.argv[6]
        nWorkers = sys.argv[7]
        outPath = sys.argv[8]

    elif len(sys.argv)>1:
        print 'Error in params'
        exit()

    idx = corpusPath.rfind('/') + 1
    folder = corpusPath[0:idx]
    filename = corpusPath[idx:]

    logger.debug('Corpus folder: '+folder)
    logger.debug('Corpus filename: '+filename)

    corpus = PlaintextCorpusReader(folder, filename, word_tokenizer=WhitespaceTokenizer(), encoding='utf-8')
    inputCompounds = corpus.words()

    logger.debug('Words in corpus')
    logger.debug(inputCompounds)

    #TODO: define threshold
    nAccuracy= 100

    debug = False

    logger.info('Getting pickled direction vectors file')
    vectors = pickle.load(open(resultsPath, 'rb'))

    logger.info('Getting pickled indexes')
    pickledIndexes = pickle.load(open(pickledIndexesPath,'rb'))
    pickledVectors = pickle.load(open(pickledVectorsPath,'rb'))

    logger.info('Getting annoy tree')
    # model = gensim.models.Word2Vec.load_word2vec_format(w2vPath, binary=True)
    annoy_tree = AnnoyIndex(500)
    annoy_tree.load(annoyTreeFile)

    if multiprocessed:
        logger.info('Instantiating pool with '+str(nWorkers))
        pool = mp.Pool(processes=int(nWorkers))
        results = pool.map(decompound, zip(inputCompounds, [nAccuracy]*len(inputCompounds)))
    else:
        results = []
        for inputCompound in inputCompounds:
            try:
                inputCompoundIndex = pickledIndexes[inputCompound]
                compoundRepresentation = pickledVectors[inputCompoundIndex]
                results.append(decompound((inputCompound, nAccuracy)))
            except KeyError:
                logger.error('No word2vec representation for input compound'+inputCompound)
                # exit()
                results.append(inputCompound)

    print results

    fout = codecs.open(outPath, 'w', encoding='utf-8')

    for i, split in enumerate(results):
        for comp, decomp1, decomp2 in split:
            fout.write(comp + '\t' + decomp1 + '\t' + decomp2 + '\n')

    fout.close()

    logger.info('End')<|endoftext|>


<commit_before>"""This script launches all of the processes necessary to train a
deep Q-network on an ALE game.

Usage:

ale_run.py [--glue_port GLUE_PORT]

All unrecognized command line arguments will be passed on to
rl_glue_ale_agent.py
"""
import subprocess
import sys
import os
import argparse

ROM_PATH = "/home/spragunr/neural_rl_libraries/roms/breakout.bin"

# Check for glue_port command line argument and set it up...
parser = argparse.ArgumentParser(description='Neural rl agent.')
parser.add_argument('--glue_port', type=str, default="4096",
                    help='rlglue port (default 4096)')
args, unknown = parser.parse_known_args()
my_env = os.environ.copy()
my_env["RLGLUE_PORT"] = args.glue_port

# Start the necessary processes:
p1 = subprocess.Popen(['rl_glue'], env=my_env)

ale_string = ("ale -game_controller rlglue -frame_skip 4 "
              "-restricted_action_set true ")
p2 = subprocess.Popen(ale_string + ROM_PATH, shell=True, env=my_env)
p3 = subprocess.Popen(['./rl_glue_ale_experiment.py'], env=my_env)
p4 = subprocess.Popen(['./rl_glue_ale_agent.py'] + sys.argv[1:], env=my_env)

p1.wait()
p2.wait()
p3.wait()
p4.wait()
<commit_msg>Make the position of the roms work for everybody.<commit_after>"""This script launches all of the processes necessary to train a
deep Q-network on an ALE game.

Usage:

ale_run.py [--glue_port GLUE_PORT]

All unrecognized command line arguments will be passed on to
rl_glue_ale_agent.py
"""
import subprocess
import sys
import os
import argparse

# Put your binaries under the directory 'deep_q_rl/roms'
ROM_PATH = "../roms/breakout.bin"

# Check for glue_port command line argument and set it up...
parser = argparse.ArgumentParser(description='Neural rl agent.')
parser.add_argument('--glue_port', type=str, default="4096",
                    help='rlglue port (default 4096)')
args, unknown = parser.parse_known_args()
my_env = os.environ.copy()
my_env["RLGLUE_PORT"] = args.glue_port

# Start the necessary processes:
p1 = subprocess.Popen(['rl_glue'], env=my_env)

ale_string = ("ale -game_controller rlglue -frame_skip 4 "
              "-restricted_action_set true ")
p2 = subprocess.Popen(ale_string + ROM_PATH, shell=True, env=my_env)
p3 = subprocess.Popen(['./rl_glue_ale_experiment.py'], env=my_env)
p4 = subprocess.Popen(['./rl_glue_ale_agent.py'] + sys.argv[1:], env=my_env)

p1.wait()
p2.wait()
p3.wait()
p4.wait()
<|endoftext|>


<commit_before>
import dbus.glib
import gobject
import sys

from account import read_account, connect

from telepathy.client.channel import Channel
from telepathy.constants import (
    CONNECTION_HANDLE_TYPE_NONE, CONNECTION_HANDLE_TYPE_CONTACT,
    CONNECTION_STATUS_CONNECTED, CONNECTION_STATUS_DISCONNECTED,
    MEDIA_STREAM_TYPE_AUDIO, MEDIA_STREAM_TYPE_VIDEO)
from telepathy.interfaces import (
    CHANNEL_INTERFACE, CHANNEL_INTERFACE_GROUP, CHANNEL_TYPE_STREAMED_MEDIA,
    CONN_INTERFACE)

def get_stream_engine():
    bus = dbus.Bus()
    return bus.get_object(
        'org.freedesktop.Telepathy.StreamEngine',
        '/org/freedesktop/Telepathy/StreamEngine')

class Call:
    def __init__(self, conn):
        self.conn = conn
        self.channel = None

        conn[CONN_INTERFACE].connect_to_signal('StatusChanged',
            self.status_changed_cb)
        conn[CONN_INTERFACE].connect_to_signal('NewChannel',
            self.new_channel_cb)

    def run(self):
        self.loop = gobject.MainLoop()

        try:
            self.loop.run()
        except KeyboardInterrupt:
            print "killed"

            if self.channel:
                print "closing channel"
                self.channel[CHANNEL_INTERFACE].Close()

    def quit(self):
        if self.loop:
            self.loop.quit()
            self.loop = None

    def status_changed_cb(self, state, reason):
        if state == CONNECTION_STATUS_DISCONNECTED:
            print 'connection closed'
            self.quit()

    def request_channel_error_cb(self, exception):
        print 'error:', exception
        self.quit()

    def new_channel_cb(self, object_path, channel_type, handle_type, handle,
            suppress_handler):
        if channel_type != CHANNEL_TYPE_STREAMED_MEDIA:
            return

        self.chan_handle_type = handle_type
        self.chan_handle = handle

        print "new streamed media channel"
        Channel(self.conn._dbus_object._named_service, object_path,
                ready_handler=self.channel_ready_cb)

    def channel_ready_cb(self, channel):
        print "channel ready"
        channel[CHANNEL_INTERFACE].connect_to_signal('Closed', self.closed_cb)
        channel[CHANNEL_INTERFACE_GROUP].connect_to_signal('MembersChanged',
            self.members_changed_cb)

        stream_engine = get_stream_engine()
        handler = dbus.Interface(stream_engine,
            'org.freedesktop.Telepathy.ChannelHandler')
        handler.HandleChannel(
            self.conn._dbus_object._named_service,
            self.conn._dbus_object._object_path,
            CHANNEL_TYPE_STREAMED_MEDIA,
            channel._dbus_object._object_path,
            self.chan_handle_type,
            self.chan_handle)

        self.channel = channel

    def closed_cb(self):
        print "channel closed"
        self.quit()

    def do_window(self):
        print "trying to open window"

        import gtk

        window = gtk.Window()
        window.set_border_width (5)
        hbox = gtk.HBox(False, 5)
        label1 = gtk.Label('Output')
        label2 = gtk.Label('Preview')
        vbox1 = gtk.VBox(False, 5)
        vbox2 = gtk.VBox(False, 5)
        socket1 = gtk.Socket()
        socket1.set_size_request(320, 240)
        socket2 = gtk.Socket()
        socket2.set_size_request(320, 240)

        vbox1.add(label1)
        vbox1.add(socket1)
        vbox2.add(label2)
        vbox2.add(socket2)
        hbox.add(vbox1)
        hbox.add(vbox2)
        window.add(hbox)
        window.show_all()

        stream_engine = get_stream_engine()
        badger = dbus.Interface(stream_engine,
            'org.freedesktop.Telepathy.StreamEngine')
        chan_path = self.channel._dbus_object._object_path
        badger.SetOutputWindow(chan_path, 2, socket1.get_id())
        badger.AddPreviewWindow(socket2.get_id())
        return False

    def members_changed_cb(self, message, added, removed, local_pending,
            remote_pending, actor, reason):
        print 'MembersChanged', (
            added, removed, local_pending, remote_pending, actor, reason)

        if added:
            gobject.timeout_add(5000, self.do_window)

class OutgoingCall(Call):
    def __init__(self, conn, contact):
        Call.__init__(self, conn)
        self.contact = contact

    def status_changed_cb (self, state, reason):
        if state == CONNECTION_STATUS_CONNECTED:
            handle = self.conn[CONN_INTERFACE].RequestHandles(
                CONNECTION_HANDLE_TYPE_CONTACT, [self.contact])[0]
            self.handle = handle

            print 'got handle %d for %s' % (handle, self.contact)

            # hack
            import time
            time.sleep(5)

            self.conn[CONN_INTERFACE].RequestChannel(
                CHANNEL_TYPE_STREAMED_MEDIA, CONNECTION_HANDLE_TYPE_NONE,
                handle, True,
                reply_handler=lambda *stuff: None,
                error_handler=self.request_channel_error_cb)

        Call.status_changed_cb(self, state, reason)

    def channel_ready_cb(self, channel):
        Call.channel_ready_cb(self, channel)

        channel[CHANNEL_INTERFACE_GROUP].AddMembers([self.handle], "")

        print "requesting audio/video streams"

        try:
            channel[CHANNEL_TYPE_STREAMED_MEDIA].RequestStreams(
                self.handle,
                [MEDIA_STREAM_TYPE_AUDIO, MEDIA_STREAM_TYPE_VIDEO]);
        except dbus.DBusException, e:
            print "failed:", e
            print "requesting audio stream"

            try:
                channel[CHANNEL_TYPE_STREAMED_MEDIA].RequestStreams(
                    self.handle, [MEDIA_STREAM_TYPE_AUDIO]);
            except dbus.DBusException, e:
                print "failed:", e
                print "giving up"
                self.quit()

class IncomingCall(Call):
    def status_changed_cb (self, state, reason):
        if state == CONNECTION_STATUS_CONNECTED:
            print "waiting for incoming call"

        Call.status_changed_cb(self, state, reason)

    def channel_ready_cb(self, channel):
        Call.channel_ready_cb(self, channel)

        print "accepting incoming call"
        pending = channel[CHANNEL_INTERFACE_GROUP].GetLocalPendingMembers()
        channel[CHANNEL_INTERFACE_GROUP].AddMembers(pending, "")

if __name__ == '__main__':
    assert len(sys.argv) in (2, 3)
    account_file = sys.argv[1]

    manager, protocol, account = read_account(account_file)
    conn = connect(manager, protocol, account)

    if len(sys.argv) > 2:
        contact = sys.argv[2]
        call = OutgoingCall(conn, sys.argv[2])
    else:
        call = IncomingCall(conn)

    print "connecting"
    conn[CONN_INTERFACE].Connect()
    call.run()

    try:
        print "disconnecting"
        conn[CONN_INTERFACE].Disconnect()
    except dbus.DBusException:
        pass

<commit_msg>examples/call.py: remove gtk code<commit_after>
import dbus.glib
import gobject
import sys

from account import read_account, connect

from telepathy.client.channel import Channel
from telepathy.constants import (
    CONNECTION_HANDLE_TYPE_NONE, CONNECTION_HANDLE_TYPE_CONTACT,
    CONNECTION_STATUS_CONNECTED, CONNECTION_STATUS_DISCONNECTED,
    MEDIA_STREAM_TYPE_AUDIO, MEDIA_STREAM_TYPE_VIDEO)
from telepathy.interfaces import (
    CHANNEL_INTERFACE, CHANNEL_INTERFACE_GROUP, CHANNEL_TYPE_STREAMED_MEDIA,
    CONN_INTERFACE)

def get_stream_engine():
    bus = dbus.Bus()
    return bus.get_object(
        'org.freedesktop.Telepathy.StreamEngine',
        '/org/freedesktop/Telepathy/StreamEngine')

class Call:
    def __init__(self, conn):
        self.conn = conn
        self.channel = None

        conn[CONN_INTERFACE].connect_to_signal('StatusChanged',
            self.status_changed_cb)
        conn[CONN_INTERFACE].connect_to_signal('NewChannel',
            self.new_channel_cb)

    def run(self):
        self.loop = gobject.MainLoop()

        try:
            self.loop.run()
        except KeyboardInterrupt:
            print "killed"

            if self.channel:
                print "closing channel"
                self.channel[CHANNEL_INTERFACE].Close()

    def quit(self):
        if self.loop:
            self.loop.quit()
            self.loop = None

    def status_changed_cb(self, state, reason):
        if state == CONNECTION_STATUS_DISCONNECTED:
            print 'connection closed'
            self.quit()

    def request_channel_error_cb(self, exception):
        print 'error:', exception
        self.quit()

    def new_channel_cb(self, object_path, channel_type, handle_type, handle,
            suppress_handler):
        if channel_type != CHANNEL_TYPE_STREAMED_MEDIA:
            return

        self.chan_handle_type = handle_type
        self.chan_handle = handle

        print "new streamed media channel"
        Channel(self.conn._dbus_object._named_service, object_path,
                ready_handler=self.channel_ready_cb)

    def channel_ready_cb(self, channel):
        print "channel ready"
        channel[CHANNEL_INTERFACE].connect_to_signal('Closed', self.closed_cb)
        channel[CHANNEL_INTERFACE_GROUP].connect_to_signal('MembersChanged',
            self.members_changed_cb)

        stream_engine = get_stream_engine()
        handler = dbus.Interface(stream_engine,
            'org.freedesktop.Telepathy.ChannelHandler')
        handler.HandleChannel(
            self.conn._dbus_object._named_service,
            self.conn._dbus_object._object_path,
            CHANNEL_TYPE_STREAMED_MEDIA,
            channel._dbus_object._object_path,
            self.chan_handle_type,
            self.chan_handle)

        self.channel = channel

    def closed_cb(self):
        print "channel closed"
        self.quit()

    def members_changed_cb(self, message, added, removed, local_pending,
            remote_pending, actor, reason):
        print 'MembersChanged', (
            added, removed, local_pending, remote_pending, actor, reason)

class OutgoingCall(Call):
    def __init__(self, conn, contact):
        Call.__init__(self, conn)
        self.contact = contact

    def status_changed_cb (self, state, reason):
        if state == CONNECTION_STATUS_CONNECTED:
            handle = self.conn[CONN_INTERFACE].RequestHandles(
                CONNECTION_HANDLE_TYPE_CONTACT, [self.contact])[0]
            self.handle = handle

            print 'got handle %d for %s' % (handle, self.contact)

            # hack
            import time
            time.sleep(5)

            self.conn[CONN_INTERFACE].RequestChannel(
                CHANNEL_TYPE_STREAMED_MEDIA, CONNECTION_HANDLE_TYPE_NONE,
                handle, True,
                reply_handler=lambda *stuff: None,
                error_handler=self.request_channel_error_cb)

        Call.status_changed_cb(self, state, reason)

    def channel_ready_cb(self, channel):
        Call.channel_ready_cb(self, channel)

        channel[CHANNEL_INTERFACE_GROUP].AddMembers([self.handle], "")

        print "requesting audio/video streams"

        try:
            channel[CHANNEL_TYPE_STREAMED_MEDIA].RequestStreams(
                self.handle,
                [MEDIA_STREAM_TYPE_AUDIO, MEDIA_STREAM_TYPE_VIDEO]);
        except dbus.DBusException, e:
            print "failed:", e
            print "requesting audio stream"

            try:
                channel[CHANNEL_TYPE_STREAMED_MEDIA].RequestStreams(
                    self.handle, [MEDIA_STREAM_TYPE_AUDIO]);
            except dbus.DBusException, e:
                print "failed:", e
                print "giving up"
                self.quit()

class IncomingCall(Call):
    def status_changed_cb (self, state, reason):
        if state == CONNECTION_STATUS_CONNECTED:
            print "waiting for incoming call"

        Call.status_changed_cb(self, state, reason)

    def channel_ready_cb(self, channel):
        Call.channel_ready_cb(self, channel)

        print "accepting incoming call"
        pending = channel[CHANNEL_INTERFACE_GROUP].GetLocalPendingMembers()
        channel[CHANNEL_INTERFACE_GROUP].AddMembers(pending, "")

if __name__ == '__main__':
    assert len(sys.argv) in (2, 3)
    account_file = sys.argv[1]

    manager, protocol, account = read_account(account_file)
    conn = connect(manager, protocol, account)

    if len(sys.argv) > 2:
        contact = sys.argv[2]
        call = OutgoingCall(conn, sys.argv[2])
    else:
        call = IncomingCall(conn)

    print "connecting"
    conn[CONN_INTERFACE].Connect()
    call.run()

    try:
        print "disconnecting"
        conn[CONN_INTERFACE].Disconnect()
    except dbus.DBusException:
        pass

<|endoftext|>


<commit_before>""" Handles loading, processing and validation of all configuration.

Configuration values are determined by checking in the following order

    - Built-in defaults
    - Config file (/etc/fedmsg.ini)
    - Command line arguments

For example, if a config value does not appear in either the config file or on
the command line, then the built-in default is used.  If a value appears in
both the config file and as a command line argument, then the command line
value is used.
"""

import argparse
import collections
import copy
import inspect
import os
import pprint
import sys

import ConfigParser

defaults = dict(
    topic_prefix="org.fedoraproject",
    environment="dev",
    io_threads=1,
    post_init_sleep=0.5,
    timeout=2,
    print_config=False,
    high_water_mark=0,  # zero means no limit
)

VALID_ENVIRONMENTS = ['dev', 'stg', 'prod']

__cache = {}

def load_config(extra_args,
                doc,
                filenames=None,
                invalidate_cache=False,
                fedmsg_command=False):
    """ Setup a config file from the following sources ordered by importance:

      - defaults
      - config file
      - command line arguments

    """
    global __cache

    if invalidate_cache:
        __cache = {}

    if __cache:
        return __cache

    config = copy.deepcopy(defaults)
    config.update(_process_config_file(filenames=filenames))

    # This is optional (and defaults to false) so that only 'fedmsg-*' commands
    # are required to provide these arguments.
    # For instance, the moksha-hub command takes a '-v' argument and internally
    # makes calls to fedmsg.  We don't want to impose all of fedmsg's CLI option
    # constraints on programs that use fedmsg, so we make it optional.
    if fedmsg_command:
        config.update(_process_arguments(extra_args, doc, config))

    # If the user specified a config file on the command line, then start over
    # but read in that file instead.
    if not filenames and config.get('config_filename', None):
        return load_config(extra_args, doc,
                           filenames=[config['config_filename']])

    # Do some post-load type massaging
    for args, kwargs in extra_args:
        # For arguments that accept multiple items (like a list of endpoints,
        # for instance), break those items down into lists from comma separated
        # strings.  That is, do so unless they are already lists.
        if kwargs.get('nargs', None) is '*' and \
           type(config[kwargs['dest']]) != list:
            config[kwargs['dest']] = [
                item.strip() for item in config[kwargs['dest']].split(',')
            ]

    # Just a little debug option.  :)
    if config['print_config']:
        pprint.pprint(config)
        sys.exit(0)

    if config['environment'] not in VALID_ENVIRONMENTS:
        raise ValueError("%r not one of %r" % (
            config['environment'], VALID_ENVIRONMENTS))

    __cache = config
    return config

def _process_arguments(declared_args, doc, config):
    parser = argparse.ArgumentParser(description=doc)

    parser.add_argument(
        '--io-threads',
        dest='io_threads',
        type=int,
        help="Number of io threads for 0mq to use",
        default=config['io_threads'],
    )
    parser.add_argument(
        '--topic-prefix',
        dest='topic_prefix',
        type=str,
        help="Prefix for the topic of each message sent.",
        default=config['topic_prefix'],
    )
    parser.add_argument(
        '--post-init-sleep',
        dest='post_init_sleep',
        type=float,
        help="Number of seconds to sleep after initializing.",
        default=config['post_init_sleep'],
    )
    parser.add_argument(
        '--config-filename',
        dest='config_filename',
        help="Config file to use.",
        default=None,
    )
    parser.add_argument(
        '--print-config',
        dest='print_config',
        help='Simply print out the configuration and exit.  No action taken.',
        default=False,
        action='store_true',
    )
    parser.add_argument(
        '--timeout',
        dest='timeout',
        help="Timeout in seconds for any blocking zmq operations.",
        type=float,
        default=config['timeout'],
    )
    parser.add_argument(
        '--high-water-mark',
        dest='high_water_mark',
        help="Limit on the number of messages in the queue before blocking.",
        type=int,
        default=config['high_water_mark'],
    )

    for args, kwargs in declared_args:

        # Replace the hard-coded extra_args default with the config file value
        # (if it exists)
        if all([k in kwargs for k in ['dest', 'default']]):
            kwargs['default'] = config.get(
                kwargs['dest'], kwargs['default'])

        # Having slurped smart defaults from the config file, add the CLI arg.
        parser.add_argument(*args, **kwargs)

    args = parser.parse_args()
    return dict(args._get_kwargs())

def _process_config_file(filenames=None):

    filenames = filenames or []

    # Validate that these files are really files
    for fname in filenames:
        if not os.path.isfile(fname):
            raise ValueError("%r is not a file." % fname)

    # If nothing specified, look in the default locations
    if not filenames:
        filenames = [
            '/etc/fedmsg.ini',
            os.path.expanduser('~/.fedmsg.ini'),
            os.getcwd() + '/fedmsg.ini',
        ]

    parser = ConfigParser.SafeConfigParser()
    files = parser.read(filenames=filenames)

    config = {}
    if parser.has_section('fedmsg'):
        config = dict(parser.items('fedmsg'))

    return config
<commit_msg>Removed spurious import.<commit_after>""" Handles loading, processing and validation of all configuration.

Configuration values are determined by checking in the following order

    - Built-in defaults
    - Config file (/etc/fedmsg.ini)
    - Command line arguments

For example, if a config value does not appear in either the config file or on
the command line, then the built-in default is used.  If a value appears in
both the config file and as a command line argument, then the command line
value is used.
"""

import argparse
import collections
import copy
import os
import pprint
import sys

import ConfigParser

defaults = dict(
    topic_prefix="org.fedoraproject",
    environment="dev",
    io_threads=1,
    post_init_sleep=0.5,
    timeout=2,
    print_config=False,
    high_water_mark=0,  # zero means no limit
)

VALID_ENVIRONMENTS = ['dev', 'stg', 'prod']

__cache = {}

def load_config(extra_args,
                doc,
                filenames=None,
                invalidate_cache=False,
                fedmsg_command=False):
    """ Setup a config file from the following sources ordered by importance:

      - defaults
      - config file
      - command line arguments

    """
    global __cache

    if invalidate_cache:
        __cache = {}

    if __cache:
        return __cache

    config = copy.deepcopy(defaults)
    config.update(_process_config_file(filenames=filenames))

    # This is optional (and defaults to false) so that only 'fedmsg-*' commands
    # are required to provide these arguments.
    # For instance, the moksha-hub command takes a '-v' argument and internally
    # makes calls to fedmsg.  We don't want to impose all of fedmsg's CLI option
    # constraints on programs that use fedmsg, so we make it optional.
    if fedmsg_command:
        config.update(_process_arguments(extra_args, doc, config))

    # If the user specified a config file on the command line, then start over
    # but read in that file instead.
    if not filenames and config.get('config_filename', None):
        return load_config(extra_args, doc,
                           filenames=[config['config_filename']])

    # Do some post-load type massaging
    for args, kwargs in extra_args:
        # For arguments that accept multiple items (like a list of endpoints,
        # for instance), break those items down into lists from comma separated
        # strings.  That is, do so unless they are already lists.
        if kwargs.get('nargs', None) is '*' and \
           type(config[kwargs['dest']]) != list:
            config[kwargs['dest']] = [
                item.strip() for item in config[kwargs['dest']].split(',')
            ]

    # Just a little debug option.  :)
    if config['print_config']:
        pprint.pprint(config)
        sys.exit(0)

    if config['environment'] not in VALID_ENVIRONMENTS:
        raise ValueError("%r not one of %r" % (
            config['environment'], VALID_ENVIRONMENTS))

    __cache = config
    return config

def _process_arguments(declared_args, doc, config):
    parser = argparse.ArgumentParser(description=doc)

    parser.add_argument(
        '--io-threads',
        dest='io_threads',
        type=int,
        help="Number of io threads for 0mq to use",
        default=config['io_threads'],
    )
    parser.add_argument(
        '--topic-prefix',
        dest='topic_prefix',
        type=str,
        help="Prefix for the topic of each message sent.",
        default=config['topic_prefix'],
    )
    parser.add_argument(
        '--post-init-sleep',
        dest='post_init_sleep',
        type=float,
        help="Number of seconds to sleep after initializing.",
        default=config['post_init_sleep'],
    )
    parser.add_argument(
        '--config-filename',
        dest='config_filename',
        help="Config file to use.",
        default=None,
    )
    parser.add_argument(
        '--print-config',
        dest='print_config',
        help='Simply print out the configuration and exit.  No action taken.',
        default=False,
        action='store_true',
    )
    parser.add_argument(
        '--timeout',
        dest='timeout',
        help="Timeout in seconds for any blocking zmq operations.",
        type=float,
        default=config['timeout'],
    )
    parser.add_argument(
        '--high-water-mark',
        dest='high_water_mark',
        help="Limit on the number of messages in the queue before blocking.",
        type=int,
        default=config['high_water_mark'],
    )

    for args, kwargs in declared_args:

        # Replace the hard-coded extra_args default with the config file value
        # (if it exists)
        if all([k in kwargs for k in ['dest', 'default']]):
            kwargs['default'] = config.get(
                kwargs['dest'], kwargs['default'])

        # Having slurped smart defaults from the config file, add the CLI arg.
        parser.add_argument(*args, **kwargs)

    args = parser.parse_args()
    return dict(args._get_kwargs())

def _process_config_file(filenames=None):

    filenames = filenames or []

    # Validate that these files are really files
    for fname in filenames:
        if not os.path.isfile(fname):
            raise ValueError("%r is not a file." % fname)

    # If nothing specified, look in the default locations
    if not filenames:
        filenames = [
            '/etc/fedmsg.ini',
            os.path.expanduser('~/.fedmsg.ini'),
            os.getcwd() + '/fedmsg.ini',
        ]

    parser = ConfigParser.SafeConfigParser()
    files = parser.read(filenames=filenames)

    config = {}
    if parser.has_section('fedmsg'):
        config = dict(parser.items('fedmsg'))

    return config
<|endoftext|>


<commit_before>import os
import sys
import clang.cindex
from clang.cindex import AccessSpecifier, CursorKind, TypeKind

def _get_annotations(node):
    return [c.displayname for c in node.get_children()
            if c.kind == CursorKind.ANNOTATE_ATTR]

class Type:
    def __init__(self, cindex_type):
        self.kind = cindex_type.kind
        self.name = cindex_type.spelling
        self.is_pointer = self.kind == TypeKind.POINTER
        self.is_reference = self.kind == TypeKind.LVALUEREFERENCE
        self.is_const = cindex_type.is_const_qualified()
        if self.is_pointer or self.is_reference:
            self.pointee = Type(cindex_type.get_pointee())
        else: self.pointee = None

    def __str__(self):
        return self.name

class Member:
    def __init__(self, cursor):
        self.type = Type(cursor.type)
        self.name = cursor.spelling

class FunctionArgument:
    def __str__(self):
        if self.name is None:
            return str(self.type)
        return "{} {}".format(self.type, self.name)

    def __init__(self, type, name=None):
        self.type = type
        self.name = name or None

class _Function(object):
    def __init__(self, cursor):
        self.name = cursor.spelling
        arguments = [x.spelling or None for x in cursor.get_arguments()]
        argument_types = [Type(x) for x in cursor.type.argument_types()]

        self.return_type = Type(cursor.type.get_result())
        self.arguments = []
        self.annotations = _get_annotations(cursor)

        for t,n in zip(argument_types,arguments):
            self.arguments.append(FunctionArgument(t,n))

    def __str__(self):
        return '{} {}({})'.format(str(self.return_type), str(self.name), ', '.join([str(a) for a in self.arguments]))

class Function(_Function):

    def __init__(self, cursor, namespaces=[]):
        _Function.__init__(self, cursor)
        self.namespace = '::'.join(namespaces)
        if self.namespace:
            self.qualified_name = '::'.join([self.namespace, self.name])
        else:
            self.qualified_name = self.name

    def __eq__(self, f):
        if self.name != f.name:
            return False
        if self.namespace != f.namespace:
            return False
        if len(self.arguments) != len(f.arguments):
            return False
        for x,fx in zip([arg.type for arg in self.arguments],[arg.type for arg in f.arguments]):
            if x.name != fx.name:
                return False
        return True

class Method(Function):

    def __init__(self, cursor):
        _Function.__init__(self, cursor)
        self.is_const = cursor.is_const_method()
        self.is_virtual = cursor.is_virtual_method()
        self.is_pure_virtual = cursor.is_pure_virtual_method()
        self.is_public = (cursor.access_specifier == AccessSpecifier.PUBLIC)

    def __str__(self):
        s = super(Function, self).__str__()
        if self.is_const:
            s = '{} const'.format(s)
        if self.is_pure_virtual:
            s = 'virtual {} = 0'.format(s)
        elif self.is_virtual:
            s = 'virtual {}'.format(s)
        return s

class Class(object):
    def __str__(self):
        return "class {}".format(self.name)

    def __init__(self, cursor, namespaces):
        self.name = cursor.spelling
        self.namespace = '::'.join(namespaces)
        if self.namespace:
            self.qualified_name = '::'.join([self.namespace, self.name])
        else:
            self.qualified_name = self.name
        self.constructors = []
        self.methods = []
        self.members = []
        self.annotations = _get_annotations(cursor)
        self.base_classes = []
        # FIXME: populate these fields with AST info
        self.source_file = str(cursor.location.file)
        self.source_line = int(cursor.location.line)
        self.source_column = int(cursor.location.column)

        for c in cursor.get_children():
            if c.kind == CursorKind.CXX_METHOD and c.type.kind == TypeKind.FUNCTIONPROTO:
                f = Method(c)
                self.methods.append(f)
            elif c.kind == CursorKind.CONSTRUCTOR and c.type.kind == TypeKind.FUNCTIONPROTO:
                f = Method(c)
                self.constructors.append(f)
            elif c.kind == CursorKind.FIELD_DECL:
                f = Member(c)
                self.members.append(f)
            elif c.kind == CursorKind.CXX_BASE_SPECIFIER:
                self.base_classes.append(c.type.spelling)

class Model(object):

    def __init__(self, translation_unit=None):
       self.functions = []
       self.classes = []
       if translation_unit is not None:
           self.add_child_nodes(translation_unit.cursor, [])

    def extend(self, translation_unit):
        m = Model(translation_unit)
        # Check for duplicates and inconsistencies.
        for new_class in m.classes:
            is_new = True
            for old_class in self.classes:
                if new_class.qualified_name == old_class.qualified_name:
                    if new_class.source_file != old_class.source_file:
                        raise Exception("Class {} is defined in multiple locations: {} {}".format(
                            old_class.qualified_name, old_class.source_file, new_class.source_file))
                    # Move on as there can only be one match
                    is_new = False
                    break

            if is_new:
                self.classes.append(new_class)

        # We only look at declarations for functions so won't raise exceptions
        for new_function in m.functions:
            is_new = True
            for old_function in self.functions:
                if new_function == old_function:
                    is_new = False
                    break
            if is_new:
                self.functions.append(new_function)

    def add_child_nodes(self, cursor, namespaces=[]):
        for c in cursor.get_children():
            if c.kind == CursorKind.CLASS_DECL or c.kind == CursorKind.STRUCT_DECL:
                self.classes.append(Class(c,namespaces))
            if c.kind == CursorKind.FUNCTION_DECL and c.type.kind == TypeKind.FUNCTIONPROTO:
                self.functions.append(Function(c,namespaces))
            elif c.kind == CursorKind.NAMESPACE:
                child_namespaces = list(namespaces)
                child_namespaces.append(c.spelling)
                self.add_child_nodes(c, child_namespaces)

def apply_class_annotations(model_class):
    for m in model_class.methods:
        if m.is_pure_virtual:
            model_class.is_abstract = True
        if m.return_type.kind == TypeKind.POINTER and m.return_type.pointee.kind != TypeKind.CHAR_S:
            m.returns_sub_object = True
            m.returns_nullable = True

    return model_class

<commit_msg>Remove duplicate blank lines in cppmodel.py (#101)<commit_after>import os
import sys
import clang.cindex
from clang.cindex import AccessSpecifier, CursorKind, TypeKind

def _get_annotations(node):
    return [c.displayname for c in node.get_children()
            if c.kind == CursorKind.ANNOTATE_ATTR]

class Type:
    def __init__(self, cindex_type):
        self.kind = cindex_type.kind
        self.name = cindex_type.spelling
        self.is_pointer = self.kind == TypeKind.POINTER
        self.is_reference = self.kind == TypeKind.LVALUEREFERENCE
        self.is_const = cindex_type.is_const_qualified()
        if self.is_pointer or self.is_reference:
            self.pointee = Type(cindex_type.get_pointee())
        else: self.pointee = None

    def __str__(self):
        return self.name

class Member:
    def __init__(self, cursor):
        self.type = Type(cursor.type)
        self.name = cursor.spelling

class FunctionArgument:
    def __str__(self):
        if self.name is None:
            return str(self.type)
        return "{} {}".format(self.type, self.name)

    def __init__(self, type, name=None):
        self.type = type
        self.name = name or None

class _Function(object):
    def __init__(self, cursor):
        self.name = cursor.spelling
        arguments = [x.spelling or None for x in cursor.get_arguments()]
        argument_types = [Type(x) for x in cursor.type.argument_types()]

        self.return_type = Type(cursor.type.get_result())
        self.arguments = []
        self.annotations = _get_annotations(cursor)

        for t,n in zip(argument_types,arguments):
            self.arguments.append(FunctionArgument(t,n))

    def __str__(self):
        return '{} {}({})'.format(str(self.return_type), str(self.name), ', '.join([str(a) for a in self.arguments]))

class Function(_Function):
    def __init__(self, cursor, namespaces=[]):
        _Function.__init__(self, cursor)
        self.namespace = '::'.join(namespaces)
        if self.namespace:
            self.qualified_name = '::'.join([self.namespace, self.name])
        else:
            self.qualified_name = self.name

    def __eq__(self, f):
        if self.name != f.name:
            return False
        if self.namespace != f.namespace:
            return False
        if len(self.arguments) != len(f.arguments):
            return False
        for x,fx in zip([arg.type for arg in self.arguments],[arg.type for arg in f.arguments]):
            if x.name != fx.name:
                return False
        return True

class Method(Function):
    def __init__(self, cursor):
        _Function.__init__(self, cursor)
        self.is_const = cursor.is_const_method()
        self.is_virtual = cursor.is_virtual_method()
        self.is_pure_virtual = cursor.is_pure_virtual_method()
        self.is_public = (cursor.access_specifier == AccessSpecifier.PUBLIC)

    def __str__(self):
        s = super(Function, self).__str__()
        if self.is_const:
            s = '{} const'.format(s)
        if self.is_pure_virtual:
            s = 'virtual {} = 0'.format(s)
        elif self.is_virtual:
            s = 'virtual {}'.format(s)
        return s

class Class(object):
    def __str__(self):
        return "class {}".format(self.name)

    def __init__(self, cursor, namespaces):
        self.name = cursor.spelling
        self.namespace = '::'.join(namespaces)
        if self.namespace:
            self.qualified_name = '::'.join([self.namespace, self.name])
        else:
            self.qualified_name = self.name
        self.constructors = []
        self.methods = []
        self.members = []
        self.annotations = _get_annotations(cursor)
        self.base_classes = []
        # FIXME: populate these fields with AST info
        self.source_file = str(cursor.location.file)
        self.source_line = int(cursor.location.line)
        self.source_column = int(cursor.location.column)

        for c in cursor.get_children():
            if c.kind == CursorKind.CXX_METHOD and c.type.kind == TypeKind.FUNCTIONPROTO:
                f = Method(c)
                self.methods.append(f)
            elif c.kind == CursorKind.CONSTRUCTOR and c.type.kind == TypeKind.FUNCTIONPROTO:
                f = Method(c)
                self.constructors.append(f)
            elif c.kind == CursorKind.FIELD_DECL:
                f = Member(c)
                self.members.append(f)
            elif c.kind == CursorKind.CXX_BASE_SPECIFIER:
                self.base_classes.append(c.type.spelling)

class Model(object):
    def __init__(self, translation_unit=None):
       self.functions = []
       self.classes = []
       if translation_unit is not None:
           self.add_child_nodes(translation_unit.cursor, [])

    def extend(self, translation_unit):
        m = Model(translation_unit)
        # Check for duplicates and inconsistencies.
        for new_class in m.classes:
            is_new = True
            for old_class in self.classes:
                if new_class.qualified_name == old_class.qualified_name:
                    if new_class.source_file != old_class.source_file:
                        raise Exception("Class {} is defined in multiple locations: {} {}".format(
                            old_class.qualified_name, old_class.source_file, new_class.source_file))
                    # Move on as there can only be one match
                    is_new = False
                    break

            if is_new:
                self.classes.append(new_class)

        # We only look at declarations for functions so won't raise exceptions
        for new_function in m.functions:
            is_new = True
            for old_function in self.functions:
                if new_function == old_function:
                    is_new = False
                    break
            if is_new:
                self.functions.append(new_function)

    def add_child_nodes(self, cursor, namespaces=[]):
        for c in cursor.get_children():
            if c.kind == CursorKind.CLASS_DECL or c.kind == CursorKind.STRUCT_DECL:
                self.classes.append(Class(c,namespaces))
            if c.kind == CursorKind.FUNCTION_DECL and c.type.kind == TypeKind.FUNCTIONPROTO:
                self.functions.append(Function(c,namespaces))
            elif c.kind == CursorKind.NAMESPACE:
                child_namespaces = list(namespaces)
                child_namespaces.append(c.spelling)
                self.add_child_nodes(c, child_namespaces)

def apply_class_annotations(model_class):
    for m in model_class.methods:
        if m.is_pure_virtual:
            model_class.is_abstract = True
        if m.return_type.kind == TypeKind.POINTER and m.return_type.pointee.kind != TypeKind.CHAR_S:
            m.returns_sub_object = True
            m.returns_nullable = True

    return model_class

<|endoftext|>


<commit_before>__author__ = 'calvin'

import collections
from functools import partial

from . import Property

class ObservableList(collections.MutableSequence):
    def __init__(self, l, dispatch_method):
        if not type(l) == list and not type(l) == tuple and not isinstance(l, ObservableList):
            raise ValueError('Observable list must only be initialized with lists as arguments')
        self.list = list(l)
        self.dispatch = dispatch_method

    def __repr__(self):
        return self.list.__repr__()

    def __get__(self, instance, owner):
        return self.list

    def __getitem__(self, item):
        return self.list[item]

    def __setitem__(self, key, value):
        if self.list[key] != value:
            self.list[key] = value
            self.dispatch(self.list)

    def __reversed__(self):
        return reversed(self.list)

    def __delitem__(self, key):
        del self.list[key]
        self.dispatch(self.list)

    def __len__(self):
        return len(self.list)

    def __iter__(self):
        return iter(self.list)

    def __nonzero__(self):
        return bool(self.list)

    def __getstate__(self):
        return self.list

    def __reduce__(self):
        return (list, tuple(), None, iter(self.list), None)

    def insert(self, index, value):
        self.list.insert(index, value)
        self.dispatch(self.list)

    def append(self, value):
        self.list.append(value)
        self.dispatch(self.list)

    def extend(self, values):
        self.list.extend(values)
        self.dispatch(self.list)

    def pop(self, index=-1):
        value = self.list.pop(index)
        self.dispatch(self.list)

        return value

    def __eq__(self, other):
        return self.list == other

    def __ne__(self, other):
        return self.list != other

    def __nonzero__(self):
        return bool(self.list)

class ListProperty(Property):

    def register(self, instance, property_name, value):
        self.value = ObservableList(value, dispatch_method=partial(instance.dispatch, property_name, instance))
        super(ListProperty, self).register(instance, property_name, self.value)

    def __set__(self, obj, value):
        p = self.instances[obj]
        # Check if we need to dispatch
        do_dispatch = len(p['value'].list) != len(value) or not ListProperty.compare_sequences(p['value'], value)
        self.instances[obj]['value'].list[:] = value        # Assign to ObservableList's value
        if do_dispatch:
            for callback in p['callbacks']:
                if callback(obj, value):
                    break

    @staticmethod
    def compare_sequences(iter1, iter2):
        """
        Compares two iterators to determine if they are equal. Used to compare lists and tuples
        # """
        for a, b in zip(iter1, iter2):
            if a != b:
                return False
        return True
<commit_msg>allow ListProperty to create numpy arrays when they are defined with a dtype.<commit_after>__author__ = 'calvin'

import collections
from functools import partial

import numpy as np

from . import Property

class ObservableList(collections.MutableSequence):
    def __init__(self, l, dispatch_method, dtype=None):
        if not type(l) == list and not type(l) == tuple and not isinstance(l, ObservableList):
            raise ValueError('Observable list must only be initialized with sequences as arguments')
        if dtype:
            self.list = np.array(l, dtype=dtype)
        else:
            self.list = list(l)

        self.dispatch = dispatch_method

    def __repr__(self):
        return self.list.__repr__()

    def __get__(self, instance, owner):
        return self.list

    def __getitem__(self, item):
        return self.list[item]

    def __setitem__(self, key, value):
        if self.list[key] != value:
            self.list[key] = value
            self.dispatch(self.list)

    def __reversed__(self):
        return reversed(self.list)

    def __delitem__(self, key):
        del self.list[key]
        self.dispatch(self.list)

    def __len__(self):
        return len(self.list)

    def __iter__(self):
        return iter(self.list)

    def __nonzero__(self):
        return bool(self.list)

    def __getstate__(self):
        return self.list

    def __reduce__(self):
        return (list, tuple(), None, iter(self.list), None)

    def insert(self, index, value):
        self.list.insert(index, value)
        self.dispatch(self.list)

    def append(self, value):
        self.list.append(value)
        self.dispatch(self.list)

    def extend(self, values):
        self.list.extend(values)
        self.dispatch(self.list)

    def pop(self, index=-1):
        value = self.list.pop(index)
        self.dispatch(self.list)

        return value

    def __eq__(self, other):
        return self.list == other

    def __ne__(self, other):
        return self.list != other

    def __nonzero__(self):
        return bool(self.list)

class ListProperty(Property):

    def register(self, instance, property_name, value, dtype=None):
        self.value = ObservableList(value,
                                    dispatch_method=partial(instance.dispatch, property_name, instance),
                                    dtype=self._additionals.get('dtype'))
        super(ListProperty, self).register(instance, property_name, self.value)

    def __set__(self, obj, value):
        p = self.instances[obj]
        # Check if we need to dispatch
        do_dispatch = len(p['value'].list) != len(value) or not ListProperty.compare_sequences(p['value'], value)
        self.instances[obj]['value'].list[:] = value        # Assign to ObservableList's value
        if do_dispatch:
            for callback in p['callbacks']:
                if callback(obj, value):
                    break

    @staticmethod
    def compare_sequences(iter1, iter2):
        """
        Compares two iterators to determine if they are equal. Used to compare lists and tuples
        # """
        for a, b in zip(iter1, iter2):
            if a != b:
                return False
        return True
<|endoftext|>