<commit_before>#!/usr/bin/env python
import sys
import os
import inspect
import shutil
import subprocess
import pip
from glob import glob

from termcolor import cprint

def install_aptitude():
    cprint('Installing OS (Debian/Ubuntu) requirements', 'green')
    subprocess.call('sudo aptitude install build-essential libhdf5-dev libgeos-dev libproj-dev libfreetype6-dev python-dev libblas-dev liblapack-dev gfortran libnetcdf-dev python-tk tcl-dev tk-dev', shell=True)
    subprocess.call('cd /usr/lib/ && sudo ln -s libgeos-3.4.2.so libgeos.so', shell=True)

def copy_files():
    cprint('Copying files to local directory', 'green')
    target_dir = os.getcwd()

    try:
        import stormtracks.installation
        source_dir = os.path.dirname(stormtracks.installation.__file__)
    except ImportError:
        cprint('COULD NOT IMPORT stormtracks.installation', 'red')
        raise

    cprint('Copying from:\n    {}\n  to\n    {}'.format(source_dir, target_dir), 'green')
    shutil.copytree(os.path.join(source_dir, 'requirements'), os.path.join(target_dir, 'requirements'))

    shutil.copyfile(os.path.join(source_dir, 'settings', 'default_stormtracks_settings.py'),
                    os.path.join(target_dir, 'default_stormtracks_settings.py'))
    if os.path.exists(os.path.join(target_dir, 'stormtracks_settings.py')):
        # os.rename(os.path.join(target_dir, 'stormtracks_settings.py'), os.path.join(target_dir, 'stormtracks_settings.py.bak'))
        cprint('stormtracks_settings.py already exists, skipping', 'yellow')
    else:
        shutil.copyfile(os.path.join(source_dir, 'settings', 'stormtracks_settings.py'),
                        os.path.join(target_dir, 'default_stormtracks_settings.py'))

    shutil.copytree(os.path.join(source_dir, 'classifiers'), os.path.join(target_dir, 'classifiers'))
    shutil.copytree(os.path.join(source_dir, 'plots'), os.path.join(target_dir, 'plots'))
    for script_file in glob(os.path.join(source_dir, 'scripts/*.py')):
        if os.path.basename(script_file) == '__init__.py':
            continue
        if os.path.exists(os.path.join(target_dir, os.path.basename(script_file))):
            cprint('{} already exists, skipping'.format(os.path.basename(script_file), 'yellow'))
        else:
            shutil.copyfile(script_file, os.path.basename(script_file))

def install():
    cprint('Installing all dependencies', 'green', attrs=['bold'])
    copy_files()
    install_pip()

def install_pip():
    cprint('Installing pip requirements', 'green')

    pip.main(['install', '-r', 'requirements/requirements_a.txt'])
    pip.main(['install', '-r', 'requirements/requirements_b.txt'])
    pip.main(['install', '-r', 'requirements/requirements_c.txt', '--allow-external', 'basemap', '--allow-unverified', 'basemap'])

def clean():
    shutil.rmtree('requirements', ignore_errors=True)
    shutil.rmtree('classifiers', ignore_errors=True)
    shutil.rmtree('plots', ignore_errors=True)
    try:
        os.remove('stormtracks_settings.py')
    except OSError:
        pass

def list_data_sources(full=False):
    from stormtracks.load_settings import settings
    cprint('Using data dir: {}'.format(settings.DATA_DIR), 'green', attrs=['bold'])

    cprint('20CR Data Sources ({}):'.format(settings.C20_FULL_DATA_DIR), 'green')
    for c20year in glob(os.path.join(settings.C20_FULL_DATA_DIR, '*')):
        cprint('  {}'.format(c20year), 'blue', attrs=['bold'])
        for c20file in sorted(glob(os.path.join(c20year, '*.nc'))):
            cprint('    {}'.format(c20file), 'blue')

    ibtracs_tarball_file = os.path.join(settings.DATA_DIR, 'ibtracs_v03r05_dataset_184210_201305.tar.gz')

    cprint('IBTrACS Data Sources ({}):'.format(settings.IBTRACS_DATA_DIR), 'green')
    cprint('  IBTrACS tarball downloaded: {}'.format(os.path.exists(ibtracs_tarball_file)), 'blue', attrs=['bold'])
    cprint('  IBTrACS tarball decompressed: {}'.format(os.path.exists(settings.IBTRACS_DATA_DIR)), 'blue', attrs=['bold'])
    if full:
        for ibfile in glob(os.path.join(settings.IBTRACS_DATA_DIR, '*')):
            cprint('  {}'.format(ibfile), 'blue')

def list_output(full=False):
    from stormtracks.load_settings import settings
    from stormtracks.results import StormtracksResultsManager, StormtracksNumpyResultsManager
    cprint('Using output dir: {}'.format(settings.OUTPUT_DIR), 'green', attrs=['bold'])
    cprint('  Tracking results: {}'.format(settings.TRACKING_RESULTS), 'green', attrs=['bold'])
    results_manager = StormtracksResultsManager(settings.TRACKING_RESULTS)
    results_years = results_manager.list_years()
    for year in results_years:
        if full:
            cprint('    Results year: {}'.format(year), 'blue', attrs=['bold'])
            ensemble_members = results_manager.list_ensemble_members(year)
            for ensemble_member in ensemble_members:
                cprint('      ensemble member: {}'.format(ensemble_member), 'blue')
                for result in results_manager.list_results(year, ensemble_member):
                    if result in ['vort_tracks_by_date', 'good_matches']:
                        cprint('        result: {}'.format(result), 'blue')
        else:
            ensemble_members = results_manager.list_ensemble_members(year)
            vort_tracks_count = 0
            good_matches_count = 0
            for ensemble_member in ensemble_members:
                for result in results_manager.list_results(year, ensemble_member):
                    if result.split('-')[0] == 'vort_tracks_by_date':
                        vort_tracks_count += 1
                    elif result.split('-')[0] == 'good_matches':
                        good_matches_count += 1

            cprint('    Tracking results year: {}; {} ensemble members; {} vort tracks, {} good matches'.
                   format(year, len(ensemble_members), vort_tracks_count, good_matches_count), 'blue', attrs=['bold'])

    cprint('  Field collection results: {}'.format(settings.FIELD_RESULTS), 'green', attrs=['bold'])
    results_manager = StormtracksResultsManager(settings.FIELD_RESULTS)
    results_years = results_manager.list_years()
    for year in results_years:
        if full:
            cprint('    Field coll. esults year: {}'.format(year), 'blue', attrs=['bold'])
            ensemble_members = results_manager.list_ensemble_members(year)
            for ensemble_member in ensemble_members:
                cprint('      ensemble member: {}'.format(ensemble_member), 'blue')
                for result in results_manager.list_results(year, ensemble_member):
                    if result in ['cyclones']:
                        cprint('        result: {}'.format(result), 'blue')
        else:
            ensemble_members = results_manager.list_ensemble_members(year)
            cyclones = 0
            for ensemble_member in ensemble_members:
                for result in results_manager.list_results(year, ensemble_member):
                    if result == 'cyclones':
                        cyclones += 1

            cprint('    Field coll. results year: {}; {} ensemble members; {} cylones'.format(year, len(ensemble_members), cyclones), 'blue', attrs=['bold'])

def main():
    if sys.argv[1] == 'install':
        install()
    elif sys.argv[1] == 'install_full':
        install_aptitude()
        install()
    elif sys.argv[1] == 'clean':
        clean()
    elif sys.argv[1] == 'reinstall':
        clean()
        install()
    elif sys.argv[1] == 'list_data_sources':
        list_data_sources()
    elif sys.argv[1] == 'list_output':
        list_output()

if __name__ == '__main__':
    main()
<commit_msg>Fix bug with settings file having wrong name(s).<commit_after>#!/usr/bin/env python
import sys
import os
import inspect
import shutil
import subprocess
import pip
from glob import glob

from termcolor import cprint

def install_aptitude():
    cprint('Installing OS (Debian/Ubuntu) requirements', 'green')
    subprocess.call('sudo aptitude install build-essential libhdf5-dev libgeos-dev libproj-dev libfreetype6-dev python-dev libblas-dev liblapack-dev gfortran libnetcdf-dev python-tk tcl-dev tk-dev', shell=True)
    subprocess.call('cd /usr/lib/ && sudo ln -s libgeos-3.4.2.so libgeos.so', shell=True)

def copy_files():
    cprint('Copying files to local directory', 'green')
    target_dir = os.getcwd()

    try:
        import stormtracks.installation
        source_dir = os.path.dirname(stormtracks.installation.__file__)
    except ImportError:
        cprint('COULD NOT IMPORT stormtracks.installation', 'red')
        raise

    cprint('Copying from:\n    {}\n  to\n    {}'.format(source_dir, target_dir), 'green')
    shutil.copytree(os.path.join(source_dir, 'requirements'), os.path.join(target_dir, 'requirements'))

    shutil.copyfile(os.path.join(source_dir, 'settings', 'default_stormtracks_settings.py'),
                    os.path.join(target_dir, 'default_stormtracks_settings.py'))
    if os.path.exists(os.path.join(target_dir, 'stormtracks_settings.py')):
        # os.rename(os.path.join(target_dir, 'stormtracks_settings.py'), os.path.join(target_dir, 'stormtracks_settings.py.bak'))
        cprint('stormtracks_settings.py already exists, skipping', 'yellow')
    else:
        shutil.copyfile(os.path.join(source_dir, 'settings', 'default_stormtracks_settings.py'),
                        os.path.join(target_dir, 'stormtracks_settings.py'))

    shutil.copytree(os.path.join(source_dir, 'classifiers'), os.path.join(target_dir, 'classifiers'))
    shutil.copytree(os.path.join(source_dir, 'plots'), os.path.join(target_dir, 'plots'))
    for script_file in glob(os.path.join(source_dir, 'scripts/*.py')):
        if os.path.basename(script_file) == '__init__.py':
            continue
        if os.path.exists(os.path.join(target_dir, os.path.basename(script_file))):
            cprint('{} already exists, skipping'.format(os.path.basename(script_file), 'yellow'))
        else:
            shutil.copyfile(script_file, os.path.basename(script_file))

def install():
    cprint('Installing all dependencies', 'green', attrs=['bold'])
    copy_files()
    install_pip()

def install_pip():
    cprint('Installing pip requirements', 'green')

    pip.main(['install', '-r', 'requirements/requirements_a.txt'])
    pip.main(['install', '-r', 'requirements/requirements_b.txt'])
    pip.main(['install', '-r', 'requirements/requirements_c.txt', '--allow-external', 'basemap', '--allow-unverified', 'basemap'])

def clean():
    shutil.rmtree('requirements', ignore_errors=True)
    shutil.rmtree('classifiers', ignore_errors=True)
    shutil.rmtree('plots', ignore_errors=True)
    try:
        os.remove('stormtracks_settings.py')
    except OSError:
        pass

def list_data_sources(full=False):
    from stormtracks.load_settings import settings
    cprint('Using data dir: {}'.format(settings.DATA_DIR), 'green', attrs=['bold'])

    cprint('20CR Data Sources ({}):'.format(settings.C20_FULL_DATA_DIR), 'green')
    for c20year in glob(os.path.join(settings.C20_FULL_DATA_DIR, '*')):
        cprint('  {}'.format(c20year), 'blue', attrs=['bold'])
        for c20file in sorted(glob(os.path.join(c20year, '*.nc'))):
            cprint('    {}'.format(c20file), 'blue')

    ibtracs_tarball_file = os.path.join(settings.DATA_DIR, 'ibtracs_v03r05_dataset_184210_201305.tar.gz')

    cprint('IBTrACS Data Sources ({}):'.format(settings.IBTRACS_DATA_DIR), 'green')
    cprint('  IBTrACS tarball downloaded: {}'.format(os.path.exists(ibtracs_tarball_file)), 'blue', attrs=['bold'])
    cprint('  IBTrACS tarball decompressed: {}'.format(os.path.exists(settings.IBTRACS_DATA_DIR)), 'blue', attrs=['bold'])
    if full:
        for ibfile in glob(os.path.join(settings.IBTRACS_DATA_DIR, '*')):
            cprint('  {}'.format(ibfile), 'blue')

def list_output(full=False):
    from stormtracks.load_settings import settings
    from stormtracks.results import StormtracksResultsManager, StormtracksNumpyResultsManager
    cprint('Using output dir: {}'.format(settings.OUTPUT_DIR), 'green', attrs=['bold'])
    cprint('  Tracking results: {}'.format(settings.TRACKING_RESULTS), 'green', attrs=['bold'])
    results_manager = StormtracksResultsManager(settings.TRACKING_RESULTS)
    results_years = results_manager.list_years()
    for year in results_years:
        if full:
            cprint('    Results year: {}'.format(year), 'blue', attrs=['bold'])
            ensemble_members = results_manager.list_ensemble_members(year)
            for ensemble_member in ensemble_members:
                cprint('      ensemble member: {}'.format(ensemble_member), 'blue')
                for result in results_manager.list_results(year, ensemble_member):
                    if result in ['vort_tracks_by_date', 'good_matches']:
                        cprint('        result: {}'.format(result), 'blue')
        else:
            ensemble_members = results_manager.list_ensemble_members(year)
            vort_tracks_count = 0
            good_matches_count = 0
            for ensemble_member in ensemble_members:
                for result in results_manager.list_results(year, ensemble_member):
                    if result.split('-')[0] == 'vort_tracks_by_date':
                        vort_tracks_count += 1
                    elif result.split('-')[0] == 'good_matches':
                        good_matches_count += 1

            cprint('    Tracking results year: {}; {} ensemble members; {} vort tracks, {} good matches'.
                   format(year, len(ensemble_members), vort_tracks_count, good_matches_count), 'blue', attrs=['bold'])

    cprint('  Field collection results: {}'.format(settings.FIELD_RESULTS), 'green', attrs=['bold'])
    results_manager = StormtracksResultsManager(settings.FIELD_RESULTS)
    results_years = results_manager.list_years()
    for year in results_years:
        if full:
            cprint('    Field coll. esults year: {}'.format(year), 'blue', attrs=['bold'])
            ensemble_members = results_manager.list_ensemble_members(year)
            for ensemble_member in ensemble_members:
                cprint('      ensemble member: {}'.format(ensemble_member), 'blue')
                for result in results_manager.list_results(year, ensemble_member):
                    if result in ['cyclones']:
                        cprint('        result: {}'.format(result), 'blue')
        else:
            ensemble_members = results_manager.list_ensemble_members(year)
            cyclones = 0
            for ensemble_member in ensemble_members:
                for result in results_manager.list_results(year, ensemble_member):
                    if result == 'cyclones':
                        cyclones += 1

            cprint('    Field coll. results year: {}; {} ensemble members; {} cylones'.format(year, len(ensemble_members), cyclones), 'blue', attrs=['bold'])

def main():
    if sys.argv[1] == 'install':
        install()
    elif sys.argv[1] == 'install_full':
        install_aptitude()
        install()
    elif sys.argv[1] == 'clean':
        clean()
    elif sys.argv[1] == 'reinstall':
        clean()
        install()
    elif sys.argv[1] == 'list_data_sources':
        list_data_sources()
    elif sys.argv[1] == 'list_output':
        list_output()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
argh
"""
from roamer.command import Command
from roamer.record import Record
from roamer.entry import Entry

class Engine(object):
    def __init__(self, original_dir, edit_dir):
        self.commands = []
        for digest, original_entry in original_dir.entries.iteritems():
            new_entries = edit_dir.find(digest)
            if new_entries is None:
                self.commands.append(Command('roamer-trash', original_entry))
                continue
            found_original = False
            for new_entry in new_entries:
                if new_entry.name == original_entry.name:
                    found_original = True
                else:
                    self.commands.append(Command('cp', original_entry, new_entry))
            if not found_original:
                self.commands.append(Command('roamer-trash', original_entry))

        add_blank_entries = edit_dir.find(None)
        if add_blank_entries:
            for entry in add_blank_entries:
                self.commands.append(Command('touch', entry))

        unknown_digets = set(edit_dir.entries.keys()) - set(original_dir.entries.keys())

        for digest in filter(None, unknown_digets):
            record = Record(original_dir)
            outside_entry = record.entries.get(digest) or record.trash_entries.get(digest)
            if outside_entry is None:
                raise Exception('digest %s not found' % digest)
            new_entry = Entry(outside_entry.name, original_dir)
            self.commands.append(Command('cp', outside_entry, new_entry))

    def print_commands(self):
        string_commands = [str(command) for command in sorted(self.commands)]
        # sort so that cp comes first.  Need to copy before removals happen
        return '\n'.join(string_commands)

    def run_commands(self):
        return [command.execute() for command in sorted(self.commands)]
<commit_msg>fix mistype<commit_after>"""
argh
"""
from roamer.command import Command
from roamer.record import Record
from roamer.entry import Entry

class Engine(object):
    def __init__(self, original_dir, edit_dir):
        self.commands = []
        for digest, original_entry in original_dir.entries.iteritems():
            new_entries = edit_dir.find(digest)
            if new_entries is None:
                self.commands.append(Command('roamer-trash', original_entry))
                continue
            found_original = False
            for new_entry in new_entries:
                if new_entry.name == original_entry.name:
                    found_original = True
                else:
                    self.commands.append(Command('cp', original_entry, new_entry))
            if not found_original:
                self.commands.append(Command('roamer-trash', original_entry))

        add_blank_entries = edit_dir.find(None)
        if add_blank_entries:
            for entry in add_blank_entries:
                self.commands.append(Command('touch', entry))

        unknown_digests = set(edit_dir.entries.keys()) - set(original_dir.entries.keys())

        for digest in filter(None, unknown_digests):
            record = Record(original_dir)
            outside_entry = record.entries.get(digest) or record.trash_entries.get(digest)
            if outside_entry is None:
                raise Exception('digest %s not found' % digest)
            new_entry = Entry(outside_entry.name, original_dir)
            self.commands.append(Command('cp', outside_entry, new_entry))

    def print_commands(self):
        string_commands = [str(command) for command in sorted(self.commands)]
        # sort so that cp comes first.  Need to copy before removals happen
        return '\n'.join(string_commands)

    def run_commands(self):
        return [command.execute() for command in sorted(self.commands)]
<|endoftext|>


<commit_before>from collections import Counter
import sys

import pygame

from interactive import Button
from life import Cell

# game rules
TO_LIVE = [2, 3]
TO_BE_BORN = [3]

def create_life(life, neighbors):
	new_life = []
    # turn neighbor positions into a list of tuples
	neighbor_dict = Counter(neighbors)
	neighbor_list = neighbor_dict.items()

	life_pos = []
	for cell in life:
		life_pos.append(cell.pos)

	for pos, count in neighbor_list:
        # give birth to cells
		if count in TO_BE_BORN and pos not in life_pos:
			new_life.append(pos)
        # cells staying alive
		if count in TO_LIVE and pos in life_pos:
			new_life.append(pos)

	return new_life

def draw_game(life, screen, height, width):
    # draw game board
    screen.fill((0,0,0))

    # draw pause button
    pygame.draw.rect(screen, (255,0,255), (900,550,100,50))
    myfont = pygame.font.SysFont('monospace', 20)
    label = myfont.render('PAUSE', 1, (255,255,0))
    screen.blit(label, (920, 560))

    # draw cells
    for cell in life:
        cell.draw(screen, height, width)

def draw_pause(screen, height, width):
    pygame.draw.rect(screen, (255,0,255), (900,550,100,50))
    myfont = pygame.font.SysFont('monospace', 20)
    label = myfont.render('RESUME', 1, (255,255,0))
    screen.blit(label, (920, 560))

def draw_title_screen(screen, height, width):
    # draw title screen and background
	screen.fill((50,50,200))
	pygame.draw.rect(screen, (0,0,0,0,), (0,0,width,height), height/9)
	pygame.draw.rect(screen, (200,50,50), (0,0,width,height), height/10)

    # draw title
	myfont = pygame.font.SysFont('monospace', 200)
	label = myfont.render("Conway's", 1, (255,255,0))
	screen.blit(label, (200, 50))
	label = myfont.render('Game of', 1, (255,255,0))
	screen.blit(label, (250, 250))
	label = myfont.render('LIFE', 1, (255,255,0))
	screen.blit(label, (300, 450))

    # draw start button
	pygame.draw.rect(screen, (255,0,255), (900,550,100,50))
	myfont = pygame.font.SysFont('monospace', 20)
	label = myfont.render('START', 1, (255,255,0))
	screen.blit(label, (920, 560))

def event_handler(state):
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
            sys.exit()
        elif event.type == pygame.KEYDOWN:
            if event.key == pygame.K_ESCAPE:
                pygame.quit()
                sys.exit()

        elif state == 'title':
            if event.type == pygame.MOUSEBUTTONUP:
                pos = pygame.mouse.get_pos()
                # check if start is clicked
                if pos[0] > 900 and pos[0] < 1000:
                    if pos[1] > 550 and pos[1] < 600:
                        return 'game'

        elif state == 'game':
            if event.type == pygame.MOUSEBUTTONUP:
                pos = pygame.mouse.get_pos()
                if pos[0] > 900 and pos[0] < 1000:
                    if pos[1] > 550 and pos[1] < 600:
                        return 'pause'

        elif state == 'pause':
            if event.type == pygame.MOUSEBUTTONUP:
                pos = pygame.mouse.get_pos()
                if pos[0] > 900 and pos[0] < 1000:
                    if pos[1] > 550 and pos[1] < 600:
                        return 'game'

            return 'pause'

    return state

def game_actions(old_life):
	life = []

	neighbors = get_neighbors(old_life)
	new_life = create_life(old_life, neighbors)

	for coordinate in new_life:
	    life.append(Cell(coordinate))

	return life

def get_neighbors(life):
	neighbors = []

	for cell in life:
		neighbors.extend(cell.list_neighbors())

	return neighbors
<commit_msg>remove the entire draw title section and move into app/scenes.py TitleScene class<commit_after>from collections import Counter
import sys

import pygame

from interactive import Button
from life import Cell

# game rules
TO_LIVE = [2, 3]
TO_BE_BORN = [3]

def create_life(life, neighbors):
	new_life = []
    # turn neighbor positions into a list of tuples
	neighbor_dict = Counter(neighbors)
	neighbor_list = neighbor_dict.items()

	life_pos = []
	for cell in life:
		life_pos.append(cell.pos)

	for pos, count in neighbor_list:
        # give birth to cells
		if count in TO_BE_BORN and pos not in life_pos:
			new_life.append(pos)
        # cells staying alive
		if count in TO_LIVE and pos in life_pos:
			new_life.append(pos)

	return new_life

def draw_game(life, screen, height, width):
    # draw game board
    screen.fill((0,0,0))

    # draw pause button
    pygame.draw.rect(screen, (255,0,255), (900,550,100,50))
    myfont = pygame.font.SysFont('monospace', 20)
    label = myfont.render('PAUSE', 1, (255,255,0))
    screen.blit(label, (920, 560))

    # draw cells
    for cell in life:
        cell.draw(screen, height, width)

def draw_pause(screen, height, width):
    pygame.draw.rect(screen, (255,0,255), (900,550,100,50))
    myfont = pygame.font.SysFont('monospace', 20)
    label = myfont.render('RESUME', 1, (255,255,0))
    screen.blit(label, (920, 560))

def event_handler(state):
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
            sys.exit()
        elif event.type == pygame.KEYDOWN:
            if event.key == pygame.K_ESCAPE:
                pygame.quit()
                sys.exit()

        elif state == 'title':
            if event.type == pygame.MOUSEBUTTONUP:
                pos = pygame.mouse.get_pos()
                # check if start is clicked
                if pos[0] > 900 and pos[0] < 1000:
                    if pos[1] > 550 and pos[1] < 600:
                        return 'game'

        elif state == 'game':
            if event.type == pygame.MOUSEBUTTONUP:
                pos = pygame.mouse.get_pos()
                if pos[0] > 900 and pos[0] < 1000:
                    if pos[1] > 550 and pos[1] < 600:
                        return 'pause'

        elif state == 'pause':
            if event.type == pygame.MOUSEBUTTONUP:
                pos = pygame.mouse.get_pos()
                if pos[0] > 900 and pos[0] < 1000:
                    if pos[1] > 550 and pos[1] < 600:
                        return 'game'

            return 'pause'

    return state

def game_actions(old_life):
	life = []

	neighbors = get_neighbors(old_life)
	new_life = create_life(old_life, neighbors)

	for coordinate in new_life:
	    life.append(Cell(coordinate))

	return life

def get_neighbors(life):
	neighbors = []

	for cell in life:
		neighbors.extend(cell.list_neighbors())

	return neighbors
<|endoftext|>


<commit_before>import hmac
import base64
import hashlib
import datetime
from context_manager import setlocale
try:
  from urllib.parse import urlparse
except:
  from urlparse import urlparse

class Signature:
  def __init__(self, secret_key):
    self.secret_key = secret_key.encode('ascii')

  def build(self, method, url, body, content_md5=None, content_type=None, httpdate=None):
    method = method.upper()

    if not content_type:
      content_type = ''

    self.content_md5 = content_md5
    if not self.content_md5:
      m = hashlib.md5()
      if not body:
        body = ''
      if isinstance(body, str):
        try:
          m.update(body.encode('ascii'))
        # For Python 2.7
        except(UnicodeDecodeError):
          m.update(body)
      else:
        m.update(body)

      self.content_md5 = base64.b64encode(m.digest()).decode()

    self.httpdate = httpdate
    if not self.httpdate:
      now = datetime.datetime.utcnow()
      with setlocale('en_US.UTF-8'):
        self.httpdate = now.strftime('%a, %d %b %Y %H:%M:%S GMT')

    url = urlparse(url)
    path = url.path
    if url.query:
      path = path + '?' + url.query

    rep = (method, content_type, self.content_md5, path, self.httpdate)
    self.canonical_string = '%s,%s,%s,%s,%s' % rep

    digest = hmac.new(
      self.secret_key,
      self.canonical_string.encode('ascii'),
      hashlib.sha1
    ).digest()
    self.signature = base64.b64encode(digest).rstrip().decode()
<commit_msg>fix(api_auth.signature): proper import of context_manager<commit_after>import hmac
import base64
import hashlib
import datetime

from mifiel.api_auth.context_manager import setlocale

try:
  from urllib.parse import urlparse
except:
  from urlparse import urlparse

class Signature:
  def __init__(self, secret_key):
    self.secret_key = secret_key.encode('ascii')

  def build(self, method, url, body, content_md5=None, content_type=None, httpdate=None):
    method = method.upper()

    if not content_type:
      content_type = ''

    self.content_md5 = content_md5
    if not self.content_md5:
      m = hashlib.md5()
      if not body:
        body = ''
      if isinstance(body, str):
        try:
          m.update(body.encode('ascii'))
        # For Python 2.7
        except(UnicodeDecodeError):
          m.update(body)
      else:
        m.update(body)

      self.content_md5 = base64.b64encode(m.digest()).decode()

    self.httpdate = httpdate
    if not self.httpdate:
      now = datetime.datetime.utcnow()
      with setlocale('en_US.UTF-8'):
        self.httpdate = now.strftime('%a, %d %b %Y %H:%M:%S GMT')

    url = urlparse(url)
    path = url.path
    if url.query:
      path = path + '?' + url.query

    rep = (method, content_type, self.content_md5, path, self.httpdate)
    self.canonical_string = '%s,%s,%s,%s,%s' % rep

    digest = hmac.new(
      self.secret_key,
      self.canonical_string.encode('ascii'),
      hashlib.sha1
    ).digest()
    self.signature = base64.b64encode(digest).rstrip().decode()
<|endoftext|>


<commit_before>"""Master process which detects hardware and launches controllers."""

import collections
import json
import logging
import multiprocessing
import select
import shutil
import socket
import threading
import time
from pathlib import Path

import pyudev
import setproctitle

from .devices import BOARDS

LOGGER = logging.getLogger(__name__)

class Connection:
    """
    A connection to a device.

    This wraps a ``socket.socket`` providing encoding and decoding so that
    consumers of this class can send and receive JSON-compatible typed data
    rather than needing to worry about lower-level details.
    """

    def __init__(self, socket):
        """Wrap the given socket."""
        self.socket = socket
        self.data = b''

    def close(self):
        """Close the connection."""
        self.socket.close()

    def send(self, message):
        """Send the given JSON-compatible message over the connection."""
        line = json.dumps(message).encode('utf-8') + b'\n'
        self.socket.sendall(line)

    def receive(self):
        """Receive a single message from the connection."""
        while b'\n' not in self.data:
            message = self.socket.recv(4096)
            if message == b'':
                return None

            self.data += message
        line = self.data.split(b'\n', 1)[0]
        self.data = self.data[len(line) + 1:]

        return json.loads(line.decode('utf-8'))

class BoardRunner(multiprocessing.Process):
    """Control process for one board."""

    def __init__(self, board, root_dir, **kwargs):
        super().__init__(**kwargs)

        self.board = board
        self.socket_path = (
            Path(root_dir) / type(board).board_type_id / board.name(board.node)
        )

        self._prepare_socket_path()

        self.connections = {}

    def _prepare_socket_path(self):
        try:
            self.socket_path.parent.mkdir(parents=True)
        except FileExistsError:
            if self.socket_path.exists():
                LOGGER.warning('removing old %r', self.socket_path)
                self._delete_socket_path()

    def _delete_socket_path(self):
        try:
            self.socket_path.unlink()
        except FileNotFoundError:
            pass

    def _create_server_socket(self):
        server_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

        server_socket.bind(str(self.socket_path))
        server_socket.listen(5)

        self.socket_path.chmod(0o777)

        LOGGER.info('Listening on:', self.socket_path)

        setproctitle.setproctitle('robotd {}: {}'.format(
            type(self.board).board_type_id,
            type(self.board).name(self.board.node),
        ))

        return server_socket

    def broadcast(self, message):
        """Broadcast a message over all connections."""
        message = dict(message)
        message['broadcast'] = True

        for connection in self.connections.values():
            try:
                connection.send(message)
            except ConnectionRefusedError:
                self.connections.remove(connection)

    def _send_board_status(self, connection):
        board_status = self.board.status()
        LOGGER.debug('Sending board status:', board_status)
        connection.send(board_status)

    def _send_command_response(self, connection, response):
        message = {'response': response}
        LOGGER.debug('Sending command response:', message)
        connection.send(message)

    def run(self):
        """
        Control this board.

        This is the entry point from the control subprocess and its job is to:

        * Create and manage the UNIX socket in `/var/robotd`,
        * Pass on commands to the `board`,
        * Call `make_safe` whenever the last user disconnects,
        * Deal with error handling and shutdown.
        """

        server_socket = self._create_server_socket()

        self.board.broadcast = self.broadcast
        self.board.start()

        try:
            while True:
                self._process_connections(server_socket)
        finally:
            self.board.make_safe()

    def _process_connections(self, server_socket):
        connection_sockets = list(self.connections.keys())

        # Wait until one of the sockets is ready to read.
        readable, _, errorable = select.select(
            # connections that want to read
            [server_socket] + connection_sockets,
            # connections that want to write
            [],
            # connections that want to error
            connection_sockets,
        )

        # New connections
        if server_socket in readable:
            new_socket, _ = server_socket.accept()
            new_connection = Connection(new_socket)
            readable.append(new_socket)
            self.connections[new_socket] = new_connection
            LOGGER.info('New connection at:', self.socket_path)
            self._send_board_status(new_connection)

        dead_sockets = []

        for sock in readable:
            try:
                connection = self.connections[sock]
            except KeyError:
                continue

            command = connection.receive()

            if command is None:
                dead_sockets.append(sock)
                continue

            if command != {}:
                response = self.board.command(command)
                if response is not None:
                    self._send_command_response(connection, response)

            self._send_board_status(connection)

        dead_sockets.extend(errorable)

        self._close_dead_sockets(dead_sockets)

        if dead_sockets and not self.connections:
            LOGGER.info('Last connection closed')
            self.board.make_safe()

    def _close_dead_sockets(self, dead_sockets):
        for sock in dead_sockets:
            try:
                del self.connections[sock]
            except KeyError:
                pass
            sock.close()

    def cleanup(self):
        """
        Clean up the UNIX socket if it's been left around.

        Called from the parent process.
        """

        self._delete_socket_path()
        self.board.stop()

class MasterProcess(object):
    """The mighty God object which manages the controllers."""

    def __init__(self, root_dir):
        self.runners = collections.defaultdict(dict)
        self.context = pyudev.Context()
        self.root_dir = Path(root_dir)

        self.root_dir.mkdir(mode=0o755, parents=True, exist_ok=True)
        self.clear_socket_files()

        self.runners_lock = threading.Lock()

        # Init the startup boards
        for board_type in BOARDS:
            if board_type.create_on_startup:
                self._start_board_instance(board_type, 'startup')

    def clear_socket_files(self):
        for path in self.root_dir.iterdir():
            shutil.rmtree(str(path))

    def tick(self):
        """Poll udev for any new or missing boards."""
        for board_type in BOARDS:
            if hasattr(board_type, 'lookup_keys'):
                nodes = self.context.list_devices(**board_type.lookup_keys)
                initialized_nodes = [n for n in nodes if n.is_initialized]
                self._process_device_list(board_type, initialized_nodes)

    def cleanup(self):
        """Shut down all the controllers."""
        for board_type in BOARDS:
            self._process_device_list(board_type, [])
        self.stop_monitor()

    def _process_device_list(self, board_type, nodes):
        with self.runners_lock:
            nodes_by_path = {
                x.device_path: x
                for x in nodes
                if board_type.included(x)
            }

            actual_paths = set(nodes_by_path.keys())
            expected_paths = set(self.runners[board_type].keys())

            missing_paths = expected_paths - actual_paths
            new_paths = actual_paths - expected_paths

            for new_device in new_paths:
                LOGGER.info(
                    'Detected new %s: %s (%s)',
                    board_type.__name__,
                    new_device,
                    board_type.name(nodes_by_path[new_device]),
                )
                self._start_board_instance(
                    board_type,
                    new_device,
                    node=nodes_by_path[new_device],
                )

            for dead_device in missing_paths:
                LOGGER.info('Disconnected %s: %s', board_type.__name__, dead_device)
                runner = self.runners[board_type][dead_device]
                runner.terminate()
                runner.join()
                runner.cleanup()
                del self.runners[board_type][dead_device]

    def _start_board_instance(self, board_type, new_device, **kwargs):
        instance = board_type(**kwargs)
        runner = BoardRunner(instance, self.root_dir)
        runner.start()
        self.runners[board_type][new_device] = runner

    def launch_monitor(self):
        self.monitor_stop_flag = False
        self.monitor_thread = threading.Thread(target=self._monitor_thread)
        self.monitor_thread.start()

    def stop_monitor(self):
        self.monitor_stop_flag = True
        self.monitor_thread.join()

    def _monitor_thread(self):
        while True:
            if self.monitor_stop_flag:
                return

            time.sleep(0.5)

            with self.runners_lock:
                for board_type, runners in list(self.runners.items()):
                    for device_id, runner_process in list(runners.items()):
                        if not runner_process.is_alive():
                            LOGGER.info('Dead worker: %s(%s)', board_type, device_id)
                            # This worker has died and needs to be reaped
                            del self.runners[board_type][device_id]

def main(**kwargs):
    """Main entry point."""
    master = MasterProcess(**kwargs)

    setproctitle.setproctitle('robotd master')

    master.launch_monitor()
    try:
        while True:
            master.tick()
            time.sleep(1)
    except KeyboardInterrupt:
        master.cleanup()

def main_cmdline():
    """Command line entry point."""
    import argparse
    parser = argparse.ArgumentParser()

    default_root_dir = Path('/var/robotd')
    parser.add_argument(
        '--root-dir',
        type=Path,
        help='directory to run root of robotd at (defaults to {})'.format(
            default_root_dir,
        ),
        default=default_root_dir,
    )
    args = parser.parse_args()

    main(root_dir=args.root_dir)

if __name__ == '__main__':
    main_cmdline()
<commit_msg>Add overlooked placeholders<commit_after>"""Master process which detects hardware and launches controllers."""

import collections
import json
import logging
import multiprocessing
import select
import shutil
import socket
import threading
import time
from pathlib import Path

import pyudev
import setproctitle

from .devices import BOARDS

LOGGER = logging.getLogger(__name__)

class Connection:
    """
    A connection to a device.

    This wraps a ``socket.socket`` providing encoding and decoding so that
    consumers of this class can send and receive JSON-compatible typed data
    rather than needing to worry about lower-level details.
    """

    def __init__(self, socket):
        """Wrap the given socket."""
        self.socket = socket
        self.data = b''

    def close(self):
        """Close the connection."""
        self.socket.close()

    def send(self, message):
        """Send the given JSON-compatible message over the connection."""
        line = json.dumps(message).encode('utf-8') + b'\n'
        self.socket.sendall(line)

    def receive(self):
        """Receive a single message from the connection."""
        while b'\n' not in self.data:
            message = self.socket.recv(4096)
            if message == b'':
                return None

            self.data += message
        line = self.data.split(b'\n', 1)[0]
        self.data = self.data[len(line) + 1:]

        return json.loads(line.decode('utf-8'))

class BoardRunner(multiprocessing.Process):
    """Control process for one board."""

    def __init__(self, board, root_dir, **kwargs):
        super().__init__(**kwargs)

        self.board = board
        self.socket_path = (
            Path(root_dir) / type(board).board_type_id / board.name(board.node)
        )

        self._prepare_socket_path()

        self.connections = {}

    def _prepare_socket_path(self):
        try:
            self.socket_path.parent.mkdir(parents=True)
        except FileExistsError:
            if self.socket_path.exists():
                LOGGER.warning('removing old %r', self.socket_path)
                self._delete_socket_path()

    def _delete_socket_path(self):
        try:
            self.socket_path.unlink()
        except FileNotFoundError:
            pass

    def _create_server_socket(self):
        server_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

        server_socket.bind(str(self.socket_path))
        server_socket.listen(5)

        self.socket_path.chmod(0o777)

        LOGGER.info('Listening on: %s', self.socket_path)

        setproctitle.setproctitle('robotd {}: {}'.format(
            type(self.board).board_type_id,
            type(self.board).name(self.board.node),
        ))

        return server_socket

    def broadcast(self, message):
        """Broadcast a message over all connections."""
        message = dict(message)
        message['broadcast'] = True

        for connection in self.connections.values():
            try:
                connection.send(message)
            except ConnectionRefusedError:
                self.connections.remove(connection)

    def _send_board_status(self, connection):
        board_status = self.board.status()
        LOGGER.debug('Sending board status: %s', board_status)
        connection.send(board_status)

    def _send_command_response(self, connection, response):
        message = {'response': response}
        LOGGER.debug('Sending command response: %s', message)
        connection.send(message)

    def run(self):
        """
        Control this board.

        This is the entry point from the control subprocess and its job is to:

        * Create and manage the UNIX socket in `/var/robotd`,
        * Pass on commands to the `board`,
        * Call `make_safe` whenever the last user disconnects,
        * Deal with error handling and shutdown.
        """

        server_socket = self._create_server_socket()

        self.board.broadcast = self.broadcast
        self.board.start()

        try:
            while True:
                self._process_connections(server_socket)
        finally:
            self.board.make_safe()

    def _process_connections(self, server_socket):
        connection_sockets = list(self.connections.keys())

        # Wait until one of the sockets is ready to read.
        readable, _, errorable = select.select(
            # connections that want to read
            [server_socket] + connection_sockets,
            # connections that want to write
            [],
            # connections that want to error
            connection_sockets,
        )

        # New connections
        if server_socket in readable:
            new_socket, _ = server_socket.accept()
            new_connection = Connection(new_socket)
            readable.append(new_socket)
            self.connections[new_socket] = new_connection
            LOGGER.info('New connection at: %s', self.socket_path)
            self._send_board_status(new_connection)

        dead_sockets = []

        for sock in readable:
            try:
                connection = self.connections[sock]
            except KeyError:
                continue

            command = connection.receive()

            if command is None:
                dead_sockets.append(sock)
                continue

            if command != {}:
                response = self.board.command(command)
                if response is not None:
                    self._send_command_response(connection, response)

            self._send_board_status(connection)

        dead_sockets.extend(errorable)

        self._close_dead_sockets(dead_sockets)

        if dead_sockets and not self.connections:
            LOGGER.info('Last connection closed')
            self.board.make_safe()

    def _close_dead_sockets(self, dead_sockets):
        for sock in dead_sockets:
            try:
                del self.connections[sock]
            except KeyError:
                pass
            sock.close()

    def cleanup(self):
        """
        Clean up the UNIX socket if it's been left around.

        Called from the parent process.
        """

        self._delete_socket_path()
        self.board.stop()

class MasterProcess(object):
    """The mighty God object which manages the controllers."""

    def __init__(self, root_dir):
        self.runners = collections.defaultdict(dict)
        self.context = pyudev.Context()
        self.root_dir = Path(root_dir)

        self.root_dir.mkdir(mode=0o755, parents=True, exist_ok=True)
        self.clear_socket_files()

        self.runners_lock = threading.Lock()

        # Init the startup boards
        for board_type in BOARDS:
            if board_type.create_on_startup:
                self._start_board_instance(board_type, 'startup')

    def clear_socket_files(self):
        for path in self.root_dir.iterdir():
            shutil.rmtree(str(path))

    def tick(self):
        """Poll udev for any new or missing boards."""
        for board_type in BOARDS:
            if hasattr(board_type, 'lookup_keys'):
                nodes = self.context.list_devices(**board_type.lookup_keys)
                initialized_nodes = [n for n in nodes if n.is_initialized]
                self._process_device_list(board_type, initialized_nodes)

    def cleanup(self):
        """Shut down all the controllers."""
        for board_type in BOARDS:
            self._process_device_list(board_type, [])
        self.stop_monitor()

    def _process_device_list(self, board_type, nodes):
        with self.runners_lock:
            nodes_by_path = {
                x.device_path: x
                for x in nodes
                if board_type.included(x)
            }

            actual_paths = set(nodes_by_path.keys())
            expected_paths = set(self.runners[board_type].keys())

            missing_paths = expected_paths - actual_paths
            new_paths = actual_paths - expected_paths

            for new_device in new_paths:
                LOGGER.info(
                    'Detected new %s: %s (%s)',
                    board_type.__name__,
                    new_device,
                    board_type.name(nodes_by_path[new_device]),
                )
                self._start_board_instance(
                    board_type,
                    new_device,
                    node=nodes_by_path[new_device],
                )

            for dead_device in missing_paths:
                LOGGER.info('Disconnected %s: %s', board_type.__name__, dead_device)
                runner = self.runners[board_type][dead_device]
                runner.terminate()
                runner.join()
                runner.cleanup()
                del self.runners[board_type][dead_device]

    def _start_board_instance(self, board_type, new_device, **kwargs):
        instance = board_type(**kwargs)
        runner = BoardRunner(instance, self.root_dir)
        runner.start()
        self.runners[board_type][new_device] = runner

    def launch_monitor(self):
        self.monitor_stop_flag = False
        self.monitor_thread = threading.Thread(target=self._monitor_thread)
        self.monitor_thread.start()

    def stop_monitor(self):
        self.monitor_stop_flag = True
        self.monitor_thread.join()

    def _monitor_thread(self):
        while True:
            if self.monitor_stop_flag:
                return

            time.sleep(0.5)

            with self.runners_lock:
                for board_type, runners in list(self.runners.items()):
                    for device_id, runner_process in list(runners.items()):
                        if not runner_process.is_alive():
                            LOGGER.info('Dead worker: %s(%s)', board_type, device_id)
                            # This worker has died and needs to be reaped
                            del self.runners[board_type][device_id]

def main(**kwargs):
    """Main entry point."""
    master = MasterProcess(**kwargs)

    setproctitle.setproctitle('robotd master')

    master.launch_monitor()
    try:
        while True:
            master.tick()
            time.sleep(1)
    except KeyboardInterrupt:
        master.cleanup()

def main_cmdline():
    """Command line entry point."""
    import argparse
    parser = argparse.ArgumentParser()

    default_root_dir = Path('/var/robotd')
    parser.add_argument(
        '--root-dir',
        type=Path,
        help='directory to run root of robotd at (defaults to {})'.format(
            default_root_dir,
        ),
        default=default_root_dir,
    )
    args = parser.parse_args()

    main(root_dir=args.root_dir)

if __name__ == '__main__':
    main_cmdline()
<|endoftext|>


<commit_before>import os,re,sys,subprocess,argparse,json
from sqlalchemy import *
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.schema import DDLElement
from connection import *

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description="""
This script adds Tile entries for a list of Image idImage values.  The
resultant tiles are associated with no Map entries

There is only one Image per Tile, but there often needs to be multiple Tile
entries for a given Image, so that a Tile containing the Image can be
associated with multiple Map entries or multiple places in the same Map.

* A new Tile entry will only be created if one does not exist for an image.
* the --force,-f option overides this and created new Tile enrties regardless

""")
    parser.add_argument('--force','-f',action='store_true',
        help='force creation of new Tile entries')
    parser.add_argument('--tName','-n',action='store',
        help='Tile name  (default is to take if from the image name)')
    parser.add_argument('--tRotate','-r',type=float,action='store',
        default=0,
        help='Tile rotate  (default is 0)')
    parser.add_argument('--tScale','-s',type=float,action='store',
        default=1,
        help='Tile scale  (default is 1)')
    parser.add_argument('--tX','-x',type=float,action='store',
        help='Tile translateX  (default is 0+10*imageN)')
    parser.add_argument('--tY','-y',type=float,action='store',
        help='Tile translateY  (default is 0+10*imageN)')
    parser.add_argument('--tDepth','-d',type=float,action='store',
        help='Tile depth (default is 10+imageN)')
    parser.add_argument('--tBackColor','-b',action='store',
        help='Tile backgroundColor  (default is null)')
    parser.add_argument('--tLink','-l',action='store',
        help='Tile ruleLink  (default is null)')
    parser.add_argument('--tMap','-m',type=int,action='store',
        help='Tile idMap  (default is null)')
    parser.add_argument('--json','-j',type=str,action='store',
        help='JSON file to assign depth and offset based on match with name and size')
    parser.add_argument('idImage', type=str, nargs="+",
        help='Image idImage numbers to be Tiles')
    # parse the command line arguments and options
    args = parser.parse_args()
    if (not args.tBackColor): args.tBackColor = null()
    if (not args.tLink): args.tLink = null()
    # if an idMap was provided check that it exists and get the name
    if (args.tMap):
        rMap = tMap.select(tMap.c.idMap==args.tMap).execute()
        if rMap.rowcount == 1:
            eMap = rMap.fetchone()
            eMapIdMap = eMap.idMap
        else:
            args.tMap = None
    # if an idMap was not provided or did not exist set the null values
    if (not args.tMap):
        eMapIdMap = null()
        args.tMap = null()
    # if a JSON file is provided, extract the layer information
    #  This will be used to assign depth and offset so long
    #  as the name matches
    params = {}
    if (args.json and os.path.isfile(args.json)):
        jsonfile = file(args.json,'r')
        params = json.load(jsonfile)

    eTiles = [];
    # If we are given a range of idImages use them all
    for i in args.idImage:
        if (type(i) == str and '..' in i):
            st,ed = re.split('\.\.',i)
            c = args.idImage.index(i) + 1
            args.idImage.insert(c,int(ed))
            for r in range(int(st),int(ed)):
                args.idImage.insert(c,r)
                c += 1
            args.idImage.remove(i)

    #print args
    for idImage in args.idImage:
        idImage = int(idImage)
        eImage = None;
        eTile = None;
        #print idImage
        rImage = tImage.select(tImage.c.idImage==idImage).execute()
        # Is there an image with the given Id?  If not, skip the entry.
        if rImage.rowcount == 1:
            eImage = rImage.fetchone()
            # set special default for tName based on Image filename
            tName = args.tName or os.path.splitext(eImage.filename)[0]
            # set special default for tX based on current Tile count
            tX = args.tX or 0.0+10.0*len(eTiles)
            # set special default for tY based on current Tile count
            tY = args.tY or 0.0+10.0*len(eTiles)
            # set special default for tDepth based on current Tile count
            tDepth = args.tDepth or 10.0+len(eTiles)
            # set visibility
            tVisible=1
            # if there is a JSON entry for this
            #  tile, use it instead
            if (params.has_key(tName)):
                tX = params[tName]['offsets'][0]
                tY = params[tName]['offsets'][1]
                tDepth = 10 + params[tName]['id']
                tVisible = params[tName]['visible']
            #print eImage.filename
            rTile = tTile.select(tTile.c.idImage==eImage.idImage).execute()
            # Is there already a Tile for this Image?  If so,
            #  do not make a new one unless the force option is true
            if rTile.rowcount < 1 or args.force:
                riTile = tTile.insert().values(name=tName,
                                               rotate=args.tRotate,
                                               scale=args.tScale,
                                               translateX=tX,
                                               translateY=tY,
                                               visible=tVisible,
                                               dmVisible=1,
                                               depth=tDepth,
                                               backgroundColor=args.tBackColor,
                                               ruleLink=args.tLink,
                                               idImage=eImage.idImage,
                                               idMap=eMapIdMap).execute()
                newIdTile = riTile.inserted_primary_key[0]
                eTile = tTile.select(tTile.c.idTile==newIdTile).execute().fetchone()
            else:
                eTile = rTile.fetchone()
            # append the Tile entry to my running list
            eTiles.append(eTile)
            # get the Map Name if the Map entry exists
            rMap = tMap.select(tMap.c.idMap==eTile.idMap).execute()
            if rMap.rowcount == 1:
                eMap = rMap.fetchone()
                eMapName = eMap.name
            else:
                eMapName = "None"
            # output the results
            print "Map `%s`(%s) has Tile `%s`(%d) of Image `%s`(%d)" % (
                        eMapName,
                        eTile.idMap,
                        eTile.name,
                        eTile.idTile,
                        eImage.filename,
                        eImage.idImage)
<commit_msg>modify addTiles.py for JSON option<commit_after>import os,re,sys,subprocess,argparse,json
from sqlalchemy import *
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.schema import DDLElement
from connection import *

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description="""
This script adds Tile entries for a list of Image idImage values.  The
resultant tiles are associated with no Map entries

There is only one Image per Tile, but there often needs to be multiple Tile
entries for a given Image, so that a Tile containing the Image can be
associated with multiple Map entries or multiple places in the same Map.

* A new Tile entry will only be created if one does not exist for an image.
* the --force,-f option overides this and created new Tile enrties regardless

""")
    parser.add_argument('--force','-f',action='store_true',
        help='force creation of new Tile entries')
    parser.add_argument('--tName','-n',action='store',
        help='Tile name  (default is to take if from the image name)')
    parser.add_argument('--tRotate','-r',type=float,action='store',
        default=0,
        help='Tile rotate  (default is 0)')
    parser.add_argument('--tScale','-s',type=float,action='store',
        default=1,
        help='Tile scale  (default is 1)')
    parser.add_argument('--tX','-x',type=float,action='store',
        help='Tile translateX  (default is 0+10*imageN)')
    parser.add_argument('--tY','-y',type=float,action='store',
        help='Tile translateY  (default is 0+10*imageN)')
    parser.add_argument('--tDepth','-d',type=float,action='store',
        help='Tile depth (default is 10+imageN)')
    parser.add_argument('--tBackColor','-b',action='store',
        help='Tile backgroundColor  (default is null)')
    parser.add_argument('--tLink','-l',action='store',
        help='Tile ruleLink  (default is null)')
    parser.add_argument('--tMap','-m',type=int,action='store',
        help='Tile idMap  (default is null)')
    parser.add_argument('--json','-j',type=str,action='store',
        help='JSON file to assign depth and offset based on match with name and size')
    parser.add_argument('idImage', type=str, nargs="+",
        help='Image idImage numbers to be Tiles')
    # parse the command line arguments and options
    args = parser.parse_args()
    if (not args.tBackColor): args.tBackColor = null()
    if (not args.tLink): args.tLink = null()
    # if an idMap was provided check that it exists and get the name
    if (args.tMap):
        rMap = tMap.select(tMap.c.idMap==args.tMap).execute()
        if rMap.rowcount == 1:
            eMap = rMap.fetchone()
            eMapIdMap = eMap.idMap
        else:
            args.tMap = None
    # if an idMap was not provided or did not exist set the null values
    if (not args.tMap):
        eMapIdMap = null()
        args.tMap = null()
    # if a JSON file is provided, extract the layer information
    #  This will be used to assign depth and offset so long
    #  as the name matches
    params = {}
    if (args.json and os.path.isfile(args.json)):
        jsonfile = file(args.json,'r')
        params = json.load(jsonfile)
        # Let's set ids based on where they are in the file
        lnum = 10
        for layer in params["layer"]:
            layer["name"]
            lnum += 2
            params[layer['name']]['id'] = lnum

    eTiles = [];
    # If we are given a range of idImages use them all
    for i in args.idImage:
        if (type(i) == str and '..' in i):
            st,ed = re.split('\.\.',i)
            c = args.idImage.index(i) + 1
            args.idImage.insert(c,int(ed))
            for r in range(int(st),int(ed)):
                args.idImage.insert(c,r)
                c += 1
            args.idImage.remove(i)

    #print args
    for idImage in args.idImage:
        idImage = int(idImage)
        eImage = None;
        eTile = None;
        #print idImage
        rImage = tImage.select(tImage.c.idImage==idImage).execute()
        # Is there an image with the given Id?  If not, skip the entry.
        if rImage.rowcount == 1:
            eImage = rImage.fetchone()
            # set special default for tName based on Image filename
            tName = args.tName or os.path.splitext(eImage.filename)[0]
            # set special default for tX based on current Tile count
            tX = args.tX or 0.0+10.0*len(eTiles)
            # set special default for tY based on current Tile count
            tY = args.tY or 0.0+10.0*len(eTiles)
            # set special default for tDepth based on current Tile count
            tDepth = args.tDepth or 10.0+len(eTiles)
            # set visibility
            tVisible=1
            # if there is a JSON entry for this
            #  tile, use it instead
            if (params.has_key(tName)):
                tX = params[tName]['offsets'][0]
                tY = params[tName]['offsets'][1]
                tDepth = 10 + params[tName]['id']
                tVisible = params[tName]['visible']
            #print eImage.filename
            rTile = tTile.select(tTile.c.idImage==eImage.idImage).execute()
            # Is there already a Tile for this Image?  If so,
            #  do not make a new one unless the force option is true
            if rTile.rowcount < 1 or args.force:
                riTile = tTile.insert().values(name=tName,
                                               rotate=args.tRotate,
                                               scale=args.tScale,
                                               translateX=tX,
                                               translateY=tY,
                                               visible=tVisible,
                                               dmVisible=1,
                                               depth=tDepth,
                                               backgroundColor=args.tBackColor,
                                               ruleLink=args.tLink,
                                               idImage=eImage.idImage,
                                               idMap=eMapIdMap).execute()
                newIdTile = riTile.inserted_primary_key[0]
                eTile = tTile.select(tTile.c.idTile==newIdTile).execute().fetchone()
            else:
                eTile = rTile.fetchone()
            # append the Tile entry to my running list
            eTiles.append(eTile)
            # get the Map Name if the Map entry exists
            rMap = tMap.select(tMap.c.idMap==eTile.idMap).execute()
            if rMap.rowcount == 1:
                eMap = rMap.fetchone()
                eMapName = eMap.name
            else:
                eMapName = "None"
            # output the results
            print "Map `%s`(%s) has Tile `%s`(%d) of Image `%s`(%d)" % (
                        eMapName,
                        eTile.idMap,
                        eTile.name,
                        eTile.idTile,
                        eImage.filename,
                        eImage.idImage)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Option persistence."""
import re
import collections
import numpy as np

from .. import logger
from conf import Conf
import cPickle as pickle
from ..milang import Scriptable
import common_proxy
from .option import ao

def dictRecursiveModel(base):
    """BUild a dictionary configuration tree from ConfigurationProxy `base`"""
    out = collections.OrderedDict()
    for path, obj in base.iteritems():
        if path == 'self':
            out[path] = obj['name']
            continue
        out[path] = dictRecursiveModel(obj)
    return out

def print_tree(tree, level=0):
    """Pretty-print a dictionary configuration `tree`"""
    pre = '   ' * level
    msg = ''
    for k, v in tree.iteritems():
        if k == 'self':
            msg += print_tree(v, level)
            continue
        # Detect subdevice
        if isinstance(v, dict) and 'self' in v:
            msg += pre + '|++> ' + k + '\n'
            msg += print_tree(v, level + 1)
            continue
        v = repr(v['current'])
        if len(v) > 50:
            v = v[:46] + ' ...'
        msg += '{}|: {} = {}\n'.format(pre, k, v)
    return msg

# TODO: parametrize in user conf
# 0=always visible; 1=user ; 2=expert ; 3=advanced ; 4=technician ;
# 5=developer; 6=never visible
class ConfigurationProxy(Scriptable, Conf):

    """A configuration object behaving like a live server"""
    separator = '/'
    _readLevel = 5
    _writeLevel = 5
    _rmodel = False

    def print_tree(self):
        print print_tree(self.tree())

    def __init__(self, desc=collections.OrderedDict({'self': {}}),
                 name='MAINSERVER', parent=False, readLevel=2, writeLevel=5, kid_base='/'):
        Scriptable.__init__(self)
        self.log = logger.BaseLogger()
        self.kid_base = kid_base
        Conf.__init__(self, desc['self'])
        self._readLevel = readLevel
        self._writeLevel = writeLevel
        self.children = desc.copy()
        """Child configuration dictionaries"""
        self.children_obj = {}
        """Instantiated children configuration proxies"""
        # Remove myself from children...
        del self.children['self']
        self.name = name
        self._parent = parent
        if not parent:
            self._Method__name = name
        else:
            self._Method__name = parent._Method__name + self.separator + name
#           if parent._parent:
#               self._Method__name=parent._Method__name+self.separator+name
#           else:
#               self._Method__name=name
        if self.has_key('devpath'):
            self['devpath'] = name
        self.autosort()

    def get(self, *a, **k):
        return self.__getitem__(*a, **k)

    def set(self, *a, **k):
        return self.__setitem__(*a, **k)

    @property
    def root(self):
        if not self._parent:
            return self
        return self._parent.root

    @property
    def naturalName(self):
        return self.name

    def get_fullpath(self):
        po = self
        path = []
        while True:
            if po.name == 'MAINSERVER':
                break
            path.append(po.name)
            po = po.parent()
            if po in [None, False]:
                break
        # Revert to get the top-down path
        path.reverse()
        return '/' + '/'.join(path) + '/'

    def _update_from_children(self):
        """ Retrieve current configuration from instantiated child objects"""
        for key, obj in self.children_obj.iteritems():
            d = {'self': obj.desc}
            obj._update_from_children()
            d.update(obj.children)
            self.children[key] = d
        self.autosort()

    def rmodel(self):
        out = dictRecursiveModel(self.children)
        out['self'] = self.name
        return out

    def tree(self):
        """Build a full configuration tree suitable for serialization and saving on a file"""
        self._update_from_children()
        tree = self.children.copy()
        tree['self'] = self.desc.copy()
        return tree

    def __len__(self):
        return len(self.desc)

    def iteritems(self):
        return self.desc.iteritems()

    def itervalues(self):
        return self.desc.itervalues()

    def values(self):
        return self.desc.values()

    def iterkeys(self):
        return self.desc.iterkeys()

    def has_key(self, k):
        return self.desc.has_key(k)

    def dumps(self):
        d = {'self': self.desc.copy()}
        self._update_from_children()
        d.update(self.children)
        return pickle.dumps(d)

    def describe(self, *a, **kw):
        return self.desc

    def connect(self, *a, **k):
        pass

    def connection(self, *a, **k):
        return True

    def paste(self, obj):
        self.desc = obj.desc.copy()
        self.children = obj.children.copy()
        self.children_obj = obj.children_obj
        self._parent = obj._parent
        self._Method__name = obj._Method__name
        self._readLevel = obj._readLevel
        self._writeLevel = obj._writeLevel

    def copy(self):
        p = ConfigurationProxy()
        p.paste(self)
        return p

    def __nonzero__(self):
        return 1

    def __getitem__(self, key, *a):
        if key == 'fullpath':
            return self.get_fullpath()
        if len(a)==1 and not self.desc.has_key(key):
            return a[0]
        return self.desc[key]['current']

    def __setitem__(self, key, val):
        if not self.desc.has_key(key):
            print 'Impossible to set key', key, val
            return False
        self.desc[key]['current'] = val
        return True

    def gettype(self, key):
        return self.desc[key]['type']

    def multiget(self, keys):
        r = {}
        for k in keys:
            r[k] = self[k]
        return r

    def __getattr__(self, path):
        if path.startswith('_'):
            return object.__getattribute__(self, path)
        elif path in dir(self):
            return object.__getattribute__(self, path)
        return self.child(path)

    def gete(self, key):
        return self.desc[key]

    def sete(self, key, val):
        if not val.has_key('priority') or val['priority']==-1:
            priorities = [opt.get('priority',0) for opt in self.desc.values()]
            if len(priorities):
                val['priority'] = max(priorities) + 1
        self.desc[key] = val

    def add_option(self, *args, **kwargs):
        """Creates a new option using the ao() utility function.
        Migrate old one if existing."""
        out = {}
        overwrite = True
        if kwargs.has_key('overwrite'):
            overwrite = kwargs.pop('overwrite')
        ao(out, *args, **kwargs)
        out = out.values()[0]
        key = out['handle']
        if out['priority'] == 0:
            out['priority'] = -1
        # If option was already defined, update old one with new values
        if self.has_key(key):
            # Do not do anything if not overwriting
            if not overwrite:
                return out
            origin = self.gete(key).entry
            origin.update(out)
            out = origin

        self.sete(out['handle'], out)
        return out

    def autosort(self):
        def sorter(item):
            key, val = item
            digits = re.sub(r"\D", '', key)
            key = int(digits) if len(digits) else key
            return key
        self.children = collections.OrderedDict(sorted(self.children.items(), key=sorter))

    def add_child(self, name, desc, overwrite=False):
        """Inserts a sub-object `name` with object tree dictionary `desc`.
        Returns a ConfigurationProxy to the new child."""
        # If desc is the description dictionary and not the object tree dictionary,
        # encapsulate it in a 'self' dict
        if not 'self' in desc:
            desc = {'self': desc}
        if overwrite or name not in self.children:
            self.children[name] = desc
        else:
            for key, val in desc['self'].iteritems():
                self.children[name]['self'][key] = val
        self.autosort()
        return self.child(name)

    def getFlags(self, opt):
        if not self.desc[opt].has_key('flags'):
            return {}
        return self.desc[opt]['flags']

    def list(self):
        return [(c, c) for c in self.children.iterkeys()]

    def has_child(self, name):
        """Returns if `name` is a child"""
        return self.children.has_key(name)

    def child(self, name):
        """Return child ConfigurationProxy object by `name`"""
        if not self.children.has_key(name):
            return None
        if not self.children_obj.has_key(name):
            kb = self.kid_base + name + self.separator
            self.children_obj[name] = ConfigurationProxy(
                self.children[name], name=name, parent=self, kid_base=kb)
        return self.children_obj[name]

    def calc_aggregate(self, aggregation, handle=False):
        function_name = re.search("(.+?)\(", aggregation).group(1)
        targets = re.search("\((.+?)\)", aggregation)
        if targets is None:
            targets = [handle]
        else:
            targets = targets.group(1).replace(' ','').split(',')
        values = collections.defaultdict(list)
        for child_name in self.children.iterkeys():
            child = self.child(child_name)
            pack = collections.defaultdict(list)
            for target in targets:
                # Ensure all targets exist
                if not child.has_key(target):
                    pack = False
                    break
                pack[target].append(child[target])
            if pack:
                for t in targets:
                    values[t]+=pack[t]
        result = None
        error = None
        #TODO: calc stdev here
        if function_name == 'mean':
            v = np.array(values[targets[0]]).astype(np.float32)
            #FIXME: hack to filter out zeros
            v = v[v!=0]
            if len(v):
                result = float(v.mean())
                error = float(v.std())
        elif function_name == 'sum':
            result = float(np.array(values[targets[0]]).astype(np.float32).sum())
        elif function_name == 'prod':
            result = float(np.array(values[targets[0]]).astype(np.float32).prod())
        elif function_name == 'table':
            result = []
            for i, x in enumerate(values[targets[0]]):
                row = []
                for t in targets:
                    row.append(values[t][i])
                result.append(row)
            # Reorder by first column
            result = sorted(result, key=lambda e: e[0])
            # Prepend table header
            # Should calculate also table header?
            result = [self[handle][0]] + result
        else:
            self.log.error('Aggregate function not found:', function_name, aggregation)
        return result, error

    def update_aggregates(self, recursive=1):
        """Updates aggregate options. recursive==1, upward; -1, downward"""
        for handle, opt in self.desc.iteritems():
            if not opt.has_key('aggregate'):
                continue
            aggregation = opt['aggregate']
            result, error = self.calc_aggregate(aggregation, handle)
            if result is not None:
                self[handle] = result
                if error is not None and opt.has_key('error'):
                    self[opt['error']] = error
            else:
                self.log.error('Aggregation failed for ', handle, aggregation)
        if recursive>0 and self.parent():
            self.parent().update_aggregates(recursive=True)
        elif recursive<0:
            for k in self.children.keys():
                self.child(k).update_aggregates(recursive=-1)

    @property
    def devices(self):
        return [self.child(name) for name in self.children.keys()]

    def parent(self):
        return self._parent

    def toPath(self, lst):
        """Returns a copy of the object at the path expressed in list lst"""
        if isinstance(lst, str):
            lst = lst.split(self.separator)
            if lst[0] in ['server', '']:
                lst.pop(0)
            if lst[-1] == '':
                lst.pop(-1)
        obj = self.copy()
        for p in lst:
            if obj is None:
                return None
            obj = obj.child(p)
        return obj

    def from_column(self, col0):
        return common_proxy.from_column(col0, self.root)

    def toMethodName(self, name):
        if name == 'MAINSERVER':
            return
        lst = name.split(self.separator)
        return self.toPath(lst)

    def searchPath(self, path):
        """Search a child with the corresponding devpath=`path` and returns its name"""
        fp = self.get_fullpath().replace('//', '/')
        if not path.startswith(fp):
            print 'searchPath: not corresponding!', path, fp
            return False
        # Cut itself from the path
        if not path.endswith('/'):
            path += '/'
        vpath = path[len(fp):].split('/')[:-1]

        # Verify existence
        obj = self
        for p in vpath:
            obj = obj.child(p)
            if obj is None:
                print 'searchPath: not found', repr(p), repr(path)
        return self.separator.join(vpath)

    def listPresets(self):
        return ['default']

    def iterprint(self, base=False, pre=''):
        """Iterative printing for debug purposes"""
        if not base:
            base = self
        for k, v in base.iteritems():
            print pre, k, v['current']
        for sub in base.children.iterkeys():
            base.child(sub).iterprint(pre=pre + '\t%s: ' % sub)

    @property
    def samples(self):
        if not self.measure.has_key('nSamples'):
            print 'no nSample option!'
            return []
        n = self.measure['nSamples']
        if n == 0:
            print 'no samples defined!'
            return []
        out = []
        for i in range(n):
            # Search direct child (instrument)
            child = self.child('sample{}'.format(i))
            if child:
                out.append(child)
                continue
            # Search referred sample (from device)
            s = 'smp{}'.format(i)
            if not self.has_key(s):
                print 'sample not found', s
                break
            # Get fullpath of the referred sample object
            s = self[s][0]
            # Get the actual object
            obj = self.root.toPath(s)
            if obj is None:
                print 'sample object not found', s
                continue
            out.append(obj)
        print 'returning samples', out
        return out

    def role2dev(self, opt):
        """Return the device object associated with role option `opt`"""
        p = self[opt]
        if p is False:
            return False
        p = p[0]
        if p in ('None', None):
            return False
        obj = self.root.toPath(p)
        if not obj:
            return False
        return obj

    def check_read(self, opt):
        return self.desc[opt].get('readLevel', 0) <= self._readLevel

    def check_write(self, opt):
        return self.desc[opt].get('writeLevel', 0) <= self._writeLevel
<commit_msg>Allow samples to be 1-based, FLTD-230<commit_after># -*- coding: utf-8 -*-
"""Option persistence."""
import re
import collections
import numpy as np

from .. import logger
from conf import Conf
import cPickle as pickle
from ..milang import Scriptable
import common_proxy
from .option import ao

def dictRecursiveModel(base):
    """BUild a dictionary configuration tree from ConfigurationProxy `base`"""
    out = collections.OrderedDict()
    for path, obj in base.iteritems():
        if path == 'self':
            out[path] = obj['name']
            continue
        out[path] = dictRecursiveModel(obj)
    return out

def print_tree(tree, level=0):
    """Pretty-print a dictionary configuration `tree`"""
    pre = '   ' * level
    msg = ''
    for k, v in tree.iteritems():
        if k == 'self':
            msg += print_tree(v, level)
            continue
        # Detect subdevice
        if isinstance(v, dict) and 'self' in v:
            msg += pre + '|++> ' + k + '\n'
            msg += print_tree(v, level + 1)
            continue
        v = repr(v['current'])
        if len(v) > 50:
            v = v[:46] + ' ...'
        msg += '{}|: {} = {}\n'.format(pre, k, v)
    return msg

# TODO: parametrize in user conf
# 0=always visible; 1=user ; 2=expert ; 3=advanced ; 4=technician ;
# 5=developer; 6=never visible
class ConfigurationProxy(Scriptable, Conf):

    """A configuration object behaving like a live server"""
    separator = '/'
    _readLevel = 5
    _writeLevel = 5
    _rmodel = False

    def print_tree(self):
        print print_tree(self.tree())

    def __init__(self, desc=collections.OrderedDict({'self': {}}),
                 name='MAINSERVER', parent=False, readLevel=2, writeLevel=5, kid_base='/'):
        Scriptable.__init__(self)
        self.log = logger.BaseLogger()
        self.kid_base = kid_base
        Conf.__init__(self, desc['self'])
        self._readLevel = readLevel
        self._writeLevel = writeLevel
        self.children = desc.copy()
        """Child configuration dictionaries"""
        self.children_obj = {}
        """Instantiated children configuration proxies"""
        # Remove myself from children...
        del self.children['self']
        self.name = name
        self._parent = parent
        if not parent:
            self._Method__name = name
        else:
            self._Method__name = parent._Method__name + self.separator + name
#           if parent._parent:
#               self._Method__name=parent._Method__name+self.separator+name
#           else:
#               self._Method__name=name
        if self.has_key('devpath'):
            self['devpath'] = name
        self.autosort()

    def get(self, *a, **k):
        return self.__getitem__(*a, **k)

    def set(self, *a, **k):
        return self.__setitem__(*a, **k)

    @property
    def root(self):
        if not self._parent:
            return self
        return self._parent.root

    @property
    def naturalName(self):
        return self.name

    def get_fullpath(self):
        po = self
        path = []
        while True:
            if po.name == 'MAINSERVER':
                break
            path.append(po.name)
            po = po.parent()
            if po in [None, False]:
                break
        # Revert to get the top-down path
        path.reverse()
        return '/' + '/'.join(path) + '/'

    def _update_from_children(self):
        """ Retrieve current configuration from instantiated child objects"""
        for key, obj in self.children_obj.iteritems():
            d = {'self': obj.desc}
            obj._update_from_children()
            d.update(obj.children)
            self.children[key] = d
        self.autosort()

    def rmodel(self):
        out = dictRecursiveModel(self.children)
        out['self'] = self.name
        return out

    def tree(self):
        """Build a full configuration tree suitable for serialization and saving on a file"""
        self._update_from_children()
        tree = self.children.copy()
        tree['self'] = self.desc.copy()
        return tree

    def __len__(self):
        return len(self.desc)

    def iteritems(self):
        return self.desc.iteritems()

    def itervalues(self):
        return self.desc.itervalues()

    def values(self):
        return self.desc.values()

    def iterkeys(self):
        return self.desc.iterkeys()

    def has_key(self, k):
        return self.desc.has_key(k)

    def dumps(self):
        d = {'self': self.desc.copy()}
        self._update_from_children()
        d.update(self.children)
        return pickle.dumps(d)

    def describe(self, *a, **kw):
        return self.desc

    def connect(self, *a, **k):
        pass

    def connection(self, *a, **k):
        return True

    def paste(self, obj):
        self.desc = obj.desc.copy()
        self.children = obj.children.copy()
        self.children_obj = obj.children_obj
        self._parent = obj._parent
        self._Method__name = obj._Method__name
        self._readLevel = obj._readLevel
        self._writeLevel = obj._writeLevel

    def copy(self):
        p = ConfigurationProxy()
        p.paste(self)
        return p

    def __nonzero__(self):
        return 1

    def __getitem__(self, key, *a):
        if key == 'fullpath':
            return self.get_fullpath()
        if len(a)==1 and not self.desc.has_key(key):
            return a[0]
        return self.desc[key]['current']

    def __setitem__(self, key, val):
        if not self.desc.has_key(key):
            print 'Impossible to set key', key, val
            return False
        self.desc[key]['current'] = val
        return True

    def gettype(self, key):
        return self.desc[key]['type']

    def multiget(self, keys):
        r = {}
        for k in keys:
            r[k] = self[k]
        return r

    def __getattr__(self, path):
        if path.startswith('_'):
            return object.__getattribute__(self, path)
        elif path in dir(self):
            return object.__getattribute__(self, path)
        return self.child(path)

    def gete(self, key):
        return self.desc[key]

    def sete(self, key, val):
        if not val.has_key('priority') or val['priority']==-1:
            priorities = [opt.get('priority',0) for opt in self.desc.values()]
            if len(priorities):
                val['priority'] = max(priorities) + 1
        self.desc[key] = val

    def add_option(self, *args, **kwargs):
        """Creates a new option using the ao() utility function.
        Migrate old one if existing."""
        out = {}
        overwrite = True
        if kwargs.has_key('overwrite'):
            overwrite = kwargs.pop('overwrite')
        ao(out, *args, **kwargs)
        out = out.values()[0]
        key = out['handle']
        if out['priority'] == 0:
            out['priority'] = -1
        # If option was already defined, update old one with new values
        if self.has_key(key):
            # Do not do anything if not overwriting
            if not overwrite:
                return out
            origin = self.gete(key).entry
            origin.update(out)
            out = origin

        self.sete(out['handle'], out)
        return out

    def autosort(self):
        def sorter(item):
            key, val = item
            digits = re.sub(r"\D", '', key)
            key = int(digits) if len(digits) else key
            return key
        self.children = collections.OrderedDict(sorted(self.children.items(), key=sorter))

    def add_child(self, name, desc, overwrite=False):
        """Inserts a sub-object `name` with object tree dictionary `desc`.
        Returns a ConfigurationProxy to the new child."""
        # If desc is the description dictionary and not the object tree dictionary,
        # encapsulate it in a 'self' dict
        if not 'self' in desc:
            desc = {'self': desc}
        if overwrite or name not in self.children:
            self.children[name] = desc
        else:
            for key, val in desc['self'].iteritems():
                self.children[name]['self'][key] = val
        self.autosort()
        return self.child(name)

    def getFlags(self, opt):
        if not self.desc[opt].has_key('flags'):
            return {}
        return self.desc[opt]['flags']

    def list(self):
        return [(c, c) for c in self.children.iterkeys()]

    def has_child(self, name):
        """Returns if `name` is a child"""
        return self.children.has_key(name)

    def child(self, name):
        """Return child ConfigurationProxy object by `name`"""
        if not self.children.has_key(name):
            return None
        if not self.children_obj.has_key(name):
            kb = self.kid_base + name + self.separator
            self.children_obj[name] = ConfigurationProxy(
                self.children[name], name=name, parent=self, kid_base=kb)
        return self.children_obj[name]

    def calc_aggregate(self, aggregation, handle=False):
        function_name = re.search("(.+?)\(", aggregation).group(1)
        targets = re.search("\((.+?)\)", aggregation)
        if targets is None:
            targets = [handle]
        else:
            targets = targets.group(1).replace(' ','').split(',')
        values = collections.defaultdict(list)
        for child_name in self.children.iterkeys():
            child = self.child(child_name)
            pack = collections.defaultdict(list)
            for target in targets:
                # Ensure all targets exist
                if not child.has_key(target):
                    pack = False
                    break
                pack[target].append(child[target])
            if pack:
                for t in targets:
                    values[t]+=pack[t]
        result = None
        error = None
        #TODO: calc stdev here
        if function_name == 'mean':
            v = np.array(values[targets[0]]).astype(np.float32)
            #FIXME: hack to filter out zeros
            v = v[v!=0]
            if len(v):
                result = float(v.mean())
                error = float(v.std())
        elif function_name == 'sum':
            result = float(np.array(values[targets[0]]).astype(np.float32).sum())
        elif function_name == 'prod':
            result = float(np.array(values[targets[0]]).astype(np.float32).prod())
        elif function_name == 'table':
            result = []
            for i, x in enumerate(values[targets[0]]):
                row = []
                for t in targets:
                    row.append(values[t][i])
                result.append(row)
            # Reorder by first column
            result = sorted(result, key=lambda e: e[0])
            # Prepend table header
            # Should calculate also table header?
            result = [self[handle][0]] + result
        else:
            self.log.error('Aggregate function not found:', function_name, aggregation)
        return result, error

    def update_aggregates(self, recursive=1):
        """Updates aggregate options. recursive==1, upward; -1, downward"""
        for handle, opt in self.desc.iteritems():
            if not opt.has_key('aggregate'):
                continue
            aggregation = opt['aggregate']
            result, error = self.calc_aggregate(aggregation, handle)
            if result is not None:
                self[handle] = result
                if error is not None and opt.has_key('error'):
                    self[opt['error']] = error
            else:
                self.log.error('Aggregation failed for ', handle, aggregation)
        if recursive>0 and self.parent():
            self.parent().update_aggregates(recursive=True)
        elif recursive<0:
            for k in self.children.keys():
                self.child(k).update_aggregates(recursive=-1)

    @property
    def devices(self):
        return [self.child(name) for name in self.children.keys()]

    def parent(self):
        return self._parent

    def toPath(self, lst):
        """Returns a copy of the object at the path expressed in list lst"""
        if isinstance(lst, str):
            lst = lst.split(self.separator)
            if lst[0] in ['server', '']:
                lst.pop(0)
            if lst[-1] == '':
                lst.pop(-1)
        obj = self.copy()
        for p in lst:
            if obj is None:
                return None
            obj = obj.child(p)
        return obj

    def from_column(self, col0):
        return common_proxy.from_column(col0, self.root)

    def toMethodName(self, name):
        if name == 'MAINSERVER':
            return
        lst = name.split(self.separator)
        return self.toPath(lst)

    def searchPath(self, path):
        """Search a child with the corresponding devpath=`path` and returns its name"""
        fp = self.get_fullpath().replace('//', '/')
        if not path.startswith(fp):
            print 'searchPath: not corresponding!', path, fp
            return False
        # Cut itself from the path
        if not path.endswith('/'):
            path += '/'
        vpath = path[len(fp):].split('/')[:-1]

        # Verify existence
        obj = self
        for p in vpath:
            obj = obj.child(p)
            if obj is None:
                print 'searchPath: not found', repr(p), repr(path)
        return self.separator.join(vpath)

    def listPresets(self):
        return ['default']

    def iterprint(self, base=False, pre=''):
        """Iterative printing for debug purposes"""
        if not base:
            base = self
        for k, v in base.iteritems():
            print pre, k, v['current']
        for sub in base.children.iterkeys():
            base.child(sub).iterprint(pre=pre + '\t%s: ' % sub)

    @property
    def samples(self):
        if not self.measure.has_key('nSamples'):
            print 'no nSample option!'
            return []
        n = self.measure['nSamples']
        if n == 0:
            print 'no samples defined!'
            return []
        out = []
        for i in range(n+1):
            # Search direct child (instrument)
            child = self.child('sample{}'.format(i))
            if child:
                out.append(child)
                continue
            # Search referred sample (from device)
            s = 'smp{}'.format(i)
            if not self.has_key(s):
                print 'sample not found', s
                break
            # Get fullpath of the referred sample object
            s = self[s][0]
            # Get the actual object
            obj = self.root.toPath(s)
            if obj is None:
                print 'sample object not found', s
                continue
            out.append(obj)
        print 'returning samples', out
        return out

    def role2dev(self, opt):
        """Return the device object associated with role option `opt`"""
        p = self[opt]
        if p is False:
            return False
        p = p[0]
        if p in ('None', None):
            return False
        obj = self.root.toPath(p)
        if not obj:
            return False
        return obj

    def check_read(self, opt):
        return self.desc[opt].get('readLevel', 0) <= self._readLevel

    def check_write(self, opt):
        return self.desc[opt].get('writeLevel', 0) <= self._writeLevel
<|endoftext|>