<commit_before>import json
import os
import re

from vee import log
from vee.cli import style, style_error, style_note, style_warning
from vee.commands.main import command, argument, group
from vee.envvars import render_envvars
from vee.exceptions import AlreadyInstalled, print_cli_exc
from vee.git import GitRepo, normalize_git_url
from vee.package import Package
from vee.packageset import PackageSet
from vee.requirements import Requirements
from vee.utils import makedirs

def iter_availible_requirements(home):
    env_repo = home.get_env_repo()
    req_set = env_repo.requirement_set()
    pkg_set = PackageSet(home=home)
    for req in req_set.iter_packages():
        pkg = pkg_set.resolve(req, check_existing=False)
        if pkg.type != 'git':
            return
        yield req, normalize_git_url(req.url, prefix=False)

@command(
    aliases=['dev'],
    help='develop a package',
    usage="""
       vee develop init [NAME]
   or: vee develop clone URL [NAME]
   or: vee develop add PATH [NAME]
   or: vee develop find PATH
   or: vee develop install NAME
   or: vee develop rescan [NAME ...]
   or: vee develop setenv NAME KEY=value [...]
   or: vee develop list [--availible] [--environ]
""".strip(),
)
def develop(args):
    pass

@develop.subcommand(
    argument('-a', '--availible', action='store_true'),
    argument('-e', '--environ', dest='show_environ', action='store_true'),
    argument('glob', nargs='?'),
    name='list',
    help='list all installed or availible tools'
)
def list_(args):

    home = args.assert_home()

    if args.availible:
        for req, url in iter_availible_requirements(home):
            print style_note(req.name, str(req))
        return

    if args.glob:
        cur = home.db.execute('SELECT * FROM development_packages WHERE name GLOB ? ORDER BY lower(name)', [args.glob])
    else:
        cur = home.db.execute('SELECT * FROM development_packages ORDER BY lower(name)')

    for row in cur:
        path = row['path'].replace(home.dev_root, '$VEE_DEV').replace(home.root, '$VEE')
        print style_note(row['name'], path)
        if args.show_environ:
            for k, v in sorted(render_envvars(json.loads(row['environ']), row['path']).iteritems()):
                if os.environ.get(k):
                    v = v.replace(os.environ[k], '$' + k)
                v = v.replace(home.dev_root, '$VEE_DEV')
                v = v.replace(home.root, '$VEE')
                print style('    %s=' % k) + v

@develop.subcommand(
    argument('--force', action='store_true'),
    argument('--repo'),
    argument('--path'),
    argument('name'),
    help='install a tool that is managed by the default repository'
)
def install(args):
    return init(args, do_install=True)

@develop.subcommand(
    argument('--force', action='store_true'),
    argument('--path'),
    argument('url'),
    argument('name', nargs='?'),
    help='clone a remote git repository to develop',
)
def clone(args):
    if not args.name:
        name = os.path.basename(args.url)
        if name.endswith('.git'):
            name = name[:-4]
        args.name = name
    return init(args, do_clone=True)

@develop.subcommand(
    argument('-r', '--recursive', action='store_true', help='add any tools found under given path'),
    argument('--force', action='store_true'),
    argument('path'),
    argument('name', nargs='?'),
    help='track an existing checkout',
)
def add(args):
    args.path = os.path.abspath(args.path)
    args.name = args.name or os.path.basename(args.path)
    return init(args, do_add=True)

@develop.subcommand(
    argument('patterns', nargs='+'),
    help='stop tracking dev packages',
)
def rm(args):
    res = 0
    con = args.assert_home().db.connect()
    for pattern in args.patterns:
        cur = con.execute('DELETE FROM development_packages WHERE name GLOB ?', [pattern])
        if not cur.rowcount:
            print style_warning('No dev packages matching "%s"' % pattern)
            res = 1
    return res

@develop.subcommand(
    argument('--force', action='store_true'),
    argument('path'),
    help='recursively find existing checkouts',
)
def find(args):
    res = e = None
    for dir_path, dir_names, file_names in os.walk(os.path.abspath(args.path)):
        if '.git' in dir_names:
            dir_names[:] = []
            args.path = dir_path
            args.name = os.path.basename(dir_path)
            res = init(args, do_add=True, is_find=True) or res
    if e:
        raise RuntimeError('There were errors adding dev packages.')
    return res

@develop.subcommand(
    argument('names', nargs='*'),
    help='recursively find existing checkouts',
)
def rescan(args):

    home = args.assert_home()
    con = home.db.connect()

    args.force = True

    pairs = []

    if args.names:
        for name in args.names:
            row = con.execute('SELECT path FROM development_packages WHERE name = ?', [name]).fetchone()
            if not row:
                log.warning('No dev package named %s' % name)
                continue
            path = row[0]
            if not os.path.exists(path):
                log.warning('Dev package %s not longer exists at %s' % (name, path))
                continue
            pairs.append((name, path))
    else:
        pairs = list(con.execute('SELECT name, path FROM development_packages'))

    for name, path in pairs:
        args.name = name
        args.path = path
        init(args, do_add=True)

@develop.subcommand(
    argument('--force', action='store_true'),
    argument('--path'),
    argument('name'),
    help='init a new git repository'
)
def init(args, do_clone=False, do_install=False, do_add=False, is_find=False):

    do_init = not (do_clone or do_install or do_add)

    name = args.name
    home = args.assert_home()

    con = home.db.connect()

    # Make sure there are no other packages already, and clear out old ones
    # which no longer exist.
    for row in con.execute('SELECT * FROM development_packages WHERE name = ?', [name]):
        if not args.force and os.path.exists(os.path.join(row['path'], '.git')):
            if is_find:
                print style_note('"%s" already exists:' % name, row['path'])
                return
            else:
                print style_error('"%s" already exists:' % name, row['path'])
                return 1
        else:
            con.execute('DELETE FROM development_packages WHERE id = ?', [row['id']])

    path = os.path.abspath(args.path or os.path.join(home.dev_root, name))

    dev_repo = GitRepo(path)

    if do_init:
        print style_note('Initing %s' % dev_repo.work_tree)
        makedirs(dev_repo.work_tree)
        dev_repo.git('init')

    elif do_clone:
        print style_note('Cloning %s' % args.url)
        makedirs(dev_repo.work_tree)
        dev_repo.clone_if_not_exists(args.url)

    elif do_install:
        # Find an existing tool.
        # TODO: put more of this into EnvironmentRepo or Requirements
        env_repo = home.get_env_repo(args.repo)
        req_path = os.path.join(env_repo.work_tree, 'requirements.txt')
        reqs = Requirements(req_path, home=home)
        for req in reqs.iter_packages():
            if req.name.lower() == name.lower():
                # Make sure it is a Git package.
                url = normalize_git_url(req.url, prefix=False)
                if url:
                    break
        else:
            print style_error('Could not find git-based "%s" in "%s" repo.' % (name, env_repo.name))
            return 2
        print style_note('Found %s in %s' % (name, env_repo.name), str(req))
        makedirs(dev_repo.work_tree)
        dev_repo.clone_if_not_exists(url, shallow=False)

    elif do_add:
        print style_note('Adding %s from %s' % (name, path))

    if not os.path.exists(path):
        log.error('%s does not exist'%  path)
        return 1

    package = Package([path], home=home, dev=True)
    try:
        package.pipeline.run_to('develop')
    except Exception as e:
        print_cli_exc(e)
        return 1

    print style_note('Linking dev package', name, path)
    con.execute('INSERT INTO development_packages (name, path, environ) VALUES (?, ?, ?)', [name, path, json.dumps(package.environ)])

@develop.subcommand(
    argument('name'),
    argument('envvars', nargs='+'),
    help='set envvars within the dev execution environment',
)
def setenv(args):

    home = args.assert_home()
    con = home.db.connect()

    row = con.execute('SELECT * FROM development_packages WHERE name = ?', [args.name]).fetchone()
    if not row:
        raise ValueError('unknown dev package %r' % args.name)

    environ = json.loads(row['environ'])

    for envvar in args.envvars:
        if re.match(r'^-\w+$', envvar):
            del environ[envvar[1:]]
        else:
            key, value = envvar.split('=', 1)
            environ[key] = value

    con.execute('UPDATE development_packages SET environ = ? WHERE id = ?', [json.dumps(environ), row['id']])

<commit_msg>Note existance of `vee dev rm`<commit_after>import json
import os
import re

from vee import log
from vee.cli import style, style_error, style_note, style_warning
from vee.commands.main import command, argument, group
from vee.envvars import render_envvars
from vee.exceptions import AlreadyInstalled, print_cli_exc
from vee.git import GitRepo, normalize_git_url
from vee.package import Package
from vee.packageset import PackageSet
from vee.requirements import Requirements
from vee.utils import makedirs

def iter_availible_requirements(home):
    env_repo = home.get_env_repo()
    req_set = env_repo.requirement_set()
    pkg_set = PackageSet(home=home)
    for req in req_set.iter_packages():
        pkg = pkg_set.resolve(req, check_existing=False)
        if pkg.type != 'git':
            return
        yield req, normalize_git_url(req.url, prefix=False)

@command(
    aliases=['dev'],
    help='develop a package',
    usage="""
       vee develop init [NAME]
   or: vee develop clone URL [NAME]
   or: vee develop add PATH [NAME]
   or: vee develop rm PATTERN
   or: vee develop find PATH
   or: vee develop install NAME
   or: vee develop rescan [NAME ...]
   or: vee develop setenv NAME KEY=value [...]
   or: vee develop list [--availible] [--environ]
""".strip(),
)
def develop(args):
    pass

@develop.subcommand(
    argument('-a', '--availible', action='store_true'),
    argument('-e', '--environ', dest='show_environ', action='store_true'),
    argument('glob', nargs='?'),
    name='list',
    help='list all installed or availible tools'
)
def list_(args):

    home = args.assert_home()

    if args.availible:
        for req, url in iter_availible_requirements(home):
            print style_note(req.name, str(req))
        return

    if args.glob:
        cur = home.db.execute('SELECT * FROM development_packages WHERE name GLOB ? ORDER BY lower(name)', [args.glob])
    else:
        cur = home.db.execute('SELECT * FROM development_packages ORDER BY lower(name)')

    for row in cur:
        path = row['path'].replace(home.dev_root, '$VEE_DEV').replace(home.root, '$VEE')
        print style_note(row['name'], path)
        if args.show_environ:
            for k, v in sorted(render_envvars(json.loads(row['environ']), row['path']).iteritems()):
                if os.environ.get(k):
                    v = v.replace(os.environ[k], '$' + k)
                v = v.replace(home.dev_root, '$VEE_DEV')
                v = v.replace(home.root, '$VEE')
                print style('    %s=' % k) + v

@develop.subcommand(
    argument('--force', action='store_true'),
    argument('--repo'),
    argument('--path'),
    argument('name'),
    help='install a tool that is managed by the default repository'
)
def install(args):
    return init(args, do_install=True)

@develop.subcommand(
    argument('--force', action='store_true'),
    argument('--path'),
    argument('url'),
    argument('name', nargs='?'),
    help='clone a remote git repository to develop',
)
def clone(args):
    if not args.name:
        name = os.path.basename(args.url)
        if name.endswith('.git'):
            name = name[:-4]
        args.name = name
    return init(args, do_clone=True)

@develop.subcommand(
    argument('-r', '--recursive', action='store_true', help='add any tools found under given path'),
    argument('--force', action='store_true'),
    argument('path'),
    argument('name', nargs='?'),
    help='track an existing checkout',
)
def add(args):
    args.path = os.path.abspath(args.path)
    args.name = args.name or os.path.basename(args.path)
    return init(args, do_add=True)

@develop.subcommand(
    argument('patterns', nargs='+'),
    help='stop tracking dev packages',
)
def rm(args):
    res = 0
    con = args.assert_home().db.connect()
    for pattern in args.patterns:
        cur = con.execute('DELETE FROM development_packages WHERE name GLOB ?', [pattern])
        if not cur.rowcount:
            print style_warning('No dev packages matching "%s"' % pattern)
            res = 1
    return res

@develop.subcommand(
    argument('--force', action='store_true'),
    argument('path'),
    help='recursively find existing checkouts',
)
def find(args):
    res = e = None
    for dir_path, dir_names, file_names in os.walk(os.path.abspath(args.path)):
        if '.git' in dir_names:
            dir_names[:] = []
            args.path = dir_path
            args.name = os.path.basename(dir_path)
            res = init(args, do_add=True, is_find=True) or res
    if e:
        raise RuntimeError('There were errors adding dev packages.')
    return res

@develop.subcommand(
    argument('names', nargs='*'),
    help='recursively find existing checkouts',
)
def rescan(args):

    home = args.assert_home()
    con = home.db.connect()

    args.force = True

    pairs = []

    if args.names:
        for name in args.names:
            row = con.execute('SELECT path FROM development_packages WHERE name = ?', [name]).fetchone()
            if not row:
                log.warning('No dev package named %s' % name)
                continue
            path = row[0]
            if not os.path.exists(path):
                log.warning('Dev package %s not longer exists at %s' % (name, path))
                continue
            pairs.append((name, path))
    else:
        pairs = list(con.execute('SELECT name, path FROM development_packages'))

    for name, path in pairs:
        args.name = name
        args.path = path
        init(args, do_add=True)

@develop.subcommand(
    argument('--force', action='store_true'),
    argument('--path'),
    argument('name'),
    help='init a new git repository'
)
def init(args, do_clone=False, do_install=False, do_add=False, is_find=False):

    do_init = not (do_clone or do_install or do_add)

    name = args.name
    home = args.assert_home()

    con = home.db.connect()

    # Make sure there are no other packages already, and clear out old ones
    # which no longer exist.
    for row in con.execute('SELECT * FROM development_packages WHERE name = ?', [name]):
        if not args.force and os.path.exists(os.path.join(row['path'], '.git')):
            if is_find:
                print style_note('"%s" already exists:' % name, row['path'])
                return
            else:
                print style_error('"%s" already exists:' % name, row['path'])
                return 1
        else:
            con.execute('DELETE FROM development_packages WHERE id = ?', [row['id']])

    path = os.path.abspath(args.path or os.path.join(home.dev_root, name))

    dev_repo = GitRepo(path)

    if do_init:
        print style_note('Initing %s' % dev_repo.work_tree)
        makedirs(dev_repo.work_tree)
        dev_repo.git('init')

    elif do_clone:
        print style_note('Cloning %s' % args.url)
        makedirs(dev_repo.work_tree)
        dev_repo.clone_if_not_exists(args.url)

    elif do_install:
        # Find an existing tool.
        # TODO: put more of this into EnvironmentRepo or Requirements
        env_repo = home.get_env_repo(args.repo)
        req_path = os.path.join(env_repo.work_tree, 'requirements.txt')
        reqs = Requirements(req_path, home=home)
        for req in reqs.iter_packages():
            if req.name.lower() == name.lower():
                # Make sure it is a Git package.
                url = normalize_git_url(req.url, prefix=False)
                if url:
                    break
        else:
            print style_error('Could not find git-based "%s" in "%s" repo.' % (name, env_repo.name))
            return 2
        print style_note('Found %s in %s' % (name, env_repo.name), str(req))
        makedirs(dev_repo.work_tree)
        dev_repo.clone_if_not_exists(url, shallow=False)

    elif do_add:
        print style_note('Adding %s from %s' % (name, path))

    if not os.path.exists(path):
        log.error('%s does not exist'%  path)
        return 1

    package = Package([path], home=home, dev=True)
    try:
        package.pipeline.run_to('develop')
    except Exception as e:
        print_cli_exc(e)
        return 1

    print style_note('Linking dev package', name, path)
    con.execute('INSERT INTO development_packages (name, path, environ) VALUES (?, ?, ?)', [name, path, json.dumps(package.environ)])

@develop.subcommand(
    argument('name'),
    argument('envvars', nargs='+'),
    help='set envvars within the dev execution environment',
)
def setenv(args):

    home = args.assert_home()
    con = home.db.connect()

    row = con.execute('SELECT * FROM development_packages WHERE name = ?', [args.name]).fetchone()
    if not row:
        raise ValueError('unknown dev package %r' % args.name)

    environ = json.loads(row['environ'])

    for envvar in args.envvars:
        if re.match(r'^-\w+$', envvar):
            del environ[envvar[1:]]
        else:
            key, value = envvar.split('=', 1)
            environ[key] = value

    con.execute('UPDATE development_packages SET environ = ? WHERE id = ?', [json.dumps(environ), row['id']])

<|endoftext|>


<commit_before>#!/usr/local/bin/python

# Create a clean archive of a photo store.
# Sends the files to a default location that can be overridden.
# Todo, would be handy to be able to create an optional log for checking before the
# archive is sent to glacier.  Just use --log=DEBUG then log at info level.
# The folder will be based on the folder being archived.
# i.e. if you archive .../pictures/Archive_PS1/store0035/
# the final archive will be /share/backup-jobs/aws-glacier/backup_store0035/store0035.tar.bz2

import os
import shutil
import sys, getopt
import tarfile
from sets import Set

IGNORE_PATTERNS = ('*.DS_Store','*.@__thumb')

dirToDefault = "/share/backup-jobs/aws-glacier"
dirToPrefix = "backup_"

ignoredFiles = set()
includedFiles = set()

class FileStatus:
   pass
class Flags:
   pass

def printHelp():
   print ''
   print sys.argv[0] + ' [-h] [l | list] [s | summary] [v | verbose] [i | input-dir] [o | output-dir]'
   print ''
   print 'Creates a tar cleaned of all files we do not want to send to offline archive.'
   print 'Ignores symbolic links'
   print 'Ignored names: [{}]'.format(', '.join(IGNORE_PATTERNS))

   print ''
   print 'Typical use cases:'
   print ''

   print 'Report a dry run:'
   print sys.argv[0] + ' -d <input directory> -l'
   print ''

   print 'Report summary:'
   print sys.argv[0] + ' -i <input directory> -s'
   print ''

   print 'Creating archives:'
   print 'With no output the default location is used: [{}]'.format(dirToDefault)
   print sys.argv[0] + ' -i <input directory>'
   print sys.argv[0] + ' -i <input directory> -o <output directory [{}]>'
   print 'Verbose and summary'
   print sys.argv[0] + ' -i <input directory> -o <output directory [{}]> -vs'

   print ''

# Collect skipped file
def addSkipFile(ignoredFiles, src, name, why):
   fileStatus = FileStatus()
   fileStatus.why = why
   fileStatus.fileName = os.path.join(src, name)
   ignoredFiles.add(fileStatus)

   return;

# Find files
def listtree(src, ignore=None):
   names = os.listdir(src)

   if ignore is not None:
      ignoredNames = ignore(src, names)
   else:
      ignoredNames = set()

   errors = []
   for name in names:
      if name in ignoredNames:
         addSkipFile(ignoredFiles, src, name, 'skip pattern match')
         continue

      srcname = os.path.join(src, name)
      try:
         if os.path.isdir(srcname) and not os.path.islink(srcname):
            listtree(srcname, ignore)
         elif os.path.islink(srcname):
            addSkipFile(ignoredFiles, src, name, 'skip symbolic link')
         else:
            includedFiles.add(os.path.abspath(srcname))

      # What about devices, sockets etc.?
      except (IOError, os.error) as why:
         errors.append((srcname, str(why)))
      # catch the Error from the recursive listtree so that we can
      # continue with other files
      except EnvironmentError as err:
         errors.extend(err.args[0])

   if errors:
      raise Exception(errors)

   return;

# Print files which will be added / not added to archive
def printList(includedFiles, ignoredFiles):
   print 'Ignore Names: [{}]'.format(', '.join(IGNORE_PATTERNS))

   print '--Include--'
   for fileName in sorted(includedFiles):
      print '[{}]'.format(fileName)

   print ''
   print '--Ignored--'
   for fileStatus in sorted(ignoredFiles):
      print '[{}],[{}]'.format(fileStatus.why, fileStatus.fileName)
   return;

# Print a sum totals.
def printSummary(includedFiles, ignoredFiles):
   print ''
   print '--Summary--'
   print '{} included files.'.format(len(includedFiles))
   print '{} ignored files.'.format(len(ignoredFiles))
   return;

# Create a new location for the file based on its name
def createToDir(dirFrom, dirTo, dirToPrefix, toDirPathEnd):

   createDir = os.path.join(dirTo, dirToPrefix + toDirPathEnd)

   if not os.path.exists(createDir):
      os.mkdir(createDir, 0755)

   return createDir;

# Build the compressed archive
def createArchive(dirFrom, dirTo, dirToPrefix, includedFiles, flags):

   toDirPathEnd = os.path.basename(os.path.normpath(dirFrom))
   archiveToDir = createToDir(dirFrom, dirTo, dirToPrefix, toDirPathEnd)
   tarPath = archiveToDir + '/' + toDirPathEnd + '.tar.bz2'

   if (flags.verbose == True):
      print 'Archive to: [{}]'.format(tarPath)

   with tarfile.open(tarPath, "w:bz2") as tar:
      for name in includedFiles:
         if (flags.verbose == True):
            print '+ {}'.format(name)
            tar.add(name)

   return;

#
# Create a clean archive.  Process incoming args.
def main(argv):
   flags = Flags()
   flags.verbose = False
   flags.summary = False
   flags.list = False

   dirFrom = ''
   dirTo = dirToDefault

   try:
      opts, args = getopt.getopt(argv, "hi:lo:sv", ["input-dir=","list","output-dir=","summary","verbose"])
   except getopt.GetoptError:
      printHelp()
      sys.exit(2)

   for opt, arg in opts:
      if opt == "-h":
         printHelp()
         sys.exit()

      if opt in ("-o", "--output-dir"):
         dirTo = arg

      if opt in ("-i", "--input-dir"):
         dirFrom = arg
         listtree(src=dirFrom, ignore=shutil.ignore_patterns(*IGNORE_PATTERNS))

      if opt in ("-s", "--summary"):
         flags.summary = True

      if opt in ("-v", "--verbose"):
         flags.verbose = True

      if opt in ("-l", "--list"):
         flags.list = True

   if (flags.list == True):
      printList(includedFiles, ignoredFiles)

   if (flags.summary == True):
      printSummary(includedFiles, ignoredFiles)

   # Do no further processing if we are only asked to list, this is a dry run.
   if (flags.list == True):
      sys.exit()

   createArchive(dirFrom, dirTo, dirToPrefix, includedFiles, flags)

   sys.exit()

if __name__ == "__main__":
   main(sys.argv[1:])

# making a tar
def make_tarfile(output_filename, source_dir):
    with tarfile.open(output_filename, "w:gz") as tar:
        tar.add(source_dir, arcname=os.path.basename(source_dir))

# The old script used to take tars and put them into a folder based on their names.
# I would suggest just creating the tar in the right place in the first place.
# This is because glacier backups begin with a source folder.
# i.e. backup_tarfilename

<commit_msg>Better solution for paths, and fix quiet mode empty file bug.<commit_after>#!/usr/local/bin/python

# Create a clean archive of a photo store.
# Sends the files to a default location that can be overridden.
# Todo, would be handy to be able to create an optional log for checking before the
# archive is sent to glacier.  Just use --log=DEBUG then log at info level.
# The folder will be based on the folder being archived.
# i.e. if you archive .../pictures/Archive_PS1/store0035/
# the final archive will be /share/backup-jobs/aws-glacier/backup_store0035/store0035.tar.bz2

import os
import shutil
import sys, getopt
import tarfile
from sets import Set

IGNORE_PATTERNS = ('*.DS_Store','*.@__thumb')

dirToDefault = "/share/backup-jobs/aws-glacier"
dirToPrefix = "backup_"

ignoredFiles = set()
includedFiles = set()

class FileStatus:
   pass
class Flags:
   pass

def printHelp():
   print ''
   print sys.argv[0] + ' [-h] [l | list] [s | summary] [v | verbose] [i | input-dir] [o | output-dir]'
   print ''
   print 'Creates a tar cleaned of all files we do not want to send to offline archive.'
   print 'Ignores symbolic links'
   print 'Ignored names: [{}]'.format(', '.join(IGNORE_PATTERNS))

   print ''
   print 'Typical use cases:'
   print ''

   print 'Report a dry run:'
   print sys.argv[0] + ' -d <input directory> -l'
   print ''

   print 'Report summary:'
   print sys.argv[0] + ' -i <input directory> -s'
   print ''

   print 'Creating archives:'
   print 'With no output the default location is used: [{}]'.format(dirToDefault)
   print sys.argv[0] + ' -i <input directory>'
   print sys.argv[0] + ' -i <input directory> -o <output directory [{}]>'
   print 'Verbose and summary'
   print sys.argv[0] + ' -i <input directory> -o <output directory [{}]> -vs'

   print ''

# Collect skipped file
def addSkipFile(ignoredFiles, src, name, why):
   fileStatus = FileStatus()
   fileStatus.why = why
   fileStatus.fileName = os.path.join(src, name)
   ignoredFiles.add(fileStatus)

   return;

# Find files
def listtree(src, ignore=None):
   names = os.listdir(src)

   if ignore is not None:
      ignoredNames = ignore(src, names)
   else:
      ignoredNames = set()

   errors = []
   for name in names:
      if name in ignoredNames:
         addSkipFile(ignoredFiles, src, name, 'skip pattern match')
         continue

      srcname = os.path.join(src, name)
      try:
         if os.path.isdir(srcname) and not os.path.islink(srcname):
            listtree(srcname, ignore)
         elif os.path.islink(srcname):
            addSkipFile(ignoredFiles, src, name, 'skip symbolic link')
         else:
            includedFiles.add(srcname)

      # What about devices, sockets etc.?
      except (IOError, os.error) as why:
         errors.append((srcname, str(why)))
      # catch the Error from the recursive listtree so that we can
      # continue with other files
      except EnvironmentError as err:
         errors.extend(err.args[0])

   if errors:
      raise Exception(errors)

   return;

# Print files which will be added / not added to archive
def printList(includedFiles, ignoredFiles):
   print 'Ignore Names: [{}]'.format(', '.join(IGNORE_PATTERNS))

   print '--Include--'
   for fileName in sorted(includedFiles):
      print '[{}]'.format(fileName)

   print ''
   print '--Ignored--'
   for fileStatus in sorted(ignoredFiles):
      print '[{}],[{}]'.format(fileStatus.why, fileStatus.fileName)
   return;

# Print a sum totals.
def printSummary(includedFiles, ignoredFiles):
   print ''
   print '--Summary--'
   print '{} included files.'.format(len(includedFiles))
   print '{} ignored files.'.format(len(ignoredFiles))
   return;

# Create a new location for the file based on its name
def createToDir(dirFrom, dirTo, dirToPrefix, toDirPathEnd):

   createDir = os.path.join(dirTo, dirToPrefix + toDirPathEnd)

   if not os.path.exists(createDir):
      os.mkdir(createDir, 0755)

   return createDir;

# Build the compressed archive
def createArchive(dirFrom, dirTo, dirToPrefix, includedFiles, flags):

   toDirPathEnd = os.path.basename(os.path.normpath(dirFrom))
   archiveToDir = createToDir(dirFrom, dirTo, dirToPrefix, toDirPathEnd)
   tarPath = archiveToDir + '/' + toDirPathEnd + '.tar.bz2'

   if (flags.verbose == True):
      print 'Archive to: [{}]'.format(tarPath)

   with tarfile.open(tarPath, "w:bz2") as tar:
      for name in includedFiles:
         if (flags.verbose == True):
            print '+ {}'.format(name)
         tar.add(name, arcname=os.path.abspath(name))

   return;

#
# Create a clean archive.  Process incoming args.
def main(argv):
   flags = Flags()
   flags.verbose = False
   flags.summary = False
   flags.list = False

   dirFrom = ''
   dirTo = dirToDefault

   try:
      opts, args = getopt.getopt(argv, "hi:lo:sv", ["input-dir=","list","output-dir=","summary","verbose"])
   except getopt.GetoptError:
      printHelp()
      sys.exit(2)

   for opt, arg in opts:
      if opt == "-h":
         printHelp()
         sys.exit()

      if opt in ("-o", "--output-dir"):
         dirTo = arg

      if opt in ("-i", "--input-dir"):
         dirFrom = arg
         listtree(src=dirFrom, ignore=shutil.ignore_patterns(*IGNORE_PATTERNS))

      if opt in ("-s", "--summary"):
         flags.summary = True

      if opt in ("-v", "--verbose"):
         flags.verbose = True

      if opt in ("-l", "--list"):
         flags.list = True

   if (flags.list == True):
      printList(includedFiles, ignoredFiles)

   if (flags.summary == True):
      printSummary(includedFiles, ignoredFiles)

   # Do no further processing if we are only asked to list, this is a dry run.
   if (flags.list == True):
      sys.exit()

   createArchive(dirFrom, dirTo, dirToPrefix, includedFiles, flags)

   sys.exit()

if __name__ == "__main__":
   main(sys.argv[1:])

<|endoftext|>


<commit_before>import os, sys, urlparse, socket, signal, resource, logging
###

L = logging.getLogger("utils")

###

def launch_server():
	'''
This function launches Ramona server - in 'os.exec' manner which means that this function will not return
and instead of that, current process will be replaced by launched server.

All file descriptors above 2 are closed.
	'''
	from .config import config, config_files

	args = []
	for cfile in config_files:
		args.append('-c')
		args.append(cfile)

	# Close all open file descriptors above standard ones.  This prevents the child from keeping
	# open any file descriptors inherited from the parent.
	os.closerange(3, MAXFD)

	os.execl(sys.executable, config.get('server', 'svrname'), "-m", "ramona.server", *args)

#

def launch_server_daemonized():
	"""
This function launches Ramona server as a UNIX daemon.
It detaches the process context from parent (caller) and session.
In comparison to launch_server() it returns.
	"""
	from .config import config

	logfname = config.get('server','log')
	if logfname == '<logdir>':
		logfname = os.path.join(config.get('general','logdir'), 'ramona.log')
	elif logfname[:1] == '<':
		L.error("Unknown log option in [server] section - server not started")
		return

	try:
		with open(logfname, 'a') as logf:

			pid = os.fork()
			if pid > 0:
				return pid

			os.setsid()

			pid = os.fork()
			if pid > 0:
				os._exit(0)

			stdin = os.open(os.devnull, os.O_RDONLY)
			os.dup2(stdin, 0)

			os.dup2(logf.fileno(), 1) # Prepare stdout
			os.dup2(logf.fileno(), 2) # Prepare stderr
		launch_server()
	except IOError, e:
		L.fatal("Cannot open logfile {0} for writing. Check the configuration in [server] section. Exiting.".format(logfname))
		return

###

class deleteing_unix_socket(socket.socket):
	'''
This class is used as wrapper to socket object that represent listening UNIX socket.
It added ability to delete socket file when destroyed.
	'''

	def __init__(self):
		socket.socket.__init__(self, socket.AF_UNIX, socket.SOCK_STREAM)
		self.__sockfile = None

	def __del__(self):
		if self.__sockfile is not None:
			fname = self.__sockfile
			self.__sockfile = None
			os.unlink(fname)
			assert not os.path.isfile(fname)

	def bind(self, fname):
		socket.socket.bind(self, fname)
		self.__sockfile = fname

###

class socket_uri(object):
	'''
Socket factory that is configured using socket URI.
	'''

	# Configure urlparce
	if 'unix' not in urlparse.uses_params: urlparse.uses_params.append('unix')

	def __init__(self, uri):
		self.uri = urlparse.urlparse(uri)
		self.uriparams = dict(urlparse.parse_qsl(self.uri.params))

		self.protocol = self.uri.scheme.lower()
		if self.protocol == 'tcp':
			try:
				_port = self.uri.port
			except ValueError:
				raise RuntimeError("Invalid port number in socket URI {0}".format(uri))

			if self.uri.path != '': raise RuntimeError("Path has to be empty in socket URI {0}".format(uri))

		elif self.protocol == 'unix':
			pass

		else:
			raise RuntimeError("Unknown/unsuported protocol '{0}' in socket URI {1}".format(self.protocol, uri))

	def create_socket_listen(self):
		if self.protocol == 'tcp':
			s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
			s.bind((self.uri.hostname, self.uri.port))

		elif self.protocol == 'unix':
			mode = self.uriparams.get('mode',None)
			if mode is None: mode = 0o600
			else: mode = int(mode,8)
			oldmask = os.umask(mode ^ 0o777)
			s = deleteing_unix_socket()
			s.bind(self.uri.path)
			os.umask(oldmask)

		else:
			raise RuntimeError("Unknown/unsuported protocol '{0}'".format(self.protocol))

		return s

	def create_socket_connect(self):
		if self.protocol == 'tcp':
			s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			s.connect((self.uri.hostname, self.uri.port))

		elif self.protocol == 'unix':
			s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
			s.connect(self.uri.path)

		else:
			raise RuntimeError("Unknown/unsuported protocol '{0}'".format(self.protocol))

		return s

###

def parse_signals(signals):
	ret = []
	signame2signum = dict((name, num) for name, num in signal.__dict__.iteritems() if name.startswith('SIG') and not name.startswith('SIG_'))
	for signame in signals.split(','):
		signame = signame.strip().upper()
		if not signame.startswith('SIG'): signame = 'SIG'+signame
		signum = signame2signum.get(signame)
		if signum is None: raise RuntimeError("Unknown signal '{0}'".format(signame))
		ret.append(signum)
	return ret

###

MAXFD = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if (MAXFD == resource.RLIM_INFINITY): MAXFD = 1024
<commit_msg>Better log message<commit_after>import os, sys, urlparse, socket, signal, resource, logging
###

L = logging.getLogger("utils")

###

def launch_server():
	'''
This function launches Ramona server - in 'os.exec' manner which means that this function will not return
and instead of that, current process will be replaced by launched server.

All file descriptors above 2 are closed.
	'''
	from .config import config, config_files

	args = []
	for cfile in config_files:
		args.append('-c')
		args.append(cfile)

	# Close all open file descriptors above standard ones.  This prevents the child from keeping
	# open any file descriptors inherited from the parent.
	os.closerange(3, MAXFD)

	os.execl(sys.executable, config.get('server', 'svrname'), "-m", "ramona.server", *args)

#

def launch_server_daemonized():
	"""
This function launches Ramona server as a UNIX daemon.
It detaches the process context from parent (caller) and session.
In comparison to launch_server() it returns.
	"""
	from .config import config

	logfname = config.get('server','log')
	if logfname == '<logdir>':
		logfname = os.path.join(config.get('general','logdir'), 'ramona.log')
	elif logfname[:1] == '<':
		L.error("Unknown log option in [server] section - server not started")
		return

	try:
		with open(logfname, 'a') as logf:

			pid = os.fork()
			if pid > 0:
				return pid

			os.setsid()

			pid = os.fork()
			if pid > 0:
				os._exit(0)

			stdin = os.open(os.devnull, os.O_RDONLY)
			os.dup2(stdin, 0)

			os.dup2(logf.fileno(), 1) # Prepare stdout
			os.dup2(logf.fileno(), 2) # Prepare stderr
		launch_server()
	except IOError, e:
		L.fatal("Cannot open logfile {0} for writing: {1}. Check the configuration in [server] section. Exiting.".format(logfname, e))
		return

###

class deleteing_unix_socket(socket.socket):
	'''
This class is used as wrapper to socket object that represent listening UNIX socket.
It added ability to delete socket file when destroyed.
	'''

	def __init__(self):
		socket.socket.__init__(self, socket.AF_UNIX, socket.SOCK_STREAM)
		self.__sockfile = None

	def __del__(self):
		if self.__sockfile is not None:
			fname = self.__sockfile
			self.__sockfile = None
			os.unlink(fname)
			assert not os.path.isfile(fname)

	def bind(self, fname):
		socket.socket.bind(self, fname)
		self.__sockfile = fname

###

class socket_uri(object):
	'''
Socket factory that is configured using socket URI.
	'''

	# Configure urlparce
	if 'unix' not in urlparse.uses_params: urlparse.uses_params.append('unix')

	def __init__(self, uri):
		self.uri = urlparse.urlparse(uri)
		self.uriparams = dict(urlparse.parse_qsl(self.uri.params))

		self.protocol = self.uri.scheme.lower()
		if self.protocol == 'tcp':
			try:
				_port = self.uri.port
			except ValueError:
				raise RuntimeError("Invalid port number in socket URI {0}".format(uri))

			if self.uri.path != '': raise RuntimeError("Path has to be empty in socket URI {0}".format(uri))

		elif self.protocol == 'unix':
			pass

		else:
			raise RuntimeError("Unknown/unsuported protocol '{0}' in socket URI {1}".format(self.protocol, uri))

	def create_socket_listen(self):
		if self.protocol == 'tcp':
			s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
			s.bind((self.uri.hostname, self.uri.port))

		elif self.protocol == 'unix':
			mode = self.uriparams.get('mode',None)
			if mode is None: mode = 0o600
			else: mode = int(mode,8)
			oldmask = os.umask(mode ^ 0o777)
			s = deleteing_unix_socket()
			s.bind(self.uri.path)
			os.umask(oldmask)

		else:
			raise RuntimeError("Unknown/unsuported protocol '{0}'".format(self.protocol))

		return s

	def create_socket_connect(self):
		if self.protocol == 'tcp':
			s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			s.connect((self.uri.hostname, self.uri.port))

		elif self.protocol == 'unix':
			s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
			s.connect(self.uri.path)

		else:
			raise RuntimeError("Unknown/unsuported protocol '{0}'".format(self.protocol))

		return s

###

def parse_signals(signals):
	ret = []
	signame2signum = dict((name, num) for name, num in signal.__dict__.iteritems() if name.startswith('SIG') and not name.startswith('SIG_'))
	for signame in signals.split(','):
		signame = signame.strip().upper()
		if not signame.startswith('SIG'): signame = 'SIG'+signame
		signum = signame2signum.get(signame)
		if signum is None: raise RuntimeError("Unknown signal '{0}'".format(signame))
		ret.append(signum)
	return ret

###

MAXFD = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if (MAXFD == resource.RLIM_INFINITY): MAXFD = 1024
<|endoftext|>


<commit_before>"""The evaluator module simplifies the evaluation of segmentation results.

The module provides the possibility of calculate several evaluation metrics in parallel and output them in any format.
"""
import csv
import os
from abc import ABCMeta, abstractmethod
from typing import Union

import numpy as np
import SimpleITK as sitk

import mialab.evaluation.metric as mtrc

class IEvaluatorWriter(metaclass=ABCMeta):
    """Represents an evaluator writer interface, which enables to write evaluation results."""

    @abstractmethod
    def write(self, data: list):
        """Writes the evaluation results.

        Args:
            data (list): The evaluation data.
        """
        raise NotImplementedError

    @abstractmethod
    def write_header(self, header: list):
        """Writes the evaluation header.

        Args:
            header (list): The evaluation header.
        """
        raise NotImplementedError

class CSVEvaluatorWriter(IEvaluatorWriter):
    """Represents a CSV (comma-separated values) evaluator writer."""

    def __init__(self, path: str):
        """Initializes a new instance of the CSVEvaluatorWriter class.

        Args:
            path (path): The file path.
        """
        super().__init__()

        self.path = path

        # check file extension
        if not self.path.lower().endswith('.csv'):
            self.path = os.path.join(self.path, '.csv')

        open(self.path, 'w', newline='')  # creates (and overrides an existing) file

    def write(self, data: list):
        """Writes the evaluation results.

        Args:
            data (list): The evaluation data.
        """

        for evaluation in data:
            self.write_line(evaluation)

    def write_header(self, header: list):
        """Writes the evaluation header.

        Args:
            header (list): The evaluation header.
        """

        self.write_line(header)

    def write_line(self, data: list):
        """Writes a line.

        Args:
            data (list): The data.
        """
        with open(self.path, 'a', newline='') as file:
            writer = csv.writer(file, delimiter=';')
            writer.writerow(data)

class ConsoleEvaluatorWriter(IEvaluatorWriter):
    """Represents a console evaluator writer."""

    def __init__(self, precision: int=3):
        """Initializes a new instance of the ConsoleEvaluatorWriter class.

        Args:
            precision (int): The float precision.
        """
        super().__init__()

        self.header = None
        self.precision = precision

    def write(self, data: list):
        """Writes the evaluation results.

        Args:
            data (list of list): The evaluation data,
                e.g. [['PATIENT1', 'BACKGROUND', 0.90], ['PATIENT1', 'TUMOR', '0.62']]
        """

        # format all floats with given precision to str
        out = []
        for line in data:
            out += [val if isinstance(val, str) else "{0:.{1}f}".format(val, self.precision) for val in line]
            out += ["\n"]

        # get length for output alignment
        length = max(len(max(out, key=len)), len(max(self.header, key=len))) + 2

        print("".join("{0:>{1}}".format(val, length) for val in self.header),
              "\n",
              "".join("{0:>{1}}".format(val, length) for val in out),
              sep='', end='')

    def write_header(self, header: list):
        """Writes the evaluation header.

        Args:
            header (list of str): The evaluation header.
        """

        self.header = header

class Evaluator:
    """Represents a metric evaluator.

    Examples:

    >>> evaluator = Evaluator(ConsoleEvaluatorWriter(5))
    >>> evaluator.add_writer(CSVEvaluatorWriter('/some/path/to/results.csv'))
    >>> evaluator.add_label(0, 'Background')
    >>> evaluator.add_label(1, 'Structure')
    >>> evaluator.add_metric(DiceCoefficient())
    >>> evaluator.add_metric(VolumeSimilarity())
    >>> evaluator.evaluate(prediction, ground_truth, 'Patient1')
    The console output would be:
              ID       LABEL        DICE     VOLSMTY
        Patient1  Background     0.99955     0.99976
        Patient1       Nerve     0.70692     0.84278
    The results.csv would contain:
    ID;LABEL;DICE;VOLSMTY
    Patient1;Background;0.999548418549;0.999757743496
    Patient1;Nerve;0.70692469107;0.842776093884
    """

    def __init__(self, writer: IEvaluatorWriter=None):
        """Initializes a new instance of the Evaluator class.

        Args:
            writer (IEvaluatorWriter): An evaluator writer.
        """

        self.metrics = []  # list of IMetrics
        self.writers = [writer] if writer is not None else []  # list of IEvaluatorWriters
        self.labels = {}  # dictionary of label: label_str
        self.is_header_written = False

    def add_label(self, label: Union[tuple, int], description: str):
        """Adds a label with its description to the evaluation.

        Args:
            label (Union[tuple, int]): The label or a tuple of labels that should be merged.
            description (str): The label's description.
        """

        self.labels[label] = description

    def add_metric(self, metric: mtrc.IMetric):
        """Adds a metric to the evaluation.

        Args:
            metric (miapy_metric.IMetric): The metric.
        """

        self.metrics.append(metric)
        self.is_header_written = False  # header changed due to new metric

    def add_writer(self, writer: IEvaluatorWriter):
        """Adds a writer to the evaluation.

        Args:
            writer (IEvaluatorWriter): The writer.
        """

        self.writers.append(writer)
        self.is_header_written = False  # re-write header

    def evaluate(self, image: Union[sitk.Image, np.ndarray], ground_truth: Union[sitk.Image, np.ndarray],
                 evaluation_id: str):
        """Evaluates the metrics on the provided image and ground truth image.

        Args:
            image (sitk.Image): The segmented image.
            ground_truth (sitk.Image): The ground truth image.
            evaluation_id (str): The identification of the evaluation.
        """

        if not self.is_header_written:
            self.write_header()

        image_array = sitk.GetArrayFromImage(image) if isinstance(image, sitk.Image) else image
        ground_truth_array = sitk.GetArrayFromImage(ground_truth) if isinstance(ground_truth, sitk.Image) else ground_truth

        results = []  # clear results

        for label, label_str in self.labels.items():
            label_results = [evaluation_id, label_str]

            # get only current label
            predictions = np.in1d(image_array.ravel(), label, True).reshape(image_array.shape).astype(np.uint8)
            labels = np.in1d(ground_truth_array.ravel(), label, True).reshape(ground_truth_array.shape).astype(np.uint8)

            # calculate the confusion matrix for IConfusionMatrixMetric
            confusion_matrix = mtrc.ConfusionMatrix(predictions, labels)

            # flag indicating whether the images have been converted for ISimpleITKImageMetric
            converted_to_image = False
            predictions_as_image = None
            labels_as_image = None

            # calculate the metrics
            for param_index, metric in enumerate(self.metrics):
                if isinstance(metric, mtrc.IConfusionMatrixMetric):
                    metric.confusion_matrix = confusion_matrix
                elif isinstance(metric, mtrc.INumpyArrayMetric):
                    metric.ground_truth = labels
                    metric.segmentation = predictions
                elif isinstance(metric, mtrc.ISimpleITKImageMetric):
                    if not converted_to_image:
                        predictions_as_image = sitk.GetImageFromArray(predictions)
                        predictions_as_image.CopyInformation(image)
                        labels_as_image = sitk.GetImageFromArray(labels)
                        labels_as_image.CopyInformation(ground_truth)
                        converted_to_image = True

                    metric.ground_truth = labels_as_image
                    metric.segmentation = predictions_as_image

                label_results.append(metric.calculate())

            results.append(label_results)

        # write the results
        for writer in self.writers:
            writer.write(results)

    def write_header(self):
        """Writes the header."""

        header = ['ID', 'LABEL']

        for metric in self.metrics:
            header.append(str(metric))

        for writer in self.writers:
            writer.write_header(header)

        self.is_header_written = True
<commit_msg>fix(Evaluator): reference to metric module<commit_after>"""The evaluator module simplifies the evaluation of segmentation results.

The module provides the possibility of calculate several evaluation metrics in parallel and output them in any format.
"""
import csv
import os
from abc import ABCMeta, abstractmethod
from typing import Union

import numpy as np
import SimpleITK as sitk

import mialab.evaluation.metric as mtrc

class IEvaluatorWriter(metaclass=ABCMeta):
    """Represents an evaluator writer interface, which enables to write evaluation results."""

    @abstractmethod
    def write(self, data: list):
        """Writes the evaluation results.

        Args:
            data (list): The evaluation data.
        """
        raise NotImplementedError

    @abstractmethod
    def write_header(self, header: list):
        """Writes the evaluation header.

        Args:
            header (list): The evaluation header.
        """
        raise NotImplementedError

class CSVEvaluatorWriter(IEvaluatorWriter):
    """Represents a CSV (comma-separated values) evaluator writer."""

    def __init__(self, path: str):
        """Initializes a new instance of the CSVEvaluatorWriter class.

        Args:
            path (path): The file path.
        """
        super().__init__()

        self.path = path

        # check file extension
        if not self.path.lower().endswith('.csv'):
            self.path = os.path.join(self.path, '.csv')

        open(self.path, 'w', newline='')  # creates (and overrides an existing) file

    def write(self, data: list):
        """Writes the evaluation results.

        Args:
            data (list): The evaluation data.
        """

        for evaluation in data:
            self.write_line(evaluation)

    def write_header(self, header: list):
        """Writes the evaluation header.

        Args:
            header (list): The evaluation header.
        """

        self.write_line(header)

    def write_line(self, data: list):
        """Writes a line.

        Args:
            data (list): The data.
        """
        with open(self.path, 'a', newline='') as file:
            writer = csv.writer(file, delimiter=';')
            writer.writerow(data)

class ConsoleEvaluatorWriter(IEvaluatorWriter):
    """Represents a console evaluator writer."""

    def __init__(self, precision: int=3):
        """Initializes a new instance of the ConsoleEvaluatorWriter class.

        Args:
            precision (int): The float precision.
        """
        super().__init__()

        self.header = None
        self.precision = precision

    def write(self, data: list):
        """Writes the evaluation results.

        Args:
            data (list of list): The evaluation data,
                e.g. [['PATIENT1', 'BACKGROUND', 0.90], ['PATIENT1', 'TUMOR', '0.62']]
        """

        # format all floats with given precision to str
        out = []
        for line in data:
            out += [val if isinstance(val, str) else "{0:.{1}f}".format(val, self.precision) for val in line]
            out += ["\n"]

        # get length for output alignment
        length = max(len(max(out, key=len)), len(max(self.header, key=len))) + 2

        print("".join("{0:>{1}}".format(val, length) for val in self.header),
              "\n",
              "".join("{0:>{1}}".format(val, length) for val in out),
              sep='', end='')

    def write_header(self, header: list):
        """Writes the evaluation header.

        Args:
            header (list of str): The evaluation header.
        """

        self.header = header

class Evaluator:
    """Represents a metric evaluator.

    Examples:

    >>> evaluator = Evaluator(ConsoleEvaluatorWriter(5))
    >>> evaluator.add_writer(CSVEvaluatorWriter('/some/path/to/results.csv'))
    >>> evaluator.add_label(0, 'Background')
    >>> evaluator.add_label(1, 'Structure')
    >>> evaluator.add_metric(mtrc.DiceCoefficient())
    >>> evaluator.add_metric(mtrc.VolumeSimilarity())
    >>> evaluator.evaluate(prediction, ground_truth, 'Patient1')
    The console output would be:
              ID       LABEL        DICE     VOLSMTY
        Patient1  Background     0.99955     0.99976
        Patient1       Nerve     0.70692     0.84278
    The results.csv would contain:
    ID;LABEL;DICE;VOLSMTY
    Patient1;Background;0.999548418549;0.999757743496
    Patient1;Nerve;0.70692469107;0.842776093884
    """

    def __init__(self, writer: IEvaluatorWriter=None):
        """Initializes a new instance of the Evaluator class.

        Args:
            writer (IEvaluatorWriter): An evaluator writer.
        """

        self.metrics = []  # list of IMetrics
        self.writers = [writer] if writer is not None else []  # list of IEvaluatorWriters
        self.labels = {}  # dictionary of label: label_str
        self.is_header_written = False

    def add_label(self, label: Union[tuple, int], description: str):
        """Adds a label with its description to the evaluation.

        Args:
            label (Union[tuple, int]): The label or a tuple of labels that should be merged.
            description (str): The label's description.
        """

        self.labels[label] = description

    def add_metric(self, metric: mtrc.IMetric):
        """Adds a metric to the evaluation.

        Args:
            metric (mtrc.IMetric): The metric.
        """

        self.metrics.append(metric)
        self.is_header_written = False  # header changed due to new metric

    def add_writer(self, writer: IEvaluatorWriter):
        """Adds a writer to the evaluation.

        Args:
            writer (IEvaluatorWriter): The writer.
        """

        self.writers.append(writer)
        self.is_header_written = False  # re-write header

    def evaluate(self, image: Union[sitk.Image, np.ndarray], ground_truth: Union[sitk.Image, np.ndarray],
                 evaluation_id: str):
        """Evaluates the metrics on the provided image and ground truth image.

        Args:
            image (sitk.Image): The segmented image.
            ground_truth (sitk.Image): The ground truth image.
            evaluation_id (str): The identification of the evaluation.
        """

        if not self.is_header_written:
            self.write_header()

        image_array = sitk.GetArrayFromImage(image) if isinstance(image, sitk.Image) else image
        ground_truth_array = sitk.GetArrayFromImage(ground_truth) if isinstance(ground_truth, sitk.Image) else ground_truth

        results = []  # clear results

        for label, label_str in self.labels.items():
            label_results = [evaluation_id, label_str]

            # get only current label
            predictions = np.in1d(image_array.ravel(), label, True).reshape(image_array.shape).astype(np.uint8)
            labels = np.in1d(ground_truth_array.ravel(), label, True).reshape(ground_truth_array.shape).astype(np.uint8)

            # calculate the confusion matrix for IConfusionMatrixMetric
            confusion_matrix = mtrc.ConfusionMatrix(predictions, labels)

            # flag indicating whether the images have been converted for ISimpleITKImageMetric
            converted_to_image = False
            predictions_as_image = None
            labels_as_image = None

            # calculate the metrics
            for param_index, metric in enumerate(self.metrics):
                if isinstance(metric, mtrc.IConfusionMatrixMetric):
                    metric.confusion_matrix = confusion_matrix
                elif isinstance(metric, mtrc.INumpyArrayMetric):
                    metric.ground_truth = labels
                    metric.segmentation = predictions
                elif isinstance(metric, mtrc.ISimpleITKImageMetric):
                    if not converted_to_image:
                        predictions_as_image = sitk.GetImageFromArray(predictions)
                        predictions_as_image.CopyInformation(image)
                        labels_as_image = sitk.GetImageFromArray(labels)
                        labels_as_image.CopyInformation(ground_truth)
                        converted_to_image = True

                    metric.ground_truth = labels_as_image
                    metric.segmentation = predictions_as_image

                label_results.append(metric.calculate())

            results.append(label_results)

        # write the results
        for writer in self.writers:
            writer.write(results)

    def write_header(self):
        """Writes the header."""

        header = ['ID', 'LABEL']

        for metric in self.metrics:
            header.append(str(metric))

        for writer in self.writers:
            writer.write_header(header)

        self.is_header_written = True
<|endoftext|>


<commit_before>'''
Python client for the Mapbox Geocoder service.
'''

import json
import requests
from mapbox import Geocoder
from cartodb_services.metrics import Traceable
from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools.qps import qps_retry

GEOCODER_NAME = 'geocoder_name'
EPHEMERAL_GEOCODER = 'mapbox.places'
PERMANENT_GEOCODER = 'mapbox.places-permanent'
DEFAULT_GEOCODER = EPHEMERAL_GEOCODER

ENTRY_FEATURES = 'features'
ENTRY_CENTER = 'center'
ENTRY_GEOMETRY = 'geometry'
ENTRY_COORDINATES = 'coordinates'
ENTRY_TYPE = 'type'
TYPE_POINT = 'Point'

class MapboxGeocoder(Traceable):
    '''
    Python wrapper for the Mapbox Geocoder service.
    '''

    def __init__(self, token, logger, service_params=None):
        service_params = service_params or {}
        self._token = token
        self._logger = logger
        self._geocoder_name = service_params.get(GEOCODER_NAME,
                                                 EPHEMERAL_GEOCODER)
        self._geocoder = Geocoder(access_token=self._token,
                                  name=self._geocoder_name)

    def _parse_geocoder_response(self, response):
        json_response = json.loads(response)

        if json_response:
            feature = json_response[ENTRY_FEATURES][0]

            return self._extract_lng_lat_from_feature(feature)
        else:
            return []

    def _extract_lng_lat_from_feature(self, feature):
        geometry = feature[ENTRY_GEOMETRY]
        if geometry[ENTRY_TYPE] == TYPE_POINT:
            location = geometry[ENTRY_COORDINATES]
        else:
            location = feature[ENTRY_CENTER]

        longitude = location[0]
        latitude = location[1]
        return [longitude, latitude]

    @qps_retry(qps=10)
    def geocode(self, searchtext, city=None, state_province=None,
                country=None):
        address = [searchtext]
        if city:
            address.append(city)
        if state_province:
            address.append(state_province)

        country = [country] if country else None

        try:
            response = self._geocoder.forward(address=', '.join(address).decode('utf-8'),
                                              country=country,
                                              limit=1)

            if response.status_code == requests.codes.ok:
                return self._parse_geocoder_response(response.text)
            elif response.status_code == requests.codes.bad_request:
                return []
            else:
                raise ServiceException(response.status_code, response)
        except requests.Timeout as te:
            # In case of timeout we want to stop the job because the server
            # could be down
            self._logger.error('Timeout connecting to Mapbox geocoding server',
                               te)
            raise ServiceException('Error geocoding {0} using Mapbox'.format(
                searchtext), None)
        except requests.ConnectionError as ce:
            # Don't raise the exception to continue with the geocoding job
            self._logger.error('Error connecting to Mapbox geocoding server',
                               exception=ce)
            return []
<commit_msg>Going green with empty responses<commit_after>'''
Python client for the Mapbox Geocoder service.
'''

import json
import requests
from mapbox import Geocoder
from cartodb_services.metrics import Traceable
from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools.qps import qps_retry

GEOCODER_NAME = 'geocoder_name'
EPHEMERAL_GEOCODER = 'mapbox.places'
PERMANENT_GEOCODER = 'mapbox.places-permanent'
DEFAULT_GEOCODER = EPHEMERAL_GEOCODER

ENTRY_FEATURES = 'features'
ENTRY_CENTER = 'center'
ENTRY_GEOMETRY = 'geometry'
ENTRY_COORDINATES = 'coordinates'
ENTRY_TYPE = 'type'
TYPE_POINT = 'Point'

class MapboxGeocoder(Traceable):
    '''
    Python wrapper for the Mapbox Geocoder service.
    '''

    def __init__(self, token, logger, service_params=None):
        service_params = service_params or {}
        self._token = token
        self._logger = logger
        self._geocoder_name = service_params.get(GEOCODER_NAME,
                                                 EPHEMERAL_GEOCODER)
        self._geocoder = Geocoder(access_token=self._token,
                                  name=self._geocoder_name)

    def _parse_geocoder_response(self, response):
        json_response = json.loads(response)

        if json_response and json_response[ENTRY_FEATURES]:
            feature = json_response[ENTRY_FEATURES][0]

            return self._extract_lng_lat_from_feature(feature)
        else:
            return []

    def _extract_lng_lat_from_feature(self, feature):
        geometry = feature[ENTRY_GEOMETRY]
        if geometry[ENTRY_TYPE] == TYPE_POINT:
            location = geometry[ENTRY_COORDINATES]
        else:
            location = feature[ENTRY_CENTER]

        longitude = location[0]
        latitude = location[1]
        return [longitude, latitude]

    @qps_retry(qps=10)
    def geocode(self, searchtext, city=None, state_province=None,
                country=None):
        address = [searchtext]
        if city:
            address.append(city)
        if state_province:
            address.append(state_province)

        country = [country] if country else None

        try:
            response = self._geocoder.forward(address=', '.join(address).decode('utf-8'),
                                              country=country,
                                              limit=1)

            if response.status_code == requests.codes.ok:
                return self._parse_geocoder_response(response.text)
            elif response.status_code == requests.codes.bad_request:
                return []
            else:
                raise ServiceException(response.status_code, response)
        except requests.Timeout as te:
            # In case of timeout we want to stop the job because the server
            # could be down
            self._logger.error('Timeout connecting to Mapbox geocoding server',
                               te)
            raise ServiceException('Error geocoding {0} using Mapbox'.format(
                searchtext), None)
        except requests.ConnectionError as ce:
            # Don't raise the exception to continue with the geocoding job
            self._logger.error('Error connecting to Mapbox geocoding server',
                               exception=ce)
            return []
<|endoftext|>


<commit_before>"""
.. module:: verzamelend
    :platform: Unix
    :synopsis: Verzamelend package.

.. moduleauthor:: Pedro Salgado <steenzout@ymail.com>
"""

from __future__ import absolute_import

import collectd

import logging

from verzamelend.version import __version__

LOGGER = logging.getLogger(__name__)

TYPE_ABSOLUTE = 'absolute'
TYPE_COUNTER = 'counter'
TYPE_DERIVE = 'derive'
TYPE_GAUGE = 'gauge'

def register_callbacks(plugin):
    """
    Registers all Collectd callbacks for the given plugin.

    This method will look for the following static methods to register callbacks:
    - configCallback() for register_config()
    - flushCallback() for register_flush
    - initCallback() for register_init()
    - logCallback() for register_log()
    - notificationCallback() for register_notification()
    - readCallback() for register_read()
    - shutdownCallback() for register_shutdown()
    - writeCallback() for register_write()

    :param plugin: the plugin instance which have the callbacks defined.
    :type plugin: object (verzamelend.Plugin)
    """
    if plugin is None:
        error_message = 'register_callbacks() plugin argument is None!'
        LOGGER.error(error_message)
        raise ValueError(error_message)

    LOGGER.info('register_callbacks for plugin %s', plugin.name)
    collectd.register_config(plugin.configCallback)
    collectd.register_flush(plugin.flushCallback)
    collectd.register_init(plugin.initCallback)
    collectd.register_log(plugin.logCallback)
    collectd.register_notification(plugin.notificationCallback)
    collectd.register_read(plugin.readCallback)
    collectd.register_shutdown(plugin.shutdownCallback)
    collectd.register_write(plugin.writeCallback)

class Configuration(object):
    """
    Class to hold Collectd plugin configuration.
    """

    LOGGER = None

    def __init__(self, config):
        """
        Initializes a Configuration instance.

        :param config: the Collectd configuration for this plugin.
        :type config: object (Config)
        """
        self.values = dict([(node.key, node.value) for node in config.children])

    def getBool(self, parameter, position=1):
        """
        Returns the boolean value of the given parameter.

        :param parameter: the configuration parameter.
        :type parameter: str
        :param position: if parameter values are a list, the position of the value being retrieved.
        :type position: int
        """
        value = self.getValue(parameter, position)
        if isinstance(value, bool):
            return value

        if value is not None and isinstance(value, basestring) and value.upper() == 'TRUE':
            return True
        else:
            return False

    def getInt(self, parameter, position=1):
        """
        Returns the integer value of the given parameter.

        :param parameter: the configuration parameter.
        :type parameter: str
        :param position: if parameter values are a list, the position of the value being retrieved.
        :type position: int
        """
        return int(self.getValue(parameter, position))

    def getStr(self, parameter, position=1):
        """
        Returns the string value of the given parameter.

        :param parameter: the configuration parameter.
        :type parameter: str
        :param position: if parameter values are a list, the position of the value being retrieved.
        :type position: int
        """
        value = self.getValue(parameter, position)
        if isinstance(value, basestring):
            return value
        else:
            return str(value)

    def getValue(self, parameter, position=None):
        """
        Returns the value of the given parameter.

        :param parameter: the configuration parameter.
        :type parameter: str
        :param position: if parameter values are a list, the position of the value being retrieved
                         (1 means 1st element on the list of values).
        :type position: int
        """
        if parameter is None:
            error_message = 'getValue() parameter argument is None!'
            Configuration.LOGGER.error(error_message)
            raise ValueError(error_message)

        if position is None or position <= 0:
            error_message = 'getValue() position argument %s must be a positive integer!'
            Configuration.LOGGER.error(error_message, position)
            raise ValueError(error_message % position)

        if position is None:
            self.values[parameter][position - 1]
        else:
            return self.values[parameter]

Configuration.LOGGER = logging.getLogger('%s.%s' % (Configuration.__module__, Configuration.__name__))

class Plugin(object):
    """
    Class to build Collectd plugins.
    """

    LOGGER = None

    def __init__(self, name):
        """
        Constructs a Plugin instance.

        :param name: the plugin name.
        :type name: str
        """
        self.name = name

    @staticmethod
    def configCallback(config):
        """
        Plugin callback for the configuration event.

        :param config: the Collectd configuration for this plugin.
        :type config: object (Config)
        """
        Plugin.LOGGER.info('configCallback()')
        configuration = Configuration(config)

    @staticmethod
    def initCallback():
        """
        Plugin callback for the init event.
        """
        Plugin.LOGGER.info('initCallback()')

    @staticmethod
    def flushCallback():
        """
        Plugin callback for flush events.
        """
        Plugin.LOGGER.info('flushCallback()')

    @staticmethod
    def logCallback(level, message):
        """
        Plugin callback for log events.

        :param level: the severity level of the message.
        :param message: the message to be sent to the log.
        :type message: str
        """
        Plugin.LOGGER.info('logCallback()')

    @staticmethod
    def notificationCallback(notification):
        """
        Plugin callback for notification events.

        :param notification: the Collectd notification.
        :type notification: object (Notification)
        """
        Plugin.LOGGER.info('notificationCallback()')

    @staticmethod
    def readCallback(data=None):
        """
        Plugin callback for read events.
        """
        Plugin.LOGGER.info('readCallback()')

    @staticmethod
    def shutdownCallback():
        """
        Plugin callback for the shutdown event.
        """
        Plugin.LOGGER.info('shutdownCallback()')

    @staticmethod
    def writeCallback(measurements, data=None):
        """
        Plugin callback for write events.
        """
        Plugin.LOGGER.info('writeCallback()')

Plugin.LOGGER = logging.getLogger('%s.%s' % (Plugin.__module__, Plugin.__name__))
<commit_msg>if position argument is passed then configuration item is expected to be a list or tuple.<commit_after>"""
.. module:: verzamelend
    :platform: Unix
    :synopsis: Verzamelend package.

.. moduleauthor:: Pedro Salgado <steenzout@ymail.com>
"""

from __future__ import absolute_import

import collectd

import logging

from verzamelend.version import __version__

LOGGER = logging.getLogger(__name__)

TYPE_ABSOLUTE = 'absolute'
TYPE_COUNTER = 'counter'
TYPE_DERIVE = 'derive'
TYPE_GAUGE = 'gauge'

def register_callbacks(plugin):
    """
    Registers all Collectd callbacks for the given plugin.

    This method will look for the following static methods to register callbacks:
    - configCallback() for register_config()
    - flushCallback() for register_flush
    - initCallback() for register_init()
    - logCallback() for register_log()
    - notificationCallback() for register_notification()
    - readCallback() for register_read()
    - shutdownCallback() for register_shutdown()
    - writeCallback() for register_write()

    :param plugin: the plugin instance which have the callbacks defined.
    :type plugin: object (verzamelend.Plugin)
    """
    if plugin is None:
        error_message = 'register_callbacks() plugin argument is None!'
        LOGGER.error(error_message)
        raise ValueError(error_message)

    LOGGER.info('register_callbacks for plugin %s', plugin.name)
    collectd.register_config(plugin.configCallback)
    collectd.register_flush(plugin.flushCallback)
    collectd.register_init(plugin.initCallback)
    collectd.register_log(plugin.logCallback)
    collectd.register_notification(plugin.notificationCallback)
    collectd.register_read(plugin.readCallback)
    collectd.register_shutdown(plugin.shutdownCallback)
    collectd.register_write(plugin.writeCallback)

class Configuration(object):
    """
    Class to hold Collectd plugin configuration.
    """

    LOGGER = None

    def __init__(self, config):
        """
        Initializes a Configuration instance.

        :param config: the Collectd configuration for this plugin.
        :type config: object (Config)
        """
        self.values = dict([(node.key, node.value) for node in config.children])

    def getBool(self, parameter, position=1):
        """
        Returns the boolean value of the given parameter.

        :param parameter: the configuration parameter.
        :type parameter: str
        :param position: if parameter values are a list, the position of the value being retrieved.
        :type position: int
        """
        value = self.getValue(parameter, position)
        if isinstance(value, bool):
            return value

        if value is not None and isinstance(value, basestring) and value.upper() == 'TRUE':
            return True
        else:
            return False

    def getInt(self, parameter, position=1):
        """
        Returns the integer value of the given parameter.

        :param parameter: the configuration parameter.
        :type parameter: str
        :param position: if parameter values are a list, the position of the value being retrieved.
        :type position: int
        """
        return int(self.getValue(parameter, position))

    def getStr(self, parameter, position=1):
        """
        Returns the string value of the given parameter.

        :param parameter: the configuration parameter.
        :type parameter: str
        :param position: if parameter values are a list, the position of the value being retrieved.
        :type position: int
        """
        value = self.getValue(parameter, position)
        if isinstance(value, basestring):
            return value
        else:
            return str(value)

    def getValue(self, parameter, position=None):
        """
        Returns the value of the given parameter.

        :param parameter: the configuration parameter.
        :type parameter: str
        :param position: if parameter values are a list, the position of the value being retrieved
                         (1 means 1st element on the list of values).
        :type position: int
        """
        if parameter is None:
            error_message = 'getValue() parameter argument is None!'
            Configuration.LOGGER.error(error_message)
            raise ValueError(error_message)

        if position <= 0:
            error_message = 'getValue() position argument %s must be a positive integer!'
            Configuration.LOGGER.error(error_message, position)
            raise ValueError(error_message % position)

        if position is not None:
            self.values[parameter][position - 1]
        else:
            return self.values[parameter]

Configuration.LOGGER = logging.getLogger('%s.%s' % (Configuration.__module__, Configuration.__name__))

class Plugin(object):
    """
    Class to build Collectd plugins.
    """

    LOGGER = None

    def __init__(self, name):
        """
        Constructs a Plugin instance.

        :param name: the plugin name.
        :type name: str
        """
        self.name = name

    @staticmethod
    def configCallback(config):
        """
        Plugin callback for the configuration event.

        :param config: the Collectd configuration for this plugin.
        :type config: object (Config)
        """
        Plugin.LOGGER.info('configCallback()')
        configuration = Configuration(config)

    @staticmethod
    def initCallback():
        """
        Plugin callback for the init event.
        """
        Plugin.LOGGER.info('initCallback()')

    @staticmethod
    def flushCallback():
        """
        Plugin callback for flush events.
        """
        Plugin.LOGGER.info('flushCallback()')

    @staticmethod
    def logCallback(level, message):
        """
        Plugin callback for log events.

        :param level: the severity level of the message.
        :param message: the message to be sent to the log.
        :type message: str
        """
        Plugin.LOGGER.info('logCallback()')

    @staticmethod
    def notificationCallback(notification):
        """
        Plugin callback for notification events.

        :param notification: the Collectd notification.
        :type notification: object (Notification)
        """
        Plugin.LOGGER.info('notificationCallback()')

    @staticmethod
    def readCallback(data=None):
        """
        Plugin callback for read events.
        """
        Plugin.LOGGER.info('readCallback()')

    @staticmethod
    def shutdownCallback():
        """
        Plugin callback for the shutdown event.
        """
        Plugin.LOGGER.info('shutdownCallback()')

    @staticmethod
    def writeCallback(measurements, data=None):
        """
        Plugin callback for write events.
        """
        Plugin.LOGGER.info('writeCallback()')

Plugin.LOGGER = logging.getLogger('%s.%s' % (Plugin.__module__, Plugin.__name__))
<|endoftext|>


<commit_before>import os
import git
import json
import shutil
import glob
import copy
import typing
import git
import yaml

from typing import List
from repairbox.build import BuildInstructions
from repairbox.artefact import Artefact

class RepairBox(object):
    """
    Used to interact with and manage a local RepairBox installation.
    """
    def __init__(self, path=None) -> None:
        """
        Creates a new RepairBox installation manager.

        Args:
            path: the absolute path of a RepairBox installation on this machine.
                If unspecified, the value of the environmental variable
                :code:`REPAIRBOX_PATH` will be used, unless unspecified, in
                which case :code:`./${HOME}/.repairbox` will be used instead.
        """
        # TODO support windows
        if path is None:
            default_path = os.path.join(os.environ.get('HOME'), '.repairbox')
            path = os.environ.get('REPAIRBOX_PATH', default_path)

        # ensure dir exists
        if not os.path.exists(path):
            os.makedirs(path)

        self.__path = path
        self.__sources = SourceManager(self)
        self.__artefacts = ArtefactManager(self)
        self.__sources.reload()

    @property
    def path(self) -> str:
        """
        The absolute path to the local installation of RepairBox's sources.
        """
        return self.__path

    def rescan(self):
        print("rescanning...")

    @property
    def sources(self):
        """
        The sources registered with this RepairBox installation.
        """
        return self.__sources

    @property
    def artefacts(self):
        """
        The artefacts registered with this RepairBox installation.
        """
        return self.__artefacts

class Source(object):
    @staticmethod
    def download(manager: 'SourceManager', url: str) -> 'Source':
        """
        Downloads a source from a given URL to disk.
        """
        assert not os.path.exists(abs_path), "already downloaded source: {}".format(url)
        git.Repo.clone_from(url, abs_path)
        return Source(manager, url)

    @staticmethod
    def url_to_rel_path(url: str) -> str:
        rel_path = url.replace('https://', '')
        rel_path = rel_path.replace('/', '_')
        rel_path = rel_path.replace('.', '_')
        return rel_path

    @staticmethod
    def url_to_abs_path(manager: 'SourceManager', url: str) -> str:
        rel_path = Source.url_to_rel_path(url)
        return os.path.join(manager.path, rel_path)

    def __init__(self, manager: 'SourceManager', url: str) -> None:
        self.__manager = manager
        self.__url = url
        self.__artefacts = {}
        self.__dependencies = {}
        self.__repo = git.Repo(self.abs_path)

        self.scan()

    @property
    def name(self) -> str:
        """
        The unique name of this source.
        """
        return self.__name

    @property
    def version(self) -> str:
        """
        The current version of this source, given as the first eight characters
        of its current revision.
        """
        sha = self.__repo.head.object.hexsha
        return self.__repo.git.rev_parse(sha, short=8)

    def scan(self) -> None:
        # determine the name of this source
        manifest_fn = os.path.join(self.abs_path, '.repairbox.yml')
        assert os.path.isfile(manifest_fn), "missing manifest file."
        with open(manifest_fn, 'r') as f:
            yml = yaml.load(f)
        self.__name = yml['dataset']['name']

        # find all dependencies
        fns = '{}/**/*.dependency.yaml'.format(self.abs_path)
        for fn in glob.iglob(fns, recursive=True):
            dep = BuildInstructions.from_file(self, fn)
            self.__dependencies[dep.tag] = dep

        # find all artefacts
        fns = '{}/**/*.artefact.yaml'.format(self.abs_path)
        for fn in glob.iglob(fns, recursive=True):
            artefact = Artefact.from_file(self, fn)
            self.__artefacts[artefact.identifier] = artefact

    def update(self) -> None:
        """
        Downloads any updates to the files for this source.
        """
        origin = self.__repo.remotes.origin
        origin.pull()
        self.scan()

    def remove(self) -> None:
        """
        Removes the files for this source from disk, and uninstalls all
        associated Docker images. This should only be called by SourceManager.
        """
        for artefact in self.artefacts:
            artefact.uninstall(force=True)

        for dep in self.__dependencies.values():
            dep.uninstall(force=True)

        shutil.rmtree(self.abs_path)

    def contains(self, identifier: str) -> None:
        """
        Checks whether an artefact with a given identifier is provided by this
        source.
        """
        return identifier in self.__artefacts

    def __getitem__(self, key):
        return self.__artefacts[key]

    @property
    def artefacts(self) -> List[Artefact]:
        return list(self.__artefacts.values())

    @property
    def dependencies(self):
        return copy.copy(self.__dependencies)

    @property
    def manager(self) -> 'SourceManager':
        return self.__manager

    @property
    def url(self) -> str:
        return self.__url

    @property
    def rel_path(self) -> str:
        """
        Returns the location of this source, relative to the sources directory.
        """
        return Source.url_to_rel_path(self.url)

    @property
    def abs_path(self) -> str:
        """
        Returns the absolute path to this source.
        """
        return Source.url_to_abs_path(self.manager, self.url)

class SourceManager(object):
    """
    Used to access and manage the sources registered with a local RepairBox
    installation.
    """
    def __init__(self, manager: 'RepairBox') -> None:
        self.__path = os.path.join(manager.path, 'sources')
        self.__manifest_fn = \
            os.path.join(self.__path, 'sources.manifest.json')
        self.__sources = {}

    @property
    def path(self):
        """
        The path to the sources directory on disk.
        """
        return self.__path

    def reload(self) -> None:
        """
        Reloads all the manifest and build files associated with registered
        sources. This method is called (internally) after adding, removing,
        or updating sources.
        """
        if not os.path.exists(self.__manifest_fn):
            self.__sources = {}
            return

        with open(self.__manifest_fn, 'r') as f:
            srcs = json.load(f)

        assert isinstance(srcs, list)
        self.__sources = {s: Source(self, s) for s in srcs}

    def __write(self) -> None:
        with open(self.__manifest_fn, 'w') as f:
            srcs = list(self.__sources.keys())
            json.dump(srcs, f, indent=2)

    def add(self, src: str) -> None:
        """
        Registers a new source with this local installation.
        """
        assert src != ""
        if src in self.__sources:
            raise Exception("source already exists: {}".format(src)) # TODO custom Error

        src = Source.download(src)

        # update the sources file
        self.__sources[src.url] = src.rel_path
        self.__write()

    def remove(self, src: str) -> None:
        """
        Removes an existing source, given by its name. The removal process
        destroys the local copies of the manifest and build files for this
        source, and uninstalls all of its associated images.

        Args:
            name:    the name of the source that should be removed.
        """
        assert src != ""
        if src not in self.__sources:
            raise Exception("source not found: {}".format(src)) # TODO custom Error

        self.__sources[src].remove()
        del self.__sources[src]
        self.__write()

    def update(self) -> None:
        """
        Downloads any available updates for all (installed) sources.
        """
        for src in self.__sources.values():
            src.update()

    @property
    def sources(self) -> List[Source]:
        """
        A pseudo-immutable list of sources associated with this local
        installation. I.e., any changes to this list are not permanent.
        """
        return self.__sources.values()

    def __getitem__(self, name: str) -> Source:
        """
        Fetches a registered source by its name.

        Args:
            name:   the name of the source.

        Example:

            .. code-block:: python

                # fetch the ManyBugs source
                rbx = RepairBox()
                src = rbx.sources['manybugs']
        """
        return self.__sources[name]

class ArtefactManager(object):
    class ArtefactIterator(object):
        def __init__(self, sources):
            self.__sources = list(sources)
            self.__artefacts = []

        def __next__(self):
            if not self.__artefacts:
                if not self.__sources:
                    raise StopIteration
                src = self.__sources.pop()
                self.__artefacts += src.artefacts
                return self.__next__()
            return self.__artefacts.pop()

    def __init__(self, manager):
        self.__manager = manager

    def __getitem__(self, name):
        for src in self.__manager.sources.sources:
            if src.contains(name):
                return src[name]
        raise IndexError('artefact not found: {}'.format(name))

    def __iter__(self):
        return ArtefactManager.ArtefactIterator(self.__manager.sources.sources)
<commit_msg>docs: updated SourceManager<commit_after>import os
import git
import json
import shutil
import glob
import copy
import typing
import git
import yaml

from typing import List
from repairbox.build import BuildInstructions
from repairbox.artefact import Artefact

class RepairBox(object):
    """
    Used to interact with and manage a local RepairBox installation.
    """
    def __init__(self, path=None) -> None:
        """
        Creates a new RepairBox installation manager.

        Args:
            path: the absolute path of a RepairBox installation on this machine.
                If unspecified, the value of the environmental variable
                :code:`REPAIRBOX_PATH` will be used, unless unspecified, in
                which case :code:`./${HOME}/.repairbox` will be used instead.
        """
        # TODO support windows
        if path is None:
            default_path = os.path.join(os.environ.get('HOME'), '.repairbox')
            path = os.environ.get('REPAIRBOX_PATH', default_path)

        # ensure dir exists
        if not os.path.exists(path):
            os.makedirs(path)

        self.__path = path
        self.__sources = SourceManager(self)
        self.__artefacts = ArtefactManager(self)
        self.__sources.reload()

    @property
    def path(self) -> str:
        """
        The absolute path to the local installation of RepairBox's sources.
        """
        return self.__path

    def rescan(self):
        print("rescanning...")

    @property
    def sources(self):
        """
        The sources registered with this RepairBox installation.
        """
        return self.__sources

    @property
    def artefacts(self):
        """
        The artefacts registered with this RepairBox installation.
        """
        return self.__artefacts

class Source(object):
    @staticmethod
    def download(manager: 'SourceManager', url: str) -> 'Source':
        """
        Downloads a source from a given URL to disk.
        """
        assert not os.path.exists(abs_path), "already downloaded source: {}".format(url)
        git.Repo.clone_from(url, abs_path)
        return Source(manager, url)

    @staticmethod
    def url_to_rel_path(url: str) -> str:
        rel_path = url.replace('https://', '')
        rel_path = rel_path.replace('/', '_')
        rel_path = rel_path.replace('.', '_')
        return rel_path

    @staticmethod
    def url_to_abs_path(manager: 'SourceManager', url: str) -> str:
        rel_path = Source.url_to_rel_path(url)
        return os.path.join(manager.path, rel_path)

    def __init__(self, manager: 'SourceManager', url: str) -> None:
        self.__manager = manager
        self.__url = url
        self.__artefacts = {}
        self.__dependencies = {}
        self.__repo = git.Repo(self.abs_path)

        self.scan()

    @property
    def name(self) -> str:
        """
        The unique name of this source.
        """
        return self.__name

    @property
    def version(self) -> str:
        """
        The current version of this source, given as the first eight characters
        of its current revision.
        """
        sha = self.__repo.head.object.hexsha
        return self.__repo.git.rev_parse(sha, short=8)

    def scan(self) -> None:
        # determine the name of this source
        manifest_fn = os.path.join(self.abs_path, '.repairbox.yml')
        assert os.path.isfile(manifest_fn), "missing manifest file."
        with open(manifest_fn, 'r') as f:
            yml = yaml.load(f)
        self.__name = yml['dataset']['name']

        # find all dependencies
        fns = '{}/**/*.dependency.yaml'.format(self.abs_path)
        for fn in glob.iglob(fns, recursive=True):
            dep = BuildInstructions.from_file(self, fn)
            self.__dependencies[dep.tag] = dep

        # find all artefacts
        fns = '{}/**/*.artefact.yaml'.format(self.abs_path)
        for fn in glob.iglob(fns, recursive=True):
            artefact = Artefact.from_file(self, fn)
            self.__artefacts[artefact.identifier] = artefact

    def update(self) -> None:
        """
        Downloads any updates to the files for this source.
        """
        origin = self.__repo.remotes.origin
        origin.pull()
        self.scan()

    def remove(self) -> None:
        """
        Removes the files for this source from disk, and uninstalls all
        associated Docker images. This should only be called by SourceManager.
        """
        for artefact in self.artefacts:
            artefact.uninstall(force=True)

        for dep in self.__dependencies.values():
            dep.uninstall(force=True)

        shutil.rmtree(self.abs_path)

    def contains(self, identifier: str) -> None:
        """
        Checks whether an artefact with a given identifier is provided by this
        source.
        """
        return identifier in self.__artefacts

    def __getitem__(self, key):
        return self.__artefacts[key]

    @property
    def artefacts(self) -> List[Artefact]:
        return list(self.__artefacts.values())

    @property
    def dependencies(self):
        return copy.copy(self.__dependencies)

    @property
    def manager(self) -> 'SourceManager':
        return self.__manager

    @property
    def url(self) -> str:
        return self.__url

    @property
    def rel_path(self) -> str:
        """
        Returns the location of this source, relative to the sources directory.
        """
        return Source.url_to_rel_path(self.url)

    @property
    def abs_path(self) -> str:
        """
        Returns the absolute path to this source.
        """
        return Source.url_to_abs_path(self.manager, self.url)

class SourceManager(object):
    """
    Used to access and manage all sources registered with a local RepairBox
    installation.
    """
    def __init__(self, manager: 'RepairBox') -> None:
        self.__path = os.path.join(manager.path, 'sources')
        self.__manifest_fn = \
            os.path.join(self.__path, 'sources.manifest.json')
        self.__sources = {}

    @property
    def path(self):
        """
        The path to the sources directory on disk.
        """
        return self.__path

    def reload(self) -> None:
        """
        Reloads all the manifest and build files associated with registered
        sources. This method is called (internally) after adding, removing,
        or updating sources.
        """
        if not os.path.exists(self.__manifest_fn):
            self.__sources = {}
            return

        with open(self.__manifest_fn, 'r') as f:
            srcs = json.load(f)

        assert isinstance(srcs, list)
        self.__sources = {s: Source(self, s) for s in srcs}

    def __write(self) -> None:
        with open(self.__manifest_fn, 'w') as f:
            srcs = list(self.__sources.keys())
            json.dump(srcs, f, indent=2)

    def add(self, url: str) -> None:
        """
        Registers a new source with this local installation.

        Args:
            url:    the URL of the Git repository for this source.

        Raises:
            Exception: if an existing, installed source uses the given URL.
        """
        assert src != ""
        if src in self.__sources:
            raise Exception("source already exists: {}".format(src)) # TODO custom Error

        src = Source.download(src)

        # update the sources file
        self.__sources[src.url] = src.rel_path
        self.__write()

    def remove(self, src: str) -> None:
        """
        Removes an existing source, given by its name. The removal process
        destroys the local copies of the manifest and build files for this
        source, and uninstalls all of its associated images.

        Args:
            name:    the name of the source that should be removed.

        Raises:
            Exception:  if no installed source exists with the given URL.
        """
        assert src != ""
        if src not in self.__sources:
            raise Exception("source not found: {}".format(src)) # TODO custom Error

        self.__sources[src].remove()
        del self.__sources[src]
        self.__write()

    def update(self) -> None:
        """
        Downloads any available updates for all (installed) sources.
        """
        for src in self.__sources.values():
            src.update()

    @property
    def sources(self) -> List[Source]:
        """
        A pseudo-immutable list of sources associated with this local
        installation. I.e., any changes to this list are not permanent.
        """
        return self.__sources.values()

    def __getitem__(self, name: str) -> Source:
        """
        Fetches a registered source by its name.

        Args:
            name:   the name of the source.

        Example:

            .. code-block:: python

                # fetch the ManyBugs source
                rbx = RepairBox()
                src = rbx.sources['manybugs']
        """
        return self.__sources[name]

class ArtefactManager(object):
    class ArtefactIterator(object):
        def __init__(self, sources):
            self.__sources = list(sources)
            self.__artefacts = []

        def __next__(self):
            if not self.__artefacts:
                if not self.__sources:
                    raise StopIteration
                src = self.__sources.pop()
                self.__artefacts += src.artefacts
                return self.__next__()
            return self.__artefacts.pop()

    def __init__(self, manager):
        self.__manager = manager

    def __getitem__(self, name):
        for src in self.__manager.sources.sources:
            if src.contains(name):
                return src[name]
        raise IndexError('artefact not found: {}'.format(name))

    def __iter__(self):
        return ArtefactManager.ArtefactIterator(self.__manager.sources.sources)
<|endoftext|>