<commit_before>import os
import Ska.arc5gl
import tempfile
from glob import glob
import re
import logging
import shutil
import Ska.DBI
#from Ska.Shell import bash
import numpy as np
from Chandra.Time import DateTime
import Ska.File

# archive_dir set by main()
archive_dir = None
arc5 = Ska.arc5gl.Arc5gl()
aca_db = Ska.DBI.DBI(dbi='sybase', server='sybase', user='aca_read')
apstat = Ska.DBI.DBI(dbi='sybase', server='sqlocc', database='axafapstat')

logger = logging.getLogger('asp1 fetch')
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

# borrowed from eng_archive
archfiles_hdr_cols = ('tstart', 'tstop', 'caldbver', 'content',
                      'ascdsver', 'revision', 'date')

#import itertools
import pyfits

def get_arch_info(i, f, archfiles, db):

    """Read filename ``f`` with index ``i`` (position within list of
    filenames).  The file has type ``filetype`` and will be added to
    MSID file at row index ``row``.  ``colnames`` is the list of
    column names for the content type (not used here).
    """

    filename = os.path.basename(f)

    # Read FITS archive file and accumulate data into dats list and
    # header into headers dict
    logger.debug('Reading (%d / %d) %s' % (i, len(archfiles), filename))
    hdus = pyfits.open(f)
    hdu = hdus[1]

    # Accumlate relevant info about archfile that will be ingested into
    # MSID h5 files.  Commit info before h5 ingest so if there is a failure
    # the needed info will be available to do the repair.
    archfiles_row = dict((x, hdu.header.get(x.upper()))
                         for x in archfiles_hdr_cols)
    archfiles_row['checksum'] = hdu._checksum
    archfiles_row['filename'] = filename
    archfiles_row['filetime'] = int(
        re.search(r'(\d+)', archfiles_row['filename']).group(1))
    filedate = DateTime(archfiles_row['filetime']).date
    year, doy = (int(x)
                 for x in re.search(r'(\d\d\d\d):(\d\d\d)', filedate).groups())
    archfiles_row['year'] = year
    archfiles_row['doy'] = doy
    hdus.close()
    return archfiles_row

def get_options():
#    from optparse import OptionParser
#    parser = OptionParser()
    import argparse
    import ConfigParser
    parser = argparse.ArgumentParser()
    parser.add_argument("-c", "--conf-file",
                        help="Config file",
                        metavar="FILE",
                        default="archive.conf")
    # if we've got any of the below-defined options
    # they go into remaining_argv instead of throwing an error
    args, remaining_argv = parser.parse_known_args()
    defaults = dict(version='last')
    if args.conf_file:
        config = ConfigParser.SafeConfigParser()
        config.read([args.conf_file])
        defaults = dict(config.items("asp_l1_defaults"))
    parser.set_defaults(**defaults)
    parser.add_argument("--obsid",
                        type=int)
    parser.add_argument("--version",
                        default='last')
    parser.add_argument("--firstrun",
                        action='store_true',
                        help="for archive init., ignore rev in aspect_1 table")
    parser.add_argument("--data-root")
    parser.add_argument("--temp-root")

    opt = parser.parse_args(remaining_argv)
    return opt

def get_file_ver(tempdir, fileglob="*fidpr1*fits*"):
    files = glob(os.path.join(tempdir, fileglob))
    if not files:
        return None
    versions = {}
    for f in files:
        fmatch = re.search('pcadf\d+N(\d{3})_', f)
        if fmatch:
            versions[int(fmatch.group(1))] = 1
        if len(versions) > 1:
            raise ValueError("Different version files in %s" % tempdir)
    # update version to number
    version = versions.keys()[0]
    return version

def get_ver_num(obsid, version='default'):
    # this is obi agnostic, all obis should be same
    # version anyway...
    tempdir = tempfile.mkdtemp()
    arc5.sendline("reset")
    arc5.sendline("cd %s" % tempdir)
    arc5.sendline("obsid=%d" % obsid)
    if version != 'default':
        arc5.sendline("version=%s" % version)
    # just fetch the aspect solution to start
    arc5.sendline("get asp1{fidprops}")
    version = get_file_ver(tempdir)
    for afile in glob(os.path.join(tempdir, "*")):
        os.remove(afile)
    os.removedirs(tempdir)
    return version

def get_asp(obsid, version='last'):
    n_version = get_ver_num(obsid, version=version)
    if n_version is None:
        raise ValueError("No ASP1 for ver %s" % version)
    # use numeric version instead of 'last' or 'default'
    version = n_version

    # set up directory for data
    strobs = "%05d_v%02d" % (obsid, version)
    chunk_dir = strobs[0:2]
    obs_dir = os.path.join(archive_dir, chunk_dir, strobs)
    if not os.path.exists(obs_dir):
        logger.info("making directory %s" % obs_dir)
        os.makedirs(obs_dir)
    else:
        logger.info("obsid dir %s already exists" % obs_dir)
        return

    # get data
    tempdir = Ska.File.TempDir().name
    arc5.sendline("reset")
    arc5.sendline("cd %s" % tempdir)
    logger.info("retrieving data for %d in %s" % (obsid, tempdir))
    arc5.sendline("obsid=%d" % obsid)
    # if multi-obi, we'll need to be specific
    obis = apstat.fetchall(
        "select distinct obi from obidet_0_5 where obsid = %d" % obsid)
    if len(obis) > 1:
        minobi = np.min(obis['obi'])
        logger.info("limiting arc5gl to obi %d" % minobi)
        arc5.sendline("obi=%d" % minobi)
    arc5.sendline("version=%s" % version)
    arc5.sendline("get asp1")
    logger.info("copying asp1 from %s to %s" % (tempdir, obs_dir))
    archfiles = glob(os.path.join(tempdir, "*"))
    if archfiles:
        db = Ska.DBI.DBI(dbi='sqlite', server='archfiles.db3',
                         autocommit=False)
        existing = db.fetchall(
            "select * from archfiles where obsid = %d and revision = '%s'"
            % (obsid, version))
        for i, f in enumerate(archfiles):
            if len(existing) and f in existing['filename']:
                raise ValueError()
            arch_info = get_arch_info(i, f, archfiles, db)
            arch_info['obsid'] = obsid
            db.insert(arch_info, 'archfiles')
            shutil.copy(f, obs_dir)
            #os.remove(f)
    #os.removedirs(tempdir)
        db.commit()
    return obs_dir, chunk_dir

# this needs help
def update_link(obsid):
    # for the obsid get the default and last version numbers
    default_ver = get_ver_num(obsid, 'default')
    last_ver = get_ver_num(obsid, 'last')

    # set up directory for data
    chunk_dir = ("%05d" % obsid)[0:2]
    chunk_dir_path = os.path.join(archive_dir, chunk_dir)
    obs_ln = os.path.join(archive_dir, chunk_dir, "%05d" % obsid)
    obs_ln_last = os.path.join(archive_dir, chunk_dir,
                               "%05d_last" % obsid)

    if default_ver is not None:
        def_ver_dir = os.path.join(archive_dir, chunk_dir,
                                   '%05d_v%02d' % (obsid, default_ver))
        # link the default version
        logger.info("linking %s -> %s" % (
                os.path.relpath(def_ver_dir, chunk_dir_path),
                obs_ln))
        # don't use os.path.exists here because if we have a broken
        # link we want to remove it too
        if os.path.islink(obs_ln):
            os.unlink(obs_ln)
        os.symlink(
                os.path.relpath(def_ver_dir, chunk_dir_path),
                obs_ln)

    # nothing to do if last_ver undefined
    if last_ver is None:
        return

    last_ver_dir = os.path.join(archive_dir, chunk_dir,
                                '%05d_v%02d' % (obsid, last_ver))
    # if last and default are different, and we have last data,
    # make a link to it
    if last_ver != default_ver:
        if os.path.exists(last_ver_dir):
            obs_ln_last = os.path.join(archive_dir, chunk_dir,
                                       "%05d_last" % obsid)
            logger.info("linking %s -> %s" % (
                    os.path.relpath(last_ver_dir, chunk_dir_path),
                    obs_ln_last))
            if os.path.islink(obs_ln_last):
                os.unlink(obs_ln_last)
            os.symlink(
                    os.path.relpath(last_ver_dir, chunk_dir_path),
                    obs_ln_last)
    else:
        # if default and last are the same and we have a last link,
        # delete it
        if os.path.exists(obs_ln_last):
            logger.info("removing outdated link %s"
                        % obs_ln_last)
            os.remove(obs_ln_last)

def get_todo_from_links():
    chunk_dirs = glob(os.path.join(archive_dir, "??"))
    todo_obs = []
    for cdir in chunk_dirs:
        last_links = glob(os.path.join(cdir, "?????_last"))
        obs_links = glob(os.path.join(cdir, "?????"))
        for link in last_links:
            lmatch = re.search('(\d{5})(_last)?$', link)
            if lmatch:
                obs = dict(obsid=int(lmatch.group(1)),
                           revision='default')
                todo_obs.append(obs)
        for link in obs_links:
            if not os.path.exists(link):
                lmatch = re.search('(\d{5})$', link)
                if lmatch:
                    obs = dict(obsid=int(lmatch.group(1)),
                               revision='default')
                    todo_obs.append(obs)

    return todo_obs

def main(opt):

    global archive_dir
    archive_dir = opt.data_root
    last_id_file = os.path.join(archive_dir, 'last_asp1_id.txt')

    # if an obsid is requested, just do that
    if opt.obsid:
        get_asp(opt.obsid, version=opt.version)
        update_link(opt.obsid)
        return

    # look for all previous "provisional" aspect solutions
    # and broken links and and update if possible
    prov_data = get_todo_from_links()
    for obs in prov_data:
        logger.info("running get_asp for obsid %d ver %s, "
                    % (obs['obsid'], obs['revision']))
        try:
            get_asp(obs['obsid'], obs['revision'])
        except ValueError as ve:
            logger.info("skipping %d, default ver not available"
                        % obs['obsid'])
            logger.debug(ve)
        update_link(obs['obsid'])

    # if no obsid specified, try to retrieve all asp_1 runs
    # since tool last run
    # use an aspect_1_id in a file
    last_id_fh = open(last_id_file)
    last_id = int(last_id_fh.read().rstrip())
    last_id_fh.close()
    todo = apstat.fetchall("""select * from aspect_1
                              where aspect_1_id > %d
                              order by aspect_1_id"""
                           % last_id)
    for obs in todo:
        logger.info("running get_asp for obsid %d aspect run on %s"
                    % (obs['obsid'], obs['ap_date']))
        if opt.firstrun:
            # ignore aspect_1 table versions and just try for default
            # also ignore errors in this case
            try:
                get_asp(obs['obsid'], version='default')
            except ValueError as ve:
                logger.info("skipping %d, default ver not available"
                            % obs['obsid'])
                logger.debug(ve)
        else:
            get_asp(obs['obsid'], version=obs['revision'])
        update_link(obs['obsid'])
        last_id_fh = open(last_id_file, 'w')
        last_id_fh.write("%d" % obs['aspect_1_id'])
        last_id_fh.close()
    if not len(todo):
        logger.info("No new aspect_1 data")

if __name__ == '__main__':
    opt = get_options()
    main(opt)
<commit_msg>Reorganized imports<commit_after>import os
import tempfile
from glob import glob
import re
import logging
import shutil
import numpy as np
import pyfits
import argparse
from configobj import ConfigObj

import Ska.arc5gl
import Ska.DBI
from Chandra.Time import DateTime
import Ska.File

# archive_dir set by main()
archive_dir = None
arc5 = Ska.arc5gl.Arc5gl()
aca_db = Ska.DBI.DBI(dbi='sybase', server='sybase', user='aca_read')
apstat = Ska.DBI.DBI(dbi='sybase', server='sqlocc', database='axafapstat')

logger = logging.getLogger('asp1 fetch')
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

# borrowed from eng_archive
archfiles_hdr_cols = ('tstart', 'tstop', 'caldbver', 'content',
                      'ascdsver', 'revision', 'date')

def get_arch_info(i, f, archfiles, db):

    """Read filename ``f`` with index ``i`` (position within list of
    filenames).  The file has type ``filetype`` and will be added to
    MSID file at row index ``row``.  ``colnames`` is the list of
    column names for the content type (not used here).
    """

    filename = os.path.basename(f)

    # Read FITS archive file and accumulate data into dats list and
    # header into headers dict
    logger.debug('Reading (%d / %d) %s' % (i, len(archfiles), filename))
    hdus = pyfits.open(f)
    hdu = hdus[1]

    # Accumlate relevant info about archfile that will be ingested into
    # MSID h5 files.  Commit info before h5 ingest so if there is a failure
    # the needed info will be available to do the repair.
    archfiles_row = dict((x, hdu.header.get(x.upper()))
                         for x in archfiles_hdr_cols)
    archfiles_row['checksum'] = hdu._checksum
    archfiles_row['filename'] = filename
    archfiles_row['filetime'] = int(
        re.search(r'(\d+)', archfiles_row['filename']).group(1))
    filedate = DateTime(archfiles_row['filetime']).date
    year, doy = (int(x)
                 for x in re.search(r'(\d\d\d\d):(\d\d\d)', filedate).groups())
    archfiles_row['year'] = year
    archfiles_row['doy'] = doy
    hdus.close()
    return archfiles_row

def get_options():
#    from optparse import OptionParser
#    parser = OptionParser()
    parser = argparse.ArgumentParser()
    parser.add_argument("-c", "--conf-file",
                        help="Config file",
                        metavar="FILE",
                        default="archive.conf")
    # if we've got any of the below-defined options
    # they go into remaining_argv instead of throwing an error
    args, remaining_argv = parser.parse_known_args()
    defaults = dict(version='last')
    if args.conf_file:
        config = ConfigParser.SafeConfigParser()
        config.read([args.conf_file])
        defaults = dict(config.items("asp_l1_defaults"))
    parser.set_defaults(**defaults)
    parser.add_argument("--obsid",
                        type=int)
    parser.add_argument("--version",
                        default='last')
    parser.add_argument("--firstrun",
                        action='store_true',
                        help="for archive init., ignore rev in aspect_1 table")
    parser.add_argument("--data-root")
    parser.add_argument("--temp-root")

    opt = parser.parse_args(remaining_argv)
    return opt

def get_file_ver(tempdir, fileglob="*fidpr1*fits*"):
    files = glob(os.path.join(tempdir, fileglob))
    if not files:
        return None
    versions = {}
    for f in files:
        fmatch = re.search('pcadf\d+N(\d{3})_', f)
        if fmatch:
            versions[int(fmatch.group(1))] = 1
        if len(versions) > 1:
            raise ValueError("Different version files in %s" % tempdir)
    # update version to number
    version = versions.keys()[0]
    return version

def get_ver_num(obsid, version='default'):
    # this is obi agnostic, all obis should be same
    # version anyway...
    tempdir = tempfile.mkdtemp()
    arc5.sendline("reset")
    arc5.sendline("cd %s" % tempdir)
    arc5.sendline("obsid=%d" % obsid)
    if version != 'default':
        arc5.sendline("version=%s" % version)
    # just fetch the aspect solution to start
    arc5.sendline("get asp1{fidprops}")
    version = get_file_ver(tempdir)
    for afile in glob(os.path.join(tempdir, "*")):
        os.remove(afile)
    os.removedirs(tempdir)
    return version

def get_asp(obsid, version='last'):
    n_version = get_ver_num(obsid, version=version)
    if n_version is None:
        raise ValueError("No ASP1 for ver %s" % version)
    # use numeric version instead of 'last' or 'default'
    version = n_version

    # set up directory for data
    strobs = "%05d_v%02d" % (obsid, version)
    chunk_dir = strobs[0:2]
    obs_dir = os.path.join(archive_dir, chunk_dir, strobs)
    if not os.path.exists(obs_dir):
        logger.info("making directory %s" % obs_dir)
        os.makedirs(obs_dir)
    else:
        logger.info("obsid dir %s already exists" % obs_dir)
        return

    # get data
    tempdir = Ska.File.TempDir().name
    arc5.sendline("reset")
    arc5.sendline("cd %s" % tempdir)
    logger.info("retrieving data for %d in %s" % (obsid, tempdir))
    arc5.sendline("obsid=%d" % obsid)
    # if multi-obi, we'll need to be specific
    obis = apstat.fetchall(
        "select distinct obi from obidet_0_5 where obsid = %d" % obsid)
    if len(obis) > 1:
        minobi = np.min(obis['obi'])
        logger.info("limiting arc5gl to obi %d" % minobi)
        arc5.sendline("obi=%d" % minobi)
    arc5.sendline("version=%s" % version)
    arc5.sendline("get asp1")
    logger.info("copying asp1 from %s to %s" % (tempdir, obs_dir))
    archfiles = glob(os.path.join(tempdir, "*"))
    if archfiles:
        db = Ska.DBI.DBI(dbi='sqlite', server='archfiles.db3',
                         autocommit=False)
        existing = db.fetchall(
            "select * from archfiles where obsid = %d and revision = '%s'"
            % (obsid, version))
        for i, f in enumerate(archfiles):
            if len(existing) and f in existing['filename']:
                raise ValueError()
            arch_info = get_arch_info(i, f, archfiles, db)
            arch_info['obsid'] = obsid
            db.insert(arch_info, 'archfiles')
            shutil.copy(f, obs_dir)
            #os.remove(f)
    #os.removedirs(tempdir)
        db.commit()
    return obs_dir, chunk_dir

# this needs help
def update_link(obsid):
    # for the obsid get the default and last version numbers
    default_ver = get_ver_num(obsid, 'default')
    last_ver = get_ver_num(obsid, 'last')

    # set up directory for data
    chunk_dir = ("%05d" % obsid)[0:2]
    chunk_dir_path = os.path.join(archive_dir, chunk_dir)
    obs_ln = os.path.join(archive_dir, chunk_dir, "%05d" % obsid)
    obs_ln_last = os.path.join(archive_dir, chunk_dir,
                               "%05d_last" % obsid)

    if default_ver is not None:
        def_ver_dir = os.path.join(archive_dir, chunk_dir,
                                   '%05d_v%02d' % (obsid, default_ver))
        # link the default version
        logger.info("linking %s -> %s" % (
                os.path.relpath(def_ver_dir, chunk_dir_path),
                obs_ln))
        # don't use os.path.exists here because if we have a broken
        # link we want to remove it too
        if os.path.islink(obs_ln):
            os.unlink(obs_ln)
        os.symlink(
                os.path.relpath(def_ver_dir, chunk_dir_path),
                obs_ln)

    # nothing to do if last_ver undefined
    if last_ver is None:
        return

    last_ver_dir = os.path.join(archive_dir, chunk_dir,
                                '%05d_v%02d' % (obsid, last_ver))
    # if last and default are different, and we have last data,
    # make a link to it
    if last_ver != default_ver:
        if os.path.exists(last_ver_dir):
            obs_ln_last = os.path.join(archive_dir, chunk_dir,
                                       "%05d_last" % obsid)
            logger.info("linking %s -> %s" % (
                    os.path.relpath(last_ver_dir, chunk_dir_path),
                    obs_ln_last))
            if os.path.islink(obs_ln_last):
                os.unlink(obs_ln_last)
            os.symlink(
                    os.path.relpath(last_ver_dir, chunk_dir_path),
                    obs_ln_last)
    else:
        # if default and last are the same and we have a last link,
        # delete it
        if os.path.exists(obs_ln_last):
            logger.info("removing outdated link %s"
                        % obs_ln_last)
            os.remove(obs_ln_last)

def get_todo_from_links():
    chunk_dirs = glob(os.path.join(archive_dir, "??"))
    todo_obs = []
    for cdir in chunk_dirs:
        last_links = glob(os.path.join(cdir, "?????_last"))
        obs_links = glob(os.path.join(cdir, "?????"))
        for link in last_links:
            lmatch = re.search('(\d{5})(_last)?$', link)
            if lmatch:
                obs = dict(obsid=int(lmatch.group(1)),
                           revision='default')
                todo_obs.append(obs)
        for link in obs_links:
            if not os.path.exists(link):
                lmatch = re.search('(\d{5})$', link)
                if lmatch:
                    obs = dict(obsid=int(lmatch.group(1)),
                               revision='default')
                    todo_obs.append(obs)

    return todo_obs

def main(opt):

    global archive_dir
    archive_dir = opt.data_root
    last_id_file = os.path.join(archive_dir, 'last_asp1_id.txt')

    # if an obsid is requested, just do that
    if opt.obsid:
        get_asp(opt.obsid, version=opt.version)
        update_link(opt.obsid)
        return

    # look for all previous "provisional" aspect solutions
    # and broken links and and update if possible
    prov_data = get_todo_from_links()
    for obs in prov_data:
        logger.info("running get_asp for obsid %d ver %s, "
                    % (obs['obsid'], obs['revision']))
        try:
            get_asp(obs['obsid'], obs['revision'])
        except ValueError as ve:
            logger.info("skipping %d, default ver not available"
                        % obs['obsid'])
            logger.debug(ve)
        update_link(obs['obsid'])

    # if no obsid specified, try to retrieve all asp_1 runs
    # since tool last run
    # use an aspect_1_id in a file
    last_id_fh = open(last_id_file)
    last_id = int(last_id_fh.read().rstrip())
    last_id_fh.close()
    todo = apstat.fetchall("""select * from aspect_1
                              where aspect_1_id > %d
                              order by aspect_1_id"""
                           % last_id)
    for obs in todo:
        logger.info("running get_asp for obsid %d aspect run on %s"
                    % (obs['obsid'], obs['ap_date']))
        if opt.firstrun:
            # ignore aspect_1 table versions and just try for default
            # also ignore errors in this case
            try:
                get_asp(obs['obsid'], version='default')
            except ValueError as ve:
                logger.info("skipping %d, default ver not available"
                            % obs['obsid'])
                logger.debug(ve)
        else:
            get_asp(obs['obsid'], version=obs['revision'])
        update_link(obs['obsid'])
        last_id_fh = open(last_id_file, 'w')
        last_id_fh.write("%d" % obs['aspect_1_id'])
        last_id_fh.close()
    if not len(todo):
        logger.info("No new aspect_1 data")

if __name__ == '__main__':
    opt = get_options()
    main(opt)
<|endoftext|>


<commit_before># Copyright (c) 2013 Yubico AB
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import os
try:
    import hidraw as hid  # Prefer hidraw
except ImportError:
    import hid
from time import time
from u2flib_host.device import U2FDevice
from u2flib_host import exc

DEVICES = [
    (0x1050, 0x0200),  # Gnubby
    (0x1050, 0x0113),  # YubiKey NEO U2F
    (0x1050, 0x0114),  # YubiKey NEO OTP+U2F
    (0x1050, 0x0115),  # YubiKey NEO U2F+CCID
    (0x1050, 0x0116),  # YubiKey NEO OTP+U2F+CCID
    (0x1050, 0x0120),  # Security Key by Yubico
    (0x1050, 0x0410),  # YubiKey Plus
    (0x1050, 0x0402),  # YubiKey 4 U2F
    (0x1050, 0x0403),  # YubiKey 4 OTP+U2F
    (0x1050, 0x0406),  # YubiKey 4 U2F+CCID
    (0x1050, 0x0407),  # YubiKey 4 OTP+U2F+CCID
]
HID_RPT_SIZE = 64

TYPE_INIT = 0x80
U2F_VENDOR_FIRST = 0x40

# USB Commands
CMD_INIT = 0x06
CMD_WINK = 0x08
CMD_PING = 0x01
CMD_APDU = 0x03
U2FHID_YUBIKEY_DEVICE_CONFIG = U2F_VENDOR_FIRST

STAT_ERR = 0xbf

TIMEOUT = 1000

def list_devices():
    devices = []
    for d in hid.enumerate(0, 0):
        usage_page = d['usage_page']
        if usage_page == 0xf1d0 and d['usage'] == 1:
            devices.append(HIDDevice(d['path']))
        # Usage page doesn't work on Linux
        elif (d['vendor_id'], d['product_id']) in DEVICES:
            device = HIDDevice(d['path'])
            try:
                device.open()
                device.close()
                devices.append(HIDDevice(d['path']))
            except exc.DeviceError:
                pass
    return devices

def _read_timeout(dev, size, timeout=1.0):
    timeout += time()
    while time() < timeout:
        resp = dev.read(size)
        if resp:
            return resp
    return []

class U2FHIDError(Exception):
    def __init__(self, code):
        super(Exception, self).__init__("U2FHIDError: 0x%02x" % code)
        self.code = code

class HIDDevice(U2FDevice):

    """
    U2FDevice implementation using the HID transport.
    """

    def __init__(self, path):
        self.path = path
        self.cid = "ffffffff".decode('hex')

    def open(self):
        self.handle = hid.device()
        self.handle.open_path(self.path)
        self.handle.set_nonblocking(True)
        self.init()

    def close(self):
        if hasattr(self, 'handle'):
            self.handle.close()
            del self.handle

    def init(self):
        nonce = os.urandom(8)
        resp = self.call(CMD_INIT, nonce)
        while resp[:8] != nonce:
            print "Wrong nonce, read again..."
            resp = self._read_resp(self.cid, CMD_INIT)
        self.cid = resp[8:12]

    def set_mode(self, mode):
        data = ("%02x0f0000" % mode).decode('hex')
        self.call(U2FHID_YUBIKEY_DEVICE_CONFIG, data)

    def _do_send_apdu(self, apdu_data):
        return self.call(CMD_APDU, apdu_data)

    def wink(self):
        self.call(CMD_WINK)

    def ping(self, msg=b'Hello U2F'):
        resp = self.call(CMD_PING, msg)
        if resp != msg:
            raise exc.DeviceError("Incorrect PING readback")
        return resp

    def _send_req(self, cid, cmd, data):
        size = len(data)
        bc_l = chr(size & 0xff)
        bc_h = chr(size >> 8 & 0xff)
        payload = cid + chr(TYPE_INIT | cmd) + bc_h + bc_l + \
            data[:HID_RPT_SIZE - 7]
        payload += '\0' * (HID_RPT_SIZE - len(payload))
        self.handle.write([0] + map(ord, payload))
        data = data[HID_RPT_SIZE - 7:]
        seq = 0
        while len(data) > 0:
            payload = cid + chr(0x7f & seq) + data[:HID_RPT_SIZE - 5]
            payload += '\0' * (HID_RPT_SIZE - len(payload))
            self.handle.write([0] + map(ord, payload))
            data = data[HID_RPT_SIZE - 5:]
            seq += 1

    def _read_resp(self, cid, cmd):
        resp = '.'
        header = cid + chr(TYPE_INIT | cmd)
        while resp and resp[:5] != header:
            resp = ''.join(map(chr, _read_timeout(self.handle, HID_RPT_SIZE)))
            if resp[:5] == cid + chr(STAT_ERR):
                raise U2FHIDError(ord(resp[6]))

        if not resp:
            raise exc.DeviceError("Invalid response from device!")

        data_len = (ord(resp[5]) << 8) + ord(resp[6])
        data = resp[7:min(7 + data_len, HID_RPT_SIZE)]
        data_len -= len(data)

        seq = 0
        while data_len > 0:
            resp = ''.join(map(chr, _read_timeout(self.handle, HID_RPT_SIZE)))
            if resp[:4] != cid:
                raise exc.DeviceError("Wrong CID from device!")
            if ord(resp[4]) != seq & 0x7f:
                raise exc.DeviceError("Wrong SEQ from device!")
            seq += 1
            new_data = resp[5:min(5 + data_len, HID_RPT_SIZE)]
            data_len -= len(new_data)
            data += new_data
        return data

    def call(self, cmd, data=''):
        if isinstance(data, int):
            data = chr(data)

        self._send_req(self.cid, cmd, data)
        return self._read_resp(self.cid, cmd)
<commit_msg>Bugfix: wrong byte used to extract error code from STAT_ERR response<commit_after># Copyright (c) 2013 Yubico AB
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import os
try:
    import hidraw as hid  # Prefer hidraw
except ImportError:
    import hid
from time import time
from u2flib_host.device import U2FDevice
from u2flib_host import exc

DEVICES = [
    (0x1050, 0x0200),  # Gnubby
    (0x1050, 0x0113),  # YubiKey NEO U2F
    (0x1050, 0x0114),  # YubiKey NEO OTP+U2F
    (0x1050, 0x0115),  # YubiKey NEO U2F+CCID
    (0x1050, 0x0116),  # YubiKey NEO OTP+U2F+CCID
    (0x1050, 0x0120),  # Security Key by Yubico
    (0x1050, 0x0410),  # YubiKey Plus
    (0x1050, 0x0402),  # YubiKey 4 U2F
    (0x1050, 0x0403),  # YubiKey 4 OTP+U2F
    (0x1050, 0x0406),  # YubiKey 4 U2F+CCID
    (0x1050, 0x0407),  # YubiKey 4 OTP+U2F+CCID
]
HID_RPT_SIZE = 64

TYPE_INIT = 0x80
U2F_VENDOR_FIRST = 0x40

# USB Commands
CMD_INIT = 0x06
CMD_WINK = 0x08
CMD_PING = 0x01
CMD_APDU = 0x03
U2FHID_YUBIKEY_DEVICE_CONFIG = U2F_VENDOR_FIRST

STAT_ERR = 0xbf

TIMEOUT = 1000

def list_devices():
    devices = []
    for d in hid.enumerate(0, 0):
        usage_page = d['usage_page']
        if usage_page == 0xf1d0 and d['usage'] == 1:
            devices.append(HIDDevice(d['path']))
        # Usage page doesn't work on Linux
        elif (d['vendor_id'], d['product_id']) in DEVICES:
            device = HIDDevice(d['path'])
            try:
                device.open()
                device.close()
                devices.append(HIDDevice(d['path']))
            except exc.DeviceError:
                pass
    return devices

def _read_timeout(dev, size, timeout=1.0):
    timeout += time()
    while time() < timeout:
        resp = dev.read(size)
        if resp:
            return resp
    return []

class U2FHIDError(Exception):
    def __init__(self, code):
        super(Exception, self).__init__("U2FHIDError: 0x%02x" % code)
        self.code = code

class HIDDevice(U2FDevice):

    """
    U2FDevice implementation using the HID transport.
    """

    def __init__(self, path):
        self.path = path
        self.cid = "ffffffff".decode('hex')

    def open(self):
        self.handle = hid.device()
        self.handle.open_path(self.path)
        self.handle.set_nonblocking(True)
        self.init()

    def close(self):
        if hasattr(self, 'handle'):
            self.handle.close()
            del self.handle

    def init(self):
        nonce = os.urandom(8)
        resp = self.call(CMD_INIT, nonce)
        while resp[:8] != nonce:
            print "Wrong nonce, read again..."
            resp = self._read_resp(self.cid, CMD_INIT)
        self.cid = resp[8:12]

    def set_mode(self, mode):
        data = ("%02x0f0000" % mode).decode('hex')
        self.call(U2FHID_YUBIKEY_DEVICE_CONFIG, data)

    def _do_send_apdu(self, apdu_data):
        return self.call(CMD_APDU, apdu_data)

    def wink(self):
        self.call(CMD_WINK)

    def ping(self, msg=b'Hello U2F'):
        resp = self.call(CMD_PING, msg)
        if resp != msg:
            raise exc.DeviceError("Incorrect PING readback")
        return resp

    def _send_req(self, cid, cmd, data):
        size = len(data)
        bc_l = chr(size & 0xff)
        bc_h = chr(size >> 8 & 0xff)
        payload = cid + chr(TYPE_INIT | cmd) + bc_h + bc_l + \
            data[:HID_RPT_SIZE - 7]
        payload += '\0' * (HID_RPT_SIZE - len(payload))
        self.handle.write([0] + map(ord, payload))
        data = data[HID_RPT_SIZE - 7:]
        seq = 0
        while len(data) > 0:
            payload = cid + chr(0x7f & seq) + data[:HID_RPT_SIZE - 5]
            payload += '\0' * (HID_RPT_SIZE - len(payload))
            self.handle.write([0] + map(ord, payload))
            data = data[HID_RPT_SIZE - 5:]
            seq += 1

    def _read_resp(self, cid, cmd):
        resp = '.'
        header = cid + chr(TYPE_INIT | cmd)
        while resp and resp[:5] != header:
            resp = ''.join(map(chr, _read_timeout(self.handle, HID_RPT_SIZE)))
            if resp[:5] == cid + chr(STAT_ERR):
                raise U2FHIDError(ord(resp[7]))

        if not resp:
            raise exc.DeviceError("Invalid response from device!")

        data_len = (ord(resp[5]) << 8) + ord(resp[6])
        data = resp[7:min(7 + data_len, HID_RPT_SIZE)]
        data_len -= len(data)

        seq = 0
        while data_len > 0:
            resp = ''.join(map(chr, _read_timeout(self.handle, HID_RPT_SIZE)))
            if resp[:4] != cid:
                raise exc.DeviceError("Wrong CID from device!")
            if ord(resp[4]) != seq & 0x7f:
                raise exc.DeviceError("Wrong SEQ from device!")
            seq += 1
            new_data = resp[5:min(5 + data_len, HID_RPT_SIZE)]
            data_len -= len(new_data)
            data += new_data
        return data

    def call(self, cmd, data=''):
        if isinstance(data, int):
            data = chr(data)

        self._send_req(self.cid, cmd, data)
        return self._read_resp(self.cid, cmd)
<|endoftext|>


<commit_before>import os
import subprocess
import spur

from django.conf import settings

shell = None
if settings.NAS_IP and settings.NAS_USERNAME and settings.REMOTE_FILE_OPERATION_ENABLED:
    shell = spur.SshShell(hostname=settings.NAS_IP, username=settings.NAS_USERNAME)

def link(source_file, destination_file):
    # Source file is the destination of the link and destionation_file is the new link path
    if shell:
        result = shell.run(["ln", map_to_nas(source_file), map_to_nas(destination_file)])
        return result.return_code
    return os.link(source_file, destination_file)

def symlink(source_file, destination_file):
    # Source file is the destination of the link and destionation_file is the new link path
    if shell:
        result = shell.run(["ln", "-s", map_to_nas(source_file), map_to_nas(destination_file)])
        return result.return_code
    return os.symlink(source_file, destination_file)

def unrar(source_file, destination_dir):
    # Source file is the destination of the link and destionation_file is the new link path
    if shell:
        result = shell.run(["unrar", "e", "-y", map_to_nas(source_file), map_to_nas(destination_dir)])
        return result.return_code
    return subprocess.check_output('unrar e -y %s %s' % (source_file, destination_dir))

def map_to_nas(local_path):
    '''
    This method maps between the compute executing the code, and the NAS on which the file are actually located.
    You can have a smb server linked to you local filesystem to browse, remove, create dir, but you can't link file
    as you need knowledge on the actual filesystem which the nas does.
    :param local_path:
    :return:
    '''
    return local_path.replace(settings.LOCAL_ROOT, settings.NAS_ROOT)<commit_msg>changed the unrar command<commit_after>import os
import subprocess
import spur

from django.conf import settings

shell = None
if settings.NAS_IP and settings.NAS_USERNAME and settings.REMOTE_FILE_OPERATION_ENABLED:
    shell = spur.SshShell(hostname=settings.NAS_IP, username=settings.NAS_USERNAME, connect_timeout=600)

def link(source_file, destination_file):
    # Source file is the destination of the link and destionation_file is the new link path
    if shell:
        result = shell.run(["ln", map_to_nas(source_file), map_to_nas(destination_file)])
        return result.return_code
    return os.link(source_file, destination_file)

def symlink(source_file, destination_file):
    # Source file is the destination of the link and destionation_file is the new link path
    if shell:
        result = shell.run(["ln", "-s", map_to_nas(source_file), map_to_nas(destination_file)])
        return result.return_code
    return os.symlink(source_file, destination_file)

def unrar(source_file, destination_dir):
    # Source file is the archive file and destionation_dir is the extraction directory
    if shell:
        result = shell.run(["/usr/local/sbin/unrar", "e", "-y", map_to_nas(source_file), map_to_nas(destination_dir)])
        return result.return_code
    return subprocess.check_output('unrar e -y %s %s' % (source_file, destination_dir))

def map_to_nas(local_path):
    '''
    This method maps between the compute executing the code, and the NAS on which the file are actually located.
    You can have a smb server linked to you local filesystem to browse, remove, create dir, but you can't link file
    as you need knowledge on the actual filesystem which the nas does.
    :param local_path:
    :return:
    '''
    return local_path.replace(settings.LOCAL_ROOT, settings.NAS_ROOT)<|endoftext|>


<commit_before>from __future__ import print_function
from __future__ import unicode_literals
from inspect import getdoc
from operator import attrgetter

import re
import signal
import sys
import os
import subprocess
import yaml

from os import path
from requests.exceptions import ReadTimeout

from docopt import docopt, DocoptExit

from compose import config
from compose.project import Project
from compose.cli.log_printer import LogPrinter

from docker.errors import APIError
from compose.project import NoSuchService, ConfigurationError
from compose.service import BuildError
from compose.legacy import LegacyContainersError

import docker
import dockerpty

from .client import docker_client
from .__init__ import __version__

def main():

    try:
        Usage()
    except KeyboardInterrupt:
        print("\nAborting.")
        sys.exit(1)
    except (NoSuchService, ConfigurationError, LegacyContainersError) as e:
        print(e.msg)
        sys.exit(1)
    except APIError as e:
        print(e.explanation)
        sys.exit(1)
    except BuildError as e:
        print("Service '{service}' failed to build: {reason}".format(
            service=e.service.name,
            reason=e.reason))
        sys.exit(1)

yaml_re = re.compile('\.yaml$|\.yml$')
envvar_re = re.compile('\$\w+')

class Usage(object):

    """Describe your infrastructure with yaml files.

    Usage:
        tug2 ps
        tug2 exec PROJECT SERVICE [COMMANDS ...]
        tug2 COMMAND PROJECT [SERVICES ...]

    Common Commands:

        ps             List all running and available projects
        up             Update and run services
        diff           Describe the changes needed to update
        cull           Stop and delete services
        logs           Display container logs
        exec           Run a command inside a container (not implemented yet)

    Management Commands:

        kill           Gracefully terminate services
        down           Stop services
        rm             Delete services
        recreate       Stop, delete, then run services
        build          Build services
        rebuild        Build services from scratch

    Options:

        -h --help      Display this usage information
        -v --version   Display the version number

    """

    def __init__(self):
        docstring = getdoc(Usage)
        options = None
        try:
            options = docopt(docstring,
                argv=sys.argv[1:],
                version=__version__,
                options_first=True)
        except DocoptExit:
            raise SystemExit(docstring)

        if 'ps' in options and options['ps']:
            self._ps()
            return

        projectname = self._clean_project_name(options['PROJECT'])

        if 'exec' in options and options['exec']:
          self._exec(projectname, options['SERVICE'], options['COMMANDS'])
          return

        # 'command' references a function on this class
        command = options['COMMAND']
        if not hasattr(self, command):
            print('{command} command not found'.format(command=command))
            sys.exit(1)

        servicenames = options['SERVICES']

        if command in ['up','build','recreate']:
            config = self._get_config(projectname, envvars=True)
        else:
            config = self._get_config(projectname)

        client = docker_client()

        project = Project.from_dicts(
            projectname,
            config,
            client)

        handle = getattr(self, command)
        handle(project, projectname, servicenames)

    def _clean_project_name(self, name):
        return yaml_re.sub('', name)

    def _evaluate_env_vars(self, openfile):
        body = openfile.read()
        matched = []
        not_found = []
        for envvar in envvar_re.findall(body):
            if envvar not in matched:
                matched.append(envvar)
                sysenvvar = os.getenv(envvar.replace('$',''))
                if sysenvvar:
                    body = body.replace(envvar, sysenvvar)
                else:
                    not_found.append(envvar)
        if not_found:
            raise ConfigurationError("Environment variables '%s' not setted or empty" % ', '.join(not_found))
        openfile.seek(0)
        return body

    def _load_yaml(self, filename):
        try:
            with open(filename, 'r') as openfile:
                return yaml.safe_load(self._evaluate_env_vars(openfile))
        except IOError as e:
            raise ConfigurationError(six.text_type(e))

    def _load(self, filename, envvars):
        working_dir = os.path.dirname(filename)
        if envvars:
            loaded = self._load_yaml(filename)
        else:
            loaded = config.load_yaml(filename)
        return config.from_dictionary(loaded,
                                      working_dir=working_dir,
                                      filename=filename)

    def _get_config(self, name, envvars=False):
        for filename in self._get_projects_in_dir(True):
            if re.search('%s(.yaml|.yml)$' % name, filename):
                return self._load(filename, envvars)
        raise ConfigurationError("Project filename '%s' not found in the current directory" % name)

    def _get_projects_in_dir(self, fullpath=False):
        projects = []
        cwd = os.getcwd()
        for filename in os.listdir(cwd):
            if yaml_re.search(filename):
                if fullpath:
                    projects.append(path.join(cwd, filename))
                else:
                    projects.append(self._clean_project_name(filename))
        return projects

    def _ps(self):
        client = docker_client()
        projectnames = self._get_projects_in_dir()
        containers = client.containers(all=True)
        unknown = {}
        for container in containers:
            unknown[container['Id']] = client.inspect_container(
                container['Id'])
        if len(projectnames) != 0:
            print()
        for projectname in projectnames:
            config = self._get_config(projectname)
            project = Project.from_dicts(
                projectname,
                config,
                client)
            services = project.get_services()

            counts = {}

            for service in services:
                c = service.containers(stopped=True) + service.containers(one_off=True)
                if len(c) == 0:
                    if not 'Uncreated' in counts:
                        counts['Uncreated'] = 0
                    counts['Uncreated'] += 1
                for container in c:
                    del unknown[container.id]
                    if not container.human_readable_state in counts:
                        counts[container.human_readable_state] = 0
                    counts[container.human_readable_state] += 1

            humancounts = []
            for state in counts:
                humancounts.append('{count} {state}'.format(
                    count=counts[state],
                    state=state))
            print('  {name: <24}{counts}'.format(
                name=projectname,
                counts=','.join(humancounts)))

        if len(unknown) != 0:
            print()
            print('  Containers not tracked by compose/tugboat:')
            print()
        for key in unknown:
            detail = unknown[key]
            name = detail['Name']
            if name.startswith('/'):
                name = name[1:]
            ip = detail['NetworkSettings']['IPAddress']
            if ip == '':
                ip = '(host)'
            print('  {name: <24}{state: <12}{ip: <17}'.format(
                name=name,
                state='',
                ip=ip))
        print()

    def ps(self, project, projectname, servicenames):
        print()
        print('  {name} services:'.format(name=projectname))
        print()
        services = project.get_services(service_names=servicenames)
        for service in services:
            containers = service.containers(stopped=True) + service.containers(one_off=True)
            if len(containers) == 0:
                print('  {name: <24}{state: <12}{ip: <17}'.format(
                    name=service.name,
                    state='Uncreated',
                    ip=''))
            for container in containers:
                name = container.name_without_project
                ip = container.get('NetworkSettings.IPAddress')
                if ip == '':
                    ip = '(host)'
                state = container.human_readable_state
                print('  {name: <24}{state: <12}{ip: <17}'.format(
                    name=name,
                    state=state,
                    ip=ip))
        print()

    def build(self, project, projectname, servicenames):
        project.build(service_names=servicenames, no_cache=False)

    def rebuild(self, project, projectname, servicenames):
        project.build(service_names=servicenames, no_cache=True)

    def kill(self, project, projectname, servicenames):
        project.kill(service_names=servicenames, signal='SIGTERM')
        self.ps(project, projectname, servicenames)

    def logs(self, project, projectname, servicenames):
        containers = project.containers(
            service_names=servicenames,
            stopped=True)
        print('Attaching to', ', '.join(c.name for c in containers))
        LogPrinter(containers, attach_params={'logs': True}).run()

    def pull(self, project, projectname, servicenames):
        project.pull(service_names=servicenames)

    def rm(self, project, projectname, servicenames):
        project.remove_stopped(service_names=servicenames)

        self.ps(project, projectname, servicenames)

    def down(self, project, projectname, servicenames):
        project.kill(service_names=servicenames, signal='SIGTERM')
        project.stop(service_names=servicenames)

        self.ps(project, projectname, servicenames)

    def cull(self, project, projectname, servicenames):
        project.kill(service_names=servicenames, signal='SIGTERM')
        project.stop(service_names=servicenames)
        project.remove_stopped(service_names=servicenames)

        self.ps(project, projectname, servicenames)

    def recreate(self, project, projectname, servicenames):
        project.restart(service_names=servicenames)

        self.ps(project, projectname, servicenames)

    def up(self, project, projectname, servicenames):
        containers = project.containers(stopped=True) + project.containers(one_off=True)
        unknown = {}
        for container in containers:
            unknown[container.id] = container
        services = project.get_services(servicenames, include_deps=True)
        plans = project._get_convergence_plans(services, smart_recreate=True)
        for service in plans:
            plan = plans[service]
            for container in plan.containers:
                del unknown[container.id]
        project.up(
            service_names=servicenames,
            smart_recreate=True)

        if len(servicenames) == 0:
            for id in unknown:
                container = unknown[id]
                if container.is_running:
                    container.kill(signal='SIGTERM')
                    try:
                        container.client.wait(container.id, timeout=10)
                    except ReadTimeout as e:
                        pass
                    container.stop()
                    try:
                        container.client.wait(container.id, timeout=10)
                    except ReadTimeout as e:
                        pass
                container.remove()

        self.ps(project, projectname, servicenames)

    def diff(self, project, projectname, servicenames):
        containers = project.containers(stopped=True) + project.containers(one_off=True)
        unknown = {}
        for container in containers:
            unknown[container.id] = container
        services = project.get_services(servicenames, include_deps=True)
        plans = project._get_convergence_plans(services, smart_recreate=True)

        print()
        print('  {name} convergence plan:'.format(name=projectname))
        print()
        for service in plans:
            plan = plans[service]
            service_containers = []
            for container in plan.containers:
                del unknown[container.id]
                service_containers.append(container.name)
            print('  {name: <24}{action: <12}{containers}'.format(
                name=service,
                action=plan.action,
                containers=', '.join(service_containers)))
        # TODO: Add this in when up starts deleting unknown containers.
        for id in unknown:
            container = unknown[id]
            print('  {name: <24}{action: <12}'.format(
                name=container.name,
                action='delete'))
        print()

    # TODO: look in the yaml file?
    def _exec(self, projectname, servicename, commands):
        containername = '{projectname}_{servicename}_1'.format(
            projectname=projectname,
            servicename=servicename)
        command = commands
        if command is None or not command:
            command = ['/bin/bash']
        print()
        print('  docker exec -it {containername} {command}'.format(
            containername=containername,
            command=' '.join(command)))
        print()
        command = ['docker', 'exec', '-it', containername] + command
        sys.exit(subprocess.call(command))<commit_msg>Changed format to ${ENV}<commit_after>from __future__ import print_function
from __future__ import unicode_literals
from inspect import getdoc
from operator import attrgetter

import re
import signal
import sys
import os
import subprocess
import yaml

from os import path
from requests.exceptions import ReadTimeout

from docopt import docopt, DocoptExit

from compose import config
from compose.project import Project
from compose.cli.log_printer import LogPrinter

from docker.errors import APIError
from compose.project import NoSuchService, ConfigurationError
from compose.service import BuildError
from compose.legacy import LegacyContainersError

import docker
import dockerpty

from .client import docker_client
from .__init__ import __version__

def main():

    try:
        Usage()
    except KeyboardInterrupt:
        print("\nAborting.")
        sys.exit(1)
    except (NoSuchService, ConfigurationError, LegacyContainersError) as e:
        print(e.msg)
        sys.exit(1)
    except APIError as e:
        print(e.explanation)
        sys.exit(1)
    except BuildError as e:
        print("Service '{service}' failed to build: {reason}".format(
            service=e.service.name,
            reason=e.reason))
        sys.exit(1)

yaml_re = re.compile('\.yaml$|\.yml$')
envvar_re = re.compile('(?<=\$\{)\w+(?=\})')

class Usage(object):

    """Describe your infrastructure with yaml files.

    Usage:
        tug2 ps
        tug2 exec PROJECT SERVICE [COMMANDS ...]
        tug2 COMMAND PROJECT [SERVICES ...]

    Common Commands:

        ps             List all running and available projects
        up             Update and run services
        diff           Describe the changes needed to update
        cull           Stop and delete services
        logs           Display container logs
        exec           Run a command inside a container (not implemented yet)

    Management Commands:

        kill           Gracefully terminate services
        down           Stop services
        rm             Delete services
        recreate       Stop, delete, then run services
        build          Build services
        rebuild        Build services from scratch

    Options:

        -h --help      Display this usage information
        -v --version   Display the version number

    """

    def __init__(self):
        docstring = getdoc(Usage)
        options = None
        try:
            options = docopt(docstring,
                argv=sys.argv[1:],
                version=__version__,
                options_first=True)
        except DocoptExit:
            raise SystemExit(docstring)

        if 'ps' in options and options['ps']:
            self._ps()
            return

        projectname = self._clean_project_name(options['PROJECT'])

        if 'exec' in options and options['exec']:
          self._exec(projectname, options['SERVICE'], options['COMMANDS'])
          return

        # 'command' references a function on this class
        command = options['COMMAND']
        if not hasattr(self, command):
            print('{command} command not found'.format(command=command))
            sys.exit(1)

        servicenames = options['SERVICES']

        if command in ['up','build','recreate']:
            config = self._get_config(projectname, envvars=True)
        else:
            config = self._get_config(projectname)

        client = docker_client()

        project = Project.from_dicts(
            projectname,
            config,
            client)

        handle = getattr(self, command)
        handle(project, projectname, servicenames)

    def _clean_project_name(self, name):
        return yaml_re.sub('', name)

    def _evaluate_env_vars(self, openfile):
        body = openfile.read()
        matched = []
        not_found = []
        for envvar in envvar_re.findall(body):
            if envvar not in matched:
                matched.append(envvar)
                sysenvvar = os.getenv(envvar)
                if sysenvvar:
                    body = body.replace('${%s}' % envvar, sysenvvar)
                else:
                    not_found.append(envvar)
        if not_found:
            raise ConfigurationError("Environment variables '%s' not setted or empty" % ', '.join(not_found))
        openfile.seek(0)
        return body

    def _load_yaml(self, filename):
        try:
            with open(filename, 'r') as openfile:
                return yaml.safe_load(self._evaluate_env_vars(openfile))
        except IOError as e:
            raise ConfigurationError(six.text_type(e))

    def _load(self, filename, envvars):
        working_dir = os.path.dirname(filename)
        if envvars:
            loaded = self._load_yaml(filename)
        else:
            loaded = config.load_yaml(filename)
        return config.from_dictionary(loaded,
                                      working_dir=working_dir,
                                      filename=filename)

    def _get_config(self, name, envvars=False):
        for filename in self._get_projects_in_dir(True):
            if re.search('%s(.yaml|.yml)$' % name, filename):
                return self._load(filename, envvars)
        raise ConfigurationError("Project filename '%s' not found in the current directory" % name)

    def _get_projects_in_dir(self, fullpath=False):
        projects = []
        cwd = os.getcwd()
        for filename in os.listdir(cwd):
            if yaml_re.search(filename):
                if fullpath:
                    projects.append(path.join(cwd, filename))
                else:
                    projects.append(self._clean_project_name(filename))
        return projects

    def _ps(self):
        client = docker_client()
        projectnames = self._get_projects_in_dir()
        containers = client.containers(all=True)
        unknown = {}
        for container in containers:
            unknown[container['Id']] = client.inspect_container(
                container['Id'])
        if len(projectnames) != 0:
            print()
        for projectname in projectnames:
            config = self._get_config(projectname)
            project = Project.from_dicts(
                projectname,
                config,
                client)
            services = project.get_services()

            counts = {}

            for service in services:
                c = service.containers(stopped=True) + service.containers(one_off=True)
                if len(c) == 0:
                    if not 'Uncreated' in counts:
                        counts['Uncreated'] = 0
                    counts['Uncreated'] += 1
                for container in c:
                    del unknown[container.id]
                    if not container.human_readable_state in counts:
                        counts[container.human_readable_state] = 0
                    counts[container.human_readable_state] += 1

            humancounts = []
            for state in counts:
                humancounts.append('{count} {state}'.format(
                    count=counts[state],
                    state=state))
            print('  {name: <24}{counts}'.format(
                name=projectname,
                counts=','.join(humancounts)))

        if len(unknown) != 0:
            print()
            print('  Containers not tracked by compose/tugboat:')
            print()
        for key in unknown:
            detail = unknown[key]
            name = detail['Name']
            if name.startswith('/'):
                name = name[1:]
            ip = detail['NetworkSettings']['IPAddress']
            if ip == '':
                ip = '(host)'
            print('  {name: <24}{state: <12}{ip: <17}'.format(
                name=name,
                state='',
                ip=ip))
        print()

    def ps(self, project, projectname, servicenames):
        print()
        print('  {name} services:'.format(name=projectname))
        print()
        services = project.get_services(service_names=servicenames)
        for service in services:
            containers = service.containers(stopped=True) + service.containers(one_off=True)
            if len(containers) == 0:
                print('  {name: <24}{state: <12}{ip: <17}'.format(
                    name=service.name,
                    state='Uncreated',
                    ip=''))
            for container in containers:
                name = container.name_without_project
                ip = container.get('NetworkSettings.IPAddress')
                if ip == '':
                    ip = '(host)'
                state = container.human_readable_state
                print('  {name: <24}{state: <12}{ip: <17}'.format(
                    name=name,
                    state=state,
                    ip=ip))
        print()

    def build(self, project, projectname, servicenames):
        project.build(service_names=servicenames, no_cache=False)

    def rebuild(self, project, projectname, servicenames):
        project.build(service_names=servicenames, no_cache=True)

    def kill(self, project, projectname, servicenames):
        project.kill(service_names=servicenames, signal='SIGTERM')
        self.ps(project, projectname, servicenames)

    def logs(self, project, projectname, servicenames):
        containers = project.containers(
            service_names=servicenames,
            stopped=True)
        print('Attaching to', ', '.join(c.name for c in containers))
        LogPrinter(containers, attach_params={'logs': True}).run()

    def pull(self, project, projectname, servicenames):
        project.pull(service_names=servicenames)

    def rm(self, project, projectname, servicenames):
        project.remove_stopped(service_names=servicenames)

        self.ps(project, projectname, servicenames)

    def down(self, project, projectname, servicenames):
        project.kill(service_names=servicenames, signal='SIGTERM')
        project.stop(service_names=servicenames)

        self.ps(project, projectname, servicenames)

    def cull(self, project, projectname, servicenames):
        project.kill(service_names=servicenames, signal='SIGTERM')
        project.stop(service_names=servicenames)
        project.remove_stopped(service_names=servicenames)

        self.ps(project, projectname, servicenames)

    def recreate(self, project, projectname, servicenames):
        project.restart(service_names=servicenames)

        self.ps(project, projectname, servicenames)

    def up(self, project, projectname, servicenames):
        containers = project.containers(stopped=True) + project.containers(one_off=True)
        unknown = {}
        for container in containers:
            unknown[container.id] = container
        services = project.get_services(servicenames, include_deps=True)
        plans = project._get_convergence_plans(services, smart_recreate=True)
        for service in plans:
            plan = plans[service]
            for container in plan.containers:
                del unknown[container.id]
        project.up(
            service_names=servicenames,
            smart_recreate=True)

        if len(servicenames) == 0:
            for id in unknown:
                container = unknown[id]
                if container.is_running:
                    container.kill(signal='SIGTERM')
                    try:
                        container.client.wait(container.id, timeout=10)
                    except ReadTimeout as e:
                        pass
                    container.stop()
                    try:
                        container.client.wait(container.id, timeout=10)
                    except ReadTimeout as e:
                        pass
                container.remove()

        self.ps(project, projectname, servicenames)

    def diff(self, project, projectname, servicenames):
        containers = project.containers(stopped=True) + project.containers(one_off=True)
        unknown = {}
        for container in containers:
            unknown[container.id] = container
        services = project.get_services(servicenames, include_deps=True)
        plans = project._get_convergence_plans(services, smart_recreate=True)

        print()
        print('  {name} convergence plan:'.format(name=projectname))
        print()
        for service in plans:
            plan = plans[service]
            service_containers = []
            for container in plan.containers:
                del unknown[container.id]
                service_containers.append(container.name)
            print('  {name: <24}{action: <12}{containers}'.format(
                name=service,
                action=plan.action,
                containers=', '.join(service_containers)))
        # TODO: Add this in when up starts deleting unknown containers.
        for id in unknown:
            container = unknown[id]
            print('  {name: <24}{action: <12}'.format(
                name=container.name,
                action='delete'))
        print()

    # TODO: look in the yaml file?
    def _exec(self, projectname, servicename, commands):
        containername = '{projectname}_{servicename}_1'.format(
            projectname=projectname,
            servicename=servicename)
        command = commands
        if command is None or not command:
            command = ['/bin/bash']
        print()
        print('  docker exec -it {containername} {command}'.format(
            containername=containername,
            command=' '.join(command)))
        print()
        command = ['docker', 'exec', '-it', containername] + command
        sys.exit(subprocess.call(command))<|endoftext|>


<commit_before>#  Licensed under the Apache License, Version 2.0 (the "License"); you may
#  not use this file except in compliance with the License. You may obtain
#  a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#  License for the specific language governing permissions and limitations
#  under the License.

import copy
import itertools
import operator

from keystoneauth1 import loading as ks_loading
from oslo_config import cfg

import neutron.agent.common.config
import neutron.agent.common.ovs_lib
import neutron.agent.dhcp.config
import neutron.agent.l2.extensions.manager
import neutron.agent.l3.config
import neutron.agent.l3.ha
import neutron.agent.linux.interface
import neutron.agent.linux.pd
import neutron.agent.linux.ra
import neutron.agent.metadata.config
import neutron.agent.ovsdb.api
import neutron.agent.securitygroups_rpc
import neutron.conf.quota
import neutron.db.agents_db
import neutron.db.agentschedulers_db
import neutron.db.dvr_mac_db
import neutron.db.extraroute_db
import neutron.db.l3_agentschedulers_db
import neutron.db.l3_dvr_db
import neutron.db.l3_gwmode_db
import neutron.db.l3_hamode_db
import neutron.db.migration.cli
import neutron.extensions.allowedaddresspairs
import neutron.extensions.l3
import neutron.extensions.securitygroup
import neutron.openstack.common.cache.cache
import neutron.plugins.ml2.config
import neutron.plugins.ml2.drivers.agent.config
import neutron.plugins.ml2.drivers.linuxbridge.agent.common.config
import neutron.plugins.ml2.drivers.macvtap.agent.config
import neutron.plugins.ml2.drivers.mech_sriov.agent.common.config
import neutron.plugins.ml2.drivers.mech_sriov.mech_driver.mech_driver
import neutron.plugins.ml2.drivers.openvswitch.agent.common.config
import neutron.plugins.ml2.drivers.type_flat
import neutron.plugins.ml2.drivers.type_geneve
import neutron.plugins.ml2.drivers.type_gre
import neutron.plugins.ml2.drivers.type_vlan
import neutron.plugins.ml2.drivers.type_vxlan
import neutron.service
import neutron.services.metering.agents.metering_agent
import neutron.services.qos.notification_drivers.manager
import neutron.wsgi

NOVA_GROUP = 'nova'

CONF = cfg.CONF

deprecations = {'nova.cafile': [cfg.DeprecatedOpt('ca_certificates_file',
                                                  group=NOVA_GROUP)],
                'nova.insecure': [cfg.DeprecatedOpt('api_insecure',
                                                    group=NOVA_GROUP)],
                'nova.timeout': [cfg.DeprecatedOpt('url_timeout',
                                                   group=NOVA_GROUP)]}

_nova_options = ks_loading.register_session_conf_options(
            CONF, NOVA_GROUP, deprecated_opts=deprecations)

def list_agent_opts():
    return [
        ('agent',
         itertools.chain(
             neutron.agent.common.config.ROOT_HELPER_OPTS,
             neutron.agent.common.config.AGENT_STATE_OPTS,
             neutron.agent.common.config.IPTABLES_OPTS,
             neutron.agent.common.config.PROCESS_MONITOR_OPTS,
             neutron.agent.common.config.AVAILABILITY_ZONE_OPTS)
         ),
        ('DEFAULT',
         itertools.chain(
             neutron.agent.common.config.INTERFACE_DRIVER_OPTS,
             neutron.agent.metadata.config.SHARED_OPTS,
             neutron.agent.metadata.config.DRIVER_OPTS)
         )
    ]

def list_extension_opts():
    return [
        ('DEFAULT',
         neutron.extensions.allowedaddresspairs.allowed_address_pair_opts),
        ('quotas',
         itertools.chain(
             neutron.conf.quota.l3_quota_opts,
             neutron.conf.quota.security_group_quota_opts)
         )
    ]

def list_db_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.db.agents_db.AGENT_OPTS,
             neutron.db.extraroute_db.extra_route_opts,
             neutron.db.l3_gwmode_db.OPTS,
             neutron.db.agentschedulers_db.AGENTS_SCHEDULER_OPTS,
             neutron.db.dvr_mac_db.dvr_mac_address_opts,
             neutron.db.l3_dvr_db.router_distributed_opts,
             neutron.db.l3_agentschedulers_db.L3_AGENTS_SCHEDULER_OPTS,
             neutron.db.l3_hamode_db.L3_HA_OPTS)
         ),
        ('database',
         neutron.db.migration.cli.get_engine_config())
    ]

def list_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.common.config.core_cli_opts,
             neutron.common.config.core_opts,
             neutron.wsgi.socket_opts,
             neutron.service.service_opts)
         ),
        (neutron.common.config.NOVA_CONF_SECTION,
         itertools.chain(
              neutron.common.config.nova_opts)
         ),
        ('quotas', neutron.conf.quota.core_quota_opts)
    ]

def list_qos_opts():
    return [
        ('DEFAULT',
         neutron.services.qos.notification_drivers.manager.QOS_PLUGIN_OPTS)
    ]

def list_base_agent_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.agent.linux.interface.OPTS,
             neutron.agent.common.config.INTERFACE_DRIVER_OPTS,
             neutron.agent.common.ovs_lib.OPTS)
         ),
        ('AGENT', neutron.agent.common.config.AGENT_STATE_OPTS)
    ]

def list_dhcp_agent_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.agent.dhcp.config.DHCP_AGENT_OPTS,
             neutron.agent.dhcp.config.DHCP_OPTS,
             neutron.agent.dhcp.config.DNSMASQ_OPTS)
         )
    ]

def list_linux_bridge_opts():
    return [
        ('linux_bridge',
         neutron.plugins.ml2.drivers.linuxbridge.agent.common.config.
         bridge_opts),
        ('vxlan',
         neutron.plugins.ml2.drivers.linuxbridge.agent.common.config.
         vxlan_opts),
        ('agent',
         neutron.plugins.ml2.drivers.agent.config.agent_opts),
        ('securitygroup',
         neutron.agent.securitygroups_rpc.security_group_opts)
    ]

def list_l3_agent_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.agent.l3.config.OPTS,
             neutron.service.service_opts,
             neutron.agent.l3.ha.OPTS,
             neutron.agent.linux.pd.OPTS,
             neutron.agent.linux.ra.OPTS)
         )
    ]

def list_macvtap_opts():
    return [
        ('macvtap',
         neutron.plugins.ml2.drivers.macvtap.agent.config.macvtap_opts),
        ('agent',
         neutron.plugins.ml2.drivers.agent.config.agent_opts),
        ('securitygroup',
         neutron.agent.securitygroups_rpc.security_group_opts)
    ]

def list_metadata_agent_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.agent.metadata.config.SHARED_OPTS,
             neutron.agent.metadata.config.METADATA_PROXY_HANDLER_OPTS,
             neutron.agent.metadata.config.UNIX_DOMAIN_METADATA_PROXY_OPTS,
             neutron.openstack.common.cache.cache._get_oslo_configs())
         ),
        ('AGENT', neutron.agent.common.config.AGENT_STATE_OPTS)
    ]

def list_metering_agent_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.services.metering.agents.metering_agent.MeteringAgent.
             Opts,
             neutron.agent.common.config.INTERFACE_DRIVER_OPTS)
         )
    ]

def list_ml2_conf_opts():
    return [
        ('ml2',
         neutron.plugins.ml2.config.ml2_opts),
        ('ml2_type_flat',
         neutron.plugins.ml2.drivers.type_flat.flat_opts),
        ('ml2_type_vlan',
         neutron.plugins.ml2.drivers.type_vlan.vlan_opts),
        ('ml2_type_gre',
         neutron.plugins.ml2.drivers.type_gre.gre_opts),
        ('ml2_type_vxlan',
         neutron.plugins.ml2.drivers.type_vxlan.vxlan_opts),
        ('ml2_type_geneve',
         neutron.plugins.ml2.drivers.type_geneve.geneve_opts),
        ('securitygroup',
         neutron.agent.securitygroups_rpc.security_group_opts)
    ]

def list_ml2_conf_sriov_opts():
    return [
        ('ml2_sriov',
         neutron.plugins.ml2.drivers.mech_sriov.mech_driver.mech_driver.
         sriov_opts)
    ]

def list_ovs_opts():
    return [
        ('ovs',
         itertools.chain(
             neutron.plugins.ml2.drivers.openvswitch.agent.common.config.
             ovs_opts,
             neutron.agent.ovsdb.api.OPTS)
         ),
        ('agent',
         neutron.plugins.ml2.drivers.openvswitch.agent.common.config.
         agent_opts),
        ('securitygroup',
         neutron.agent.securitygroups_rpc.security_group_opts)
    ]

def list_sriov_agent_opts():
    return [
        ('ml2_sriov',
         neutron.plugins.ml2.drivers.mech_sriov.agent.common.config.
         sriov_nic_opts),
        ('agent',
         neutron.agent.l2.extensions.manager.L2_AGENT_EXT_MANAGER_OPTS)
    ]

def list_auth_opts():
    opt_list = copy.deepcopy(_nova_options)
    opt_list.insert(0, ks_loading.get_auth_common_conf_options()[0])
    # NOTE(mhickey): There are a lot of auth plugins, we just generate
    # the config options for a few common ones
    plugins = ['password', 'v2password', 'v3password']
    for name in plugins:
        for plugin_option in ks_loading.get_auth_plugin_conf_options(name):
            if all(option.name != plugin_option.name for option in opt_list):
                opt_list.append(plugin_option)
    opt_list.sort(key=operator.attrgetter('name'))
    return [(NOVA_GROUP, opt_list)]
<commit_msg>sriov_nic config options were declared under wrong group<commit_after>#  Licensed under the Apache License, Version 2.0 (the "License"); you may
#  not use this file except in compliance with the License. You may obtain
#  a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#  License for the specific language governing permissions and limitations
#  under the License.

import copy
import itertools
import operator

from keystoneauth1 import loading as ks_loading
from oslo_config import cfg

import neutron.agent.common.config
import neutron.agent.common.ovs_lib
import neutron.agent.dhcp.config
import neutron.agent.l2.extensions.manager
import neutron.agent.l3.config
import neutron.agent.l3.ha
import neutron.agent.linux.interface
import neutron.agent.linux.pd
import neutron.agent.linux.ra
import neutron.agent.metadata.config
import neutron.agent.ovsdb.api
import neutron.agent.securitygroups_rpc
import neutron.conf.quota
import neutron.db.agents_db
import neutron.db.agentschedulers_db
import neutron.db.dvr_mac_db
import neutron.db.extraroute_db
import neutron.db.l3_agentschedulers_db
import neutron.db.l3_dvr_db
import neutron.db.l3_gwmode_db
import neutron.db.l3_hamode_db
import neutron.db.migration.cli
import neutron.extensions.allowedaddresspairs
import neutron.extensions.l3
import neutron.extensions.securitygroup
import neutron.openstack.common.cache.cache
import neutron.plugins.ml2.config
import neutron.plugins.ml2.drivers.agent.config
import neutron.plugins.ml2.drivers.linuxbridge.agent.common.config
import neutron.plugins.ml2.drivers.macvtap.agent.config
import neutron.plugins.ml2.drivers.mech_sriov.agent.common.config
import neutron.plugins.ml2.drivers.mech_sriov.mech_driver.mech_driver
import neutron.plugins.ml2.drivers.openvswitch.agent.common.config
import neutron.plugins.ml2.drivers.type_flat
import neutron.plugins.ml2.drivers.type_geneve
import neutron.plugins.ml2.drivers.type_gre
import neutron.plugins.ml2.drivers.type_vlan
import neutron.plugins.ml2.drivers.type_vxlan
import neutron.service
import neutron.services.metering.agents.metering_agent
import neutron.services.qos.notification_drivers.manager
import neutron.wsgi

NOVA_GROUP = 'nova'

CONF = cfg.CONF

deprecations = {'nova.cafile': [cfg.DeprecatedOpt('ca_certificates_file',
                                                  group=NOVA_GROUP)],
                'nova.insecure': [cfg.DeprecatedOpt('api_insecure',
                                                    group=NOVA_GROUP)],
                'nova.timeout': [cfg.DeprecatedOpt('url_timeout',
                                                   group=NOVA_GROUP)]}

_nova_options = ks_loading.register_session_conf_options(
            CONF, NOVA_GROUP, deprecated_opts=deprecations)

def list_agent_opts():
    return [
        ('agent',
         itertools.chain(
             neutron.agent.common.config.ROOT_HELPER_OPTS,
             neutron.agent.common.config.AGENT_STATE_OPTS,
             neutron.agent.common.config.IPTABLES_OPTS,
             neutron.agent.common.config.PROCESS_MONITOR_OPTS,
             neutron.agent.common.config.AVAILABILITY_ZONE_OPTS)
         ),
        ('DEFAULT',
         itertools.chain(
             neutron.agent.common.config.INTERFACE_DRIVER_OPTS,
             neutron.agent.metadata.config.SHARED_OPTS,
             neutron.agent.metadata.config.DRIVER_OPTS)
         )
    ]

def list_extension_opts():
    return [
        ('DEFAULT',
         neutron.extensions.allowedaddresspairs.allowed_address_pair_opts),
        ('quotas',
         itertools.chain(
             neutron.conf.quota.l3_quota_opts,
             neutron.conf.quota.security_group_quota_opts)
         )
    ]

def list_db_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.db.agents_db.AGENT_OPTS,
             neutron.db.extraroute_db.extra_route_opts,
             neutron.db.l3_gwmode_db.OPTS,
             neutron.db.agentschedulers_db.AGENTS_SCHEDULER_OPTS,
             neutron.db.dvr_mac_db.dvr_mac_address_opts,
             neutron.db.l3_dvr_db.router_distributed_opts,
             neutron.db.l3_agentschedulers_db.L3_AGENTS_SCHEDULER_OPTS,
             neutron.db.l3_hamode_db.L3_HA_OPTS)
         ),
        ('database',
         neutron.db.migration.cli.get_engine_config())
    ]

def list_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.common.config.core_cli_opts,
             neutron.common.config.core_opts,
             neutron.wsgi.socket_opts,
             neutron.service.service_opts)
         ),
        (neutron.common.config.NOVA_CONF_SECTION,
         itertools.chain(
              neutron.common.config.nova_opts)
         ),
        ('quotas', neutron.conf.quota.core_quota_opts)
    ]

def list_qos_opts():
    return [
        ('DEFAULT',
         neutron.services.qos.notification_drivers.manager.QOS_PLUGIN_OPTS)
    ]

def list_base_agent_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.agent.linux.interface.OPTS,
             neutron.agent.common.config.INTERFACE_DRIVER_OPTS,
             neutron.agent.common.ovs_lib.OPTS)
         ),
        ('AGENT', neutron.agent.common.config.AGENT_STATE_OPTS)
    ]

def list_dhcp_agent_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.agent.dhcp.config.DHCP_AGENT_OPTS,
             neutron.agent.dhcp.config.DHCP_OPTS,
             neutron.agent.dhcp.config.DNSMASQ_OPTS)
         )
    ]

def list_linux_bridge_opts():
    return [
        ('linux_bridge',
         neutron.plugins.ml2.drivers.linuxbridge.agent.common.config.
         bridge_opts),
        ('vxlan',
         neutron.plugins.ml2.drivers.linuxbridge.agent.common.config.
         vxlan_opts),
        ('agent',
         neutron.plugins.ml2.drivers.agent.config.agent_opts),
        ('securitygroup',
         neutron.agent.securitygroups_rpc.security_group_opts)
    ]

def list_l3_agent_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.agent.l3.config.OPTS,
             neutron.service.service_opts,
             neutron.agent.l3.ha.OPTS,
             neutron.agent.linux.pd.OPTS,
             neutron.agent.linux.ra.OPTS)
         )
    ]

def list_macvtap_opts():
    return [
        ('macvtap',
         neutron.plugins.ml2.drivers.macvtap.agent.config.macvtap_opts),
        ('agent',
         neutron.plugins.ml2.drivers.agent.config.agent_opts),
        ('securitygroup',
         neutron.agent.securitygroups_rpc.security_group_opts)
    ]

def list_metadata_agent_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.agent.metadata.config.SHARED_OPTS,
             neutron.agent.metadata.config.METADATA_PROXY_HANDLER_OPTS,
             neutron.agent.metadata.config.UNIX_DOMAIN_METADATA_PROXY_OPTS,
             neutron.openstack.common.cache.cache._get_oslo_configs())
         ),
        ('AGENT', neutron.agent.common.config.AGENT_STATE_OPTS)
    ]

def list_metering_agent_opts():
    return [
        ('DEFAULT',
         itertools.chain(
             neutron.services.metering.agents.metering_agent.MeteringAgent.
             Opts,
             neutron.agent.common.config.INTERFACE_DRIVER_OPTS)
         )
    ]

def list_ml2_conf_opts():
    return [
        ('ml2',
         neutron.plugins.ml2.config.ml2_opts),
        ('ml2_type_flat',
         neutron.plugins.ml2.drivers.type_flat.flat_opts),
        ('ml2_type_vlan',
         neutron.plugins.ml2.drivers.type_vlan.vlan_opts),
        ('ml2_type_gre',
         neutron.plugins.ml2.drivers.type_gre.gre_opts),
        ('ml2_type_vxlan',
         neutron.plugins.ml2.drivers.type_vxlan.vxlan_opts),
        ('ml2_type_geneve',
         neutron.plugins.ml2.drivers.type_geneve.geneve_opts),
        ('securitygroup',
         neutron.agent.securitygroups_rpc.security_group_opts)
    ]

def list_ml2_conf_sriov_opts():
    return [
        ('ml2_sriov',
         neutron.plugins.ml2.drivers.mech_sriov.mech_driver.mech_driver.
         sriov_opts)
    ]

def list_ovs_opts():
    return [
        ('ovs',
         itertools.chain(
             neutron.plugins.ml2.drivers.openvswitch.agent.common.config.
             ovs_opts,
             neutron.agent.ovsdb.api.OPTS)
         ),
        ('agent',
         neutron.plugins.ml2.drivers.openvswitch.agent.common.config.
         agent_opts),
        ('securitygroup',
         neutron.agent.securitygroups_rpc.security_group_opts)
    ]

def list_sriov_agent_opts():
    return [
        ('sriov_nic',
         neutron.plugins.ml2.drivers.mech_sriov.agent.common.config.
         sriov_nic_opts),
        ('agent',
         neutron.agent.l2.extensions.manager.L2_AGENT_EXT_MANAGER_OPTS)
    ]

def list_auth_opts():
    opt_list = copy.deepcopy(_nova_options)
    opt_list.insert(0, ks_loading.get_auth_common_conf_options()[0])
    # NOTE(mhickey): There are a lot of auth plugins, we just generate
    # the config options for a few common ones
    plugins = ['password', 'v2password', 'v3password']
    for name in plugins:
        for plugin_option in ks_loading.get_auth_plugin_conf_options(name):
            if all(option.name != plugin_option.name for option in opt_list):
                opt_list.append(plugin_option)
    opt_list.sort(key=operator.attrgetter('name'))
    return [(NOVA_GROUP, opt_list)]
<|endoftext|>


<commit_before>"""
 SlipStream Client
 =====
 Copyright (C) 2014 SixSq Sarl (sixsq.com)
 =====
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
import sys

from slipstream.command.CloudClientCommand import CloudClientCommand
from slipstream.NodeDecorator import NodeDecorator, KEY_RUN_CATEGORY, RUN_CATEGORY_DEPLOYMENT
from slipstream.NodeInstance import NodeInstance
from slipstream.ConfigHolder import ConfigHolder
from slipstream.util import nostdouterr

saved_stdout = sys.stdout

def publish_vm_info(self, vm, node_instance):
    print >> saved_stdout, '%s,%s' % (self._vm_get_id(vm), self._vm_get_ip(vm))

class RunInstancesCommand(CloudClientCommand):

    IMAGE_ID_KEY = 'image-id'
    PLATFORM_KEY = 'platform'
    LOGIN_USER_KEY = 'login-username'
    NETWORK_TYPE = 'network-type'
    EXTRA_DISK_VOLATILE = 'extra-disk-volatile'
    DEFAULT_TIMEOUT = 600

    def get_cloud_specific_node_inst_cloud_params(self):
        return {}

    def get_cloud_specific_node_inst_image_attributes(self):
        return {}

    def get_cloud_specific_node_inst_attributes(self):
        return {}

    def __init__(self):
        super(RunInstancesCommand, self).__init__()

    def _set_command_specific_options(self, parser):
        parser.add_option('--' + self.IMAGE_ID_KEY, dest=self.IMAGE_ID_KEY,
                          help='Image ID', default='', metavar='IMAGEID')

        parser.add_option('--' + self.PLATFORM_KEY, dest=self.PLATFORM_KEY,
                          help='Platform (eg: Ubuntu, CentOS, Windows, ...)',
                          default='linux', metavar='PLATFORM')

        parser.add_option('--' + self.NETWORK_TYPE, dest=self.NETWORK_TYPE,
                          help='Network type (public or private)',
                          default='Public', metavar='NETWORK-TYPE')

        parser.add_option('--' + self.EXTRA_DISK_VOLATILE, dest=self.EXTRA_DISK_VOLATILE,
                          help='Size of the volatile extra disk (in GB)',
                          default='', metavar='EXTRA-DISK-VOLATILE')

        parser.add_option('--' + self.LOGIN_USER_KEY, dest=self.LOGIN_USER_KEY,
                          help='Username of the user to use to contextualize the instance',
                          default='', metavar='USER')

        parser.add_option('--' + NodeDecorator.NATIVE_CONTEXTUALIZATION_KEY, dest=NodeDecorator.NATIVE_CONTEXTUALIZATION_KEY,
                          help='When SlipStream should use the native Cloud contextualization mechanisme (instead of SSH/WinRM)\nPossible values: never|always|linux only|windows only',
                          default='', metavar='NATIVE_CONTEXTUALIZATION')

    def _get_command_mandatory_options(self):
        return [self.IMAGE_ID_KEY,
                self.PLATFORM_KEY,
                self.NETWORK_TYPE]

    def _get_command_specific_user_cloud_params(self):
        native_contextualization = self.get_option(NodeDecorator.NATIVE_CONTEXTUALIZATION_KEY)
        if native_contextualization:
            return {NodeDecorator.NATIVE_CONTEXTUALIZATION_KEY: native_contextualization}
        else:
            return {}

    def _get_default_timeout(self):
        return self.DEFAULT_TIMEOUT

    def _get_node_instance(self):
        runtime_parameters = {
            NodeDecorator.NODE_INSTANCE_NAME_KEY: self.get_node_instance_name(),
            'cloudservice': self._cloud_instance_name,
            'image.platform': self.get_option(self.PLATFORM_KEY),
            'image.imageId': self.get_option(self.IMAGE_ID_KEY),
            'image.id': self.get_option(self.IMAGE_ID_KEY),
            'network': self.get_option(self.NETWORK_TYPE)
        }

        if self.get_option(self.EXTRA_DISK_VOLATILE):
            runtime_parameters.update({'extra.disk.volatile':
                                        self.get_option(self.EXTRA_DISK_VOLATILE)})

        if self.get_option(self.LOGIN_USER_KEY):
            runtime_parameters.update({NodeInstance.IMAGE_ATTRIBUTE_PREFIX + '.' + NodeDecorator.LOGIN_USER_KEY:
                                       self.get_option(self.LOGIN_USER_KEY)})

        return NodeInstance(runtime_parameters)

    def do_work(self):
        node_instance = self._get_node_instance()
        node_instance.set_cloud_parameters(self.get_cloud_specific_node_inst_cloud_params())
        node_instance.set_image_attributes(self.get_cloud_specific_node_inst_image_attributes())
        node_instance.set_attributes(self.get_cloud_specific_node_inst_attributes())

        with nostdouterr(self.get_option('verbose')):
            self._run_instance(node_instance)

    def _run_instance(self, node_instance):
        nodename = node_instance.get_name()

        cloud_connector_class = self.get_connector_class()
        cloud_connector_class._publish_vm_info = publish_vm_info

        verbose_level = self.get_option('verbose') and 3 or 0
        ch = ConfigHolder(options={'verboseLevel': verbose_level,
                                   'http_max_retries': 0,
                                    KEY_RUN_CATEGORY: RUN_CATEGORY_DEPLOYMENT},
                          context={'foo': 'bar'},
                          config={'foo': 'bar'})

        cc = cloud_connector_class(ch)
        cc.start_nodes_and_clients(self.user_info, {nodename: node_instance},
                                   self.get_initialization_extra_kwargs())
<commit_msg> Replaced spaces by dashes in "native contextualization options"<commit_after>"""
 SlipStream Client
 =====
 Copyright (C) 2014 SixSq Sarl (sixsq.com)
 =====
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
import sys

from slipstream.command.CloudClientCommand import CloudClientCommand
from slipstream.NodeDecorator import NodeDecorator, KEY_RUN_CATEGORY, RUN_CATEGORY_DEPLOYMENT
from slipstream.NodeInstance import NodeInstance
from slipstream.ConfigHolder import ConfigHolder
from slipstream.util import nostdouterr

saved_stdout = sys.stdout

def publish_vm_info(self, vm, node_instance):
    print >> saved_stdout, '%s,%s' % (self._vm_get_id(vm), self._vm_get_ip(vm))

class RunInstancesCommand(CloudClientCommand):

    IMAGE_ID_KEY = 'image-id'
    PLATFORM_KEY = 'platform'
    LOGIN_USER_KEY = 'login-username'
    NETWORK_TYPE = 'network-type'
    EXTRA_DISK_VOLATILE = 'extra-disk-volatile'
    DEFAULT_TIMEOUT = 600

    def get_cloud_specific_node_inst_cloud_params(self):
        return {}

    def get_cloud_specific_node_inst_image_attributes(self):
        return {}

    def get_cloud_specific_node_inst_attributes(self):
        return {}

    def __init__(self):
        super(RunInstancesCommand, self).__init__()

    def _set_command_specific_options(self, parser):
        parser.add_option('--' + self.IMAGE_ID_KEY, dest=self.IMAGE_ID_KEY,
                          help='Image ID', default='', metavar='IMAGEID')

        parser.add_option('--' + self.PLATFORM_KEY, dest=self.PLATFORM_KEY,
                          help='Platform (eg: Ubuntu, CentOS, Windows, ...)',
                          default='linux', metavar='PLATFORM')

        parser.add_option('--' + self.NETWORK_TYPE, dest=self.NETWORK_TYPE,
                          help='Network type (public or private)',
                          default='Public', metavar='NETWORK-TYPE')

        parser.add_option('--' + self.EXTRA_DISK_VOLATILE, dest=self.EXTRA_DISK_VOLATILE,
                          help='Size of the volatile extra disk (in GB)',
                          default='', metavar='EXTRA-DISK-VOLATILE')

        parser.add_option('--' + self.LOGIN_USER_KEY, dest=self.LOGIN_USER_KEY,
                          help='Username of the user to use to contextualize the instance',
                          default='', metavar='USER')

        parser.add_option('--' + NodeDecorator.NATIVE_CONTEXTUALIZATION_KEY, dest=NodeDecorator.NATIVE_CONTEXTUALIZATION_KEY,
                          help='When SlipStream should use the native Cloud contextualization (instead of SSH/WinRM)\nPossible values: never|always|linux-only|windows-only',
                          default='', metavar='NATIVE_CONTEXTUALIZATION')

    def _get_command_mandatory_options(self):
        return [self.IMAGE_ID_KEY,
                self.PLATFORM_KEY,
                self.NETWORK_TYPE]

    def _get_command_specific_user_cloud_params(self):
        native_contextualization = self.get_option(NodeDecorator.NATIVE_CONTEXTUALIZATION_KEY)
        if native_contextualization:
            return {NodeDecorator.NATIVE_CONTEXTUALIZATION_KEY: native_contextualization}
        else:
            return {}

    def _get_default_timeout(self):
        return self.DEFAULT_TIMEOUT

    def _get_node_instance(self):
        runtime_parameters = {
            NodeDecorator.NODE_INSTANCE_NAME_KEY: self.get_node_instance_name(),
            'cloudservice': self._cloud_instance_name,
            'image.platform': self.get_option(self.PLATFORM_KEY),
            'image.imageId': self.get_option(self.IMAGE_ID_KEY),
            'image.id': self.get_option(self.IMAGE_ID_KEY),
            'network': self.get_option(self.NETWORK_TYPE)
        }

        if self.get_option(self.EXTRA_DISK_VOLATILE):
            runtime_parameters.update({'extra.disk.volatile':
                                        self.get_option(self.EXTRA_DISK_VOLATILE)})

        if self.get_option(self.LOGIN_USER_KEY):
            runtime_parameters.update({NodeInstance.IMAGE_ATTRIBUTE_PREFIX + '.' + NodeDecorator.LOGIN_USER_KEY:
                                       self.get_option(self.LOGIN_USER_KEY)})

        return NodeInstance(runtime_parameters)

    def do_work(self):
        node_instance = self._get_node_instance()
        node_instance.set_cloud_parameters(self.get_cloud_specific_node_inst_cloud_params())
        node_instance.set_image_attributes(self.get_cloud_specific_node_inst_image_attributes())
        node_instance.set_attributes(self.get_cloud_specific_node_inst_attributes())

        with nostdouterr(self.get_option('verbose')):
            self._run_instance(node_instance)

    def _run_instance(self, node_instance):
        nodename = node_instance.get_name()

        cloud_connector_class = self.get_connector_class()
        cloud_connector_class._publish_vm_info = publish_vm_info

        verbose_level = self.get_option('verbose') and 3 or 0
        ch = ConfigHolder(options={'verboseLevel': verbose_level,
                                   'http_max_retries': 0,
                                    KEY_RUN_CATEGORY: RUN_CATEGORY_DEPLOYMENT},
                          context={'foo': 'bar'},
                          config={'foo': 'bar'})

        cc = cloud_connector_class(ch)
        cc.start_nodes_and_clients(self.user_info, {nodename: node_instance},
                                   self.get_initialization_extra_kwargs())
<|endoftext|>