<commit_before>#!/usr/bin/env python

from __future__ import division
from os.path import isfile, exists
from os.path import join as pjoin
from os import mkdir
from argparse import ArgumentParser
from numpy import zeros
from biom.parse import parse_biom_table
from americangut.make_phyla_plots import (map_to_2D_dict,
                              summarize_human_taxa,
                              plot_american_gut,
                              load_category_files)

__author__ = "Justine Debelius"
__copyright__ = "Copyright 2013, The American Gut Project"
__credits__ = ["Justine Debelius"]
__license__ = "BSD"
__version__ = "unversioned"
__maintainer__ = "Justine Debelius"
__email__ = "Justine.Debelius@colorado.edu"

def main(otu_table, mapping_data, categories, output_dir, \
    samples_to_plot = None, legend = False, xaxis = True):
    """Creates stacked bar plots for an otu table
    INPUTS:
        otu_table -- an open OTU table

        mapping_data -- a tab delimited string containing the mapping data
                    passed from the mapping file.

        categories -- a dictionary keying a mapping category to the
                    corresponding sample IDs and taxonomy for a collapsed
                    biom table

        output_dir -- the location of the directory where output files should be
                    saved. If this directory does not exist, it will be created.

        samples_to_plot -- a list of sample ids to plot. If no value is passed,
                    then all samples in the biom table are analyzed.

    OUTPUTS:
        A pdf of stacked taxonomy will be generated for each sample and saved
        in the output directory. These will follow the file name format
        Figure_4_<SAMPLEID>.pdf
    """
    # Sets constants
    LEVEL = 2
    FILEPREFIX = 'Figure_4_'
    MICHAEL_POLLAN = '000007108.1075657'
    NUM_TAXA = 9
    NUM_CATS_TO_PLOT = 7

    # Loads the mapping file
    map_dict = map_to_2D_dict(mapping_data)

    (common_taxa, whole_sample_ids, whole_summary) = \
        summarize_human_taxa(otu_table, LEVEL)

    # Converts final taxa to a clean list
    common_phyla = []
    for taxon in common_taxa:
        common_phyla.append(taxon[1].strip(' p__').strip('[').strip(']'))
    common_taxa = common_phyla

    # Checks that the correct sample ids are plotted
    if samples_to_plot == None:
        sample_ids = whole_sample_ids
    else:
        sample_ids = samples_to_plot

    # Identifies Michael Pollan's pre-ABX sample
    mp_sample_pos = whole_sample_ids.index(MICHAEL_POLLAN)
    mp_sample_taxa = whole_summary[:,mp_sample_pos]

    # Loads the category dictionary
    categories = load_category_files(category_fp, LEVEL)

    # Generates a figure for each sample
    for idx, sample_id in enumerate(whole_sample_ids):
        if sample_id in sample_ids:
            # Preallocates a numpy array for the plotting data
            tax_array = zeros((NUM_TAXA, NUM_CATS_TO_PLOT))
            meta_data = map_dict[sample_id]
            cat_list = ['You', 'Average', 'Similar Diet', ' Similar BMI',
                        'Same Gender', 'Similar Age',
                        'Michael Pollan', '']

            #cat_list.append('Your Fecal Sample')
            #cat_list.append('Average Fecal Samples')

            tax_array[:,0] = whole_summary[:,idx]
            tax_array[:,1] = mean(whole_summary, 1)

            cat_watch = 2
            # Identifies the appropriate metadata categories
            for cat in categories:
                # Pulls metadata for the sample and category
                mapping_key = meta_data[cat]
                # Pulls taxonomic summary and group descriptions for the category
                tax_summary = categories[cat]['Taxa Summary']
                group_descriptions = categories[cat]['Groups']
                # Amends plotting tables
                try:
                    mapping_col = group_descriptions.index(mapping_key)
                except:
                    raise ValueError, 'The %s cannot be found in %s.' \
                    % (mapping_key, cat)
                tax_array[:,cat_watch] = tax_summary[:,mapping_col]

                cat_watch = cat_watch + 1

            tax_array[:,-1] = mp_sample_taxa
            # Plots the data
            filename = pjoin(output_dir, '%s%s.pdf' \
                % (FILEPREFIX, sample_id))
            plot_american_gut(tax_array, filename)

# Sets up the command line interface
## This uses argparse instead of optparse since optparse is being phased out
## with new versions of Python.

# Creates the parser object
parser = ArgumentParser(description = 'Creates stacked bar plots for an OTU'\
                        ' table.')
parser.add_argument('-i', '--input', required = True, \
                    help = 'OTU table file path [REQUIRED]')
parser.add_argument('-m', '--mapping', required = True, \
                    help = 'Mapping file path [REQUIRED]')
parser.add_argument('-o', '--output', required = True, \
                    help = 'Path to the output directory [REQUIRED]'),
parser.add_argument('-c', '--categories', \
                    help = 'Category associations with a collapsed OTU file '\
                    'path. The string should be associated with a colon, for'\
                    ' example, "SEX:sex.biom,DIET_TYPE:diet.biom"'),
parser.add_argument('-s', '--samples_to_plot', default = None, \
                    help = 'Sample IDs you wish to plot. If no value is '\
                    'specified, all samples are plotted.')

if __name__ == '__main__':
    # Sets the plotting level at phylum
    LEVEL = 2

    args = parser.parse_args()

    # Checks the biom table is sane
    if not args.input:
        parser.error("An input BIOM table is required.")
    elif not isfile(args.input):
        parse.error('The supplied biom table does not exist in the path.')
    else:
        otu_table = parse_biom_table(open(args.input, 'U'))

    # Checks the mapping file is sane
    if not args.mapping:
        parser.error("An input mapping file is required.")
    elif not isfile(args.mapping):
        parser.error('he supplied file does not exist in the path')
    else:
        mapping = open(args.mapping, 'U')

    # Checks the output directory is sane
    if not args.output:
        parser.error("An output directory is required.")
    elif not exists(args.output):
        mkdir(args.output)
    output_dir = args.output

    # Parses the category argument
    if not args.categories:
        categories = {}
    else:
        category_fp = dict([c.strip().split(':') \
            for c in args.categories.split(',')])
        categories = load_category_files(category_fp, LEVEL)

    # Deals with the sample list
    if args.samples_to_plot:
        samples = args.samples_to_plot
        samples = samples.split(',')
    else:
        samples = None

    main(otu_table, mapping, output_dir = output_dir, \
        categories = categories, samples_to_plot = samples)
<commit_msg>another...<commit_after>#!/usr/bin/env python

from __future__ import division
from os.path import isfile, exists
from os.path import join as pjoin
from os import mkdir
from argparse import ArgumentParser
from numpy import zeros, mean
from biom.parse import parse_biom_table
from americangut.make_phyla_plots import (map_to_2D_dict,
                              summarize_human_taxa,
                              plot_american_gut,
                              load_category_files)

__author__ = "Justine Debelius"
__copyright__ = "Copyright 2013, The American Gut Project"
__credits__ = ["Justine Debelius"]
__license__ = "BSD"
__version__ = "unversioned"
__maintainer__ = "Justine Debelius"
__email__ = "Justine.Debelius@colorado.edu"

def main(otu_table, mapping_data, categories, output_dir, \
    samples_to_plot = None, legend = False, xaxis = True):
    """Creates stacked bar plots for an otu table
    INPUTS:
        otu_table -- an open OTU table

        mapping_data -- a tab delimited string containing the mapping data
                    passed from the mapping file.

        categories -- a dictionary keying a mapping category to the
                    corresponding sample IDs and taxonomy for a collapsed
                    biom table

        output_dir -- the location of the directory where output files should be
                    saved. If this directory does not exist, it will be created.

        samples_to_plot -- a list of sample ids to plot. If no value is passed,
                    then all samples in the biom table are analyzed.

    OUTPUTS:
        A pdf of stacked taxonomy will be generated for each sample and saved
        in the output directory. These will follow the file name format
        Figure_4_<SAMPLEID>.pdf
    """
    # Sets constants
    LEVEL = 2
    FILEPREFIX = 'Figure_4_'
    MICHAEL_POLLAN = '000007108.1075657'
    NUM_TAXA = 9
    NUM_CATS_TO_PLOT = 7

    # Loads the mapping file
    map_dict = map_to_2D_dict(mapping_data)

    (common_taxa, whole_sample_ids, whole_summary) = \
        summarize_human_taxa(otu_table, LEVEL)

    # Converts final taxa to a clean list
    common_phyla = []
    for taxon in common_taxa:
        common_phyla.append(taxon[1].strip(' p__').strip('[').strip(']'))
    common_taxa = common_phyla

    # Checks that the correct sample ids are plotted
    if samples_to_plot == None:
        sample_ids = whole_sample_ids
    else:
        sample_ids = samples_to_plot

    # Identifies Michael Pollan's pre-ABX sample
    mp_sample_pos = whole_sample_ids.index(MICHAEL_POLLAN)
    mp_sample_taxa = whole_summary[:,mp_sample_pos]

    # Loads the category dictionary
    categories = load_category_files(category_fp, LEVEL)

    # Generates a figure for each sample
    for idx, sample_id in enumerate(whole_sample_ids):
        if sample_id in sample_ids:
            # Preallocates a numpy array for the plotting data
            tax_array = zeros((NUM_TAXA, NUM_CATS_TO_PLOT))
            meta_data = map_dict[sample_id]
            cat_list = ['You', 'Average', 'Similar Diet', ' Similar BMI',
                        'Same Gender', 'Similar Age',
                        'Michael Pollan', '']

            #cat_list.append('Your Fecal Sample')
            #cat_list.append('Average Fecal Samples')

            tax_array[:,0] = whole_summary[:,idx]
            tax_array[:,1] = mean(whole_summary, 1)

            cat_watch = 2
            # Identifies the appropriate metadata categories
            for cat in categories:
                # Pulls metadata for the sample and category
                mapping_key = meta_data[cat]
                # Pulls taxonomic summary and group descriptions for the category
                tax_summary = categories[cat]['Taxa Summary']
                group_descriptions = categories[cat]['Groups']
                # Amends plotting tables
                try:
                    mapping_col = group_descriptions.index(mapping_key)
                except:
                    raise ValueError, 'The %s cannot be found in %s.' \
                    % (mapping_key, cat)
                tax_array[:,cat_watch] = tax_summary[:,mapping_col]

                cat_watch = cat_watch + 1

            tax_array[:,-1] = mp_sample_taxa
            # Plots the data
            filename = pjoin(output_dir, '%s%s.pdf' \
                % (FILEPREFIX, sample_id))
            plot_american_gut(tax_array, filename)

# Sets up the command line interface
## This uses argparse instead of optparse since optparse is being phased out
## with new versions of Python.

# Creates the parser object
parser = ArgumentParser(description = 'Creates stacked bar plots for an OTU'\
                        ' table.')
parser.add_argument('-i', '--input', required = True, \
                    help = 'OTU table file path [REQUIRED]')
parser.add_argument('-m', '--mapping', required = True, \
                    help = 'Mapping file path [REQUIRED]')
parser.add_argument('-o', '--output', required = True, \
                    help = 'Path to the output directory [REQUIRED]'),
parser.add_argument('-c', '--categories', \
                    help = 'Category associations with a collapsed OTU file '\
                    'path. The string should be associated with a colon, for'\
                    ' example, "SEX:sex.biom,DIET_TYPE:diet.biom"'),
parser.add_argument('-s', '--samples_to_plot', default = None, \
                    help = 'Sample IDs you wish to plot. If no value is '\
                    'specified, all samples are plotted.')

if __name__ == '__main__':
    # Sets the plotting level at phylum
    LEVEL = 2

    args = parser.parse_args()

    # Checks the biom table is sane
    if not args.input:
        parser.error("An input BIOM table is required.")
    elif not isfile(args.input):
        parse.error('The supplied biom table does not exist in the path.')
    else:
        otu_table = parse_biom_table(open(args.input, 'U'))

    # Checks the mapping file is sane
    if not args.mapping:
        parser.error("An input mapping file is required.")
    elif not isfile(args.mapping):
        parser.error('he supplied file does not exist in the path')
    else:
        mapping = open(args.mapping, 'U')

    # Checks the output directory is sane
    if not args.output:
        parser.error("An output directory is required.")
    elif not exists(args.output):
        mkdir(args.output)
    output_dir = args.output

    # Parses the category argument
    if not args.categories:
        categories = {}
    else:
        category_fp = dict([c.strip().split(':') \
            for c in args.categories.split(',')])
        categories = load_category_files(category_fp, LEVEL)

    # Deals with the sample list
    if args.samples_to_plot:
        samples = args.samples_to_plot
        samples = samples.split(',')
    else:
        samples = None

    main(otu_table, mapping, output_dir = output_dir, \
        categories = categories, samples_to_plot = samples)
<|endoftext|>


<commit_before># Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from buildbot.scheduler import Dependent
from buildbot.scheduler import Scheduler
from buildbot.scheduler import Triggerable

from master import slaves_list

def GetArchiveUrl(project, machine_name, builder_name, zip_os_name,
                  static_host=None):
  # static host can be used for connections made through an intermediary
  if static_host:
    host = static_host
  else:
    # This is slightly tricky since it depends on os.getcwd() being right.
    slaves = slaves_list.SlavesList('slaves.cfg', project)
    host = slaves.GetSlaveName(project, machine_name)
    if not host:
      raise ValueError("%s isn't reachable" % machine_name)
  return ('http://%s/b/build/slave/%s/chrome_staging/full-build-%s.zip' % (
              host, builder_name, zip_os_name))

def GetGSUtilUrl(gs_bucket, root_folder):
  return 'gs://%s/%s' % (gs_bucket, root_folder)

class Helper(object):
  def __init__(self, defaults):
    self._defaults = defaults
    self._builders = []
    self._factories = {}
    self._schedulers = {}

  def Builder(self, name, factory, gatekeeper=None, scheduler=None,
              builddir=None, auto_reboot=True, notify_on_missing=False):
    category = self._defaults.get('category')
    self._builders.append({'name': name,
                           'factory': factory,
                           'gatekeeper': gatekeeper,
                           'schedulers': scheduler.split('|'),
                           'builddir': builddir,
                           'category': category,
                           'auto_reboot': auto_reboot,
                           'notify_on_missing': notify_on_missing})

  def Dependent(self, name, parent):
    if name in self._schedulers:
      raise ValueError('Scheduler %s already exist' % name)
    self._schedulers[name] = {'type': 'Dependent',
                              'parent': parent,
                              'builders': []}

  def Triggerable(self, name):
    if name in self._schedulers:
      raise ValueError('Scheduler %s already exist' % name)
    self._schedulers[name] = {'type': 'Triggerable',
                              'builders': []}

  def Factory(self, name, factory):
    if name in self._factories:
      raise ValueError('Factory %s already exist' % name)
    self._factories[name] = factory

  def Scheduler(self, name, branch, treeStableTimer=60, categories=None):
    if name in self._schedulers:
      raise ValueError('Scheduler %s already exist' % name)
    self._schedulers[name] = {'type': 'Scheduler',
                              'branch': branch,
                              'treeStableTimer': treeStableTimer,
                              'builders': [],
                              'categories': categories}

  def Update(self, c):
    for builder in self._builders:
      # Update the schedulers with the builder.
      schedulers = builder['schedulers']
      if schedulers:
        for scheduler in schedulers:
          self._schedulers[scheduler]['builders'].append(builder['name'])

      # Construct the category.
      categories = []
      if builder.get('category', None):
        categories.append(builder['category'])
      if builder.get('gatekeeper', None):
        categories.extend(builder['gatekeeper'].split('|'))
      category = '|'.join(categories)

      # Append the builder to the list.
      new_builder = {'name': builder['name'],
                     'factory': self._factories[builder['factory']],
                     'category': category,
                     'auto_reboot': builder['auto_reboot']}
      if builder['builddir']:
        new_builder['builddir'] =  builder['builddir']
      c['builders'].append(new_builder)

    # Process the main schedulers
    for s_name in self._schedulers:
      scheduler = self._schedulers[s_name]
      if scheduler['type'] == 'Scheduler':
        instance = Scheduler(name=s_name,
                             branch=scheduler['branch'],
                             treeStableTimer=scheduler['treeStableTimer'],
                             builderNames=scheduler['builders'],
                             categories=scheduler['categories'])
        scheduler['instance'] = instance
        c['schedulers'].append(instance)

    # Process the dependent schedulers
    for s_name in self._schedulers:
      scheduler = self._schedulers[s_name]
      if scheduler['type'] == 'Dependent':
        c['schedulers'].append(
            Dependent(s_name,
                      self._schedulers[scheduler['parent']]['instance'],
                      scheduler['builders']))

    # Process the treggerable schedulers
    for s_name in self._schedulers:
      scheduler = self._schedulers[s_name]
      if scheduler['type'] == 'Triggerable':
        c['schedulers'].append(Triggerable(s_name,
                                           scheduler['builders']))
<commit_msg>Add nightly build method to helper class to easily generate nightly schedulers.<commit_after># Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from buildbot.scheduler import Dependent
from buildbot.scheduler import Nightly
from buildbot.scheduler import Scheduler
from buildbot.scheduler import Triggerable

from master import slaves_list

def GetArchiveUrl(project, machine_name, builder_name, zip_os_name,
                  static_host=None):
  # static host can be used for connections made through an intermediary
  if static_host:
    host = static_host
  else:
    # This is slightly tricky since it depends on os.getcwd() being right.
    slaves = slaves_list.SlavesList('slaves.cfg', project)
    host = slaves.GetSlaveName(project, machine_name)
    if not host:
      raise ValueError("%s isn't reachable" % machine_name)
  return ('http://%s/b/build/slave/%s/chrome_staging/full-build-%s.zip' % (
              host, builder_name, zip_os_name))

def GetGSUtilUrl(gs_bucket, root_folder):
  return 'gs://%s/%s' % (gs_bucket, root_folder)

class Helper(object):
  def __init__(self, defaults):
    self._defaults = defaults
    self._builders = []
    self._factories = {}
    self._schedulers = {}

  def Builder(self, name, factory, gatekeeper=None, scheduler=None,
              builddir=None, auto_reboot=True, notify_on_missing=False):
    category = self._defaults.get('category')
    self._builders.append({'name': name,
                           'factory': factory,
                           'gatekeeper': gatekeeper,
                           'schedulers': scheduler.split('|'),
                           'builddir': builddir,
                           'category': category,
                           'auto_reboot': auto_reboot,
                           'notify_on_missing': notify_on_missing})

  def Hourly(self, name, branch, hour='*'):
    """Helper method for the Nightly scheduler."""
    if name in self._schedulers:
      raise ValueError('Scheduler %s already exists' % name)
    self._schedulers[name] = {'type': 'Nightly',
                              'builders': [],
                              'branch': branch,
                              'hour': hour}

  def Dependent(self, name, parent):
    if name in self._schedulers:
      raise ValueError('Scheduler %s already exists' % name)
    self._schedulers[name] = {'type': 'Dependent',
                              'parent': parent,
                              'builders': []}

  def Triggerable(self, name):
    if name in self._schedulers:
      raise ValueError('Scheduler %s already exists' % name)
    self._schedulers[name] = {'type': 'Triggerable',
                              'builders': []}

  def Factory(self, name, factory):
    if name in self._factories:
      raise ValueError('Factory %s already exists' % name)
    self._factories[name] = factory

  def Scheduler(self, name, branch, treeStableTimer=60, categories=None):
    if name in self._schedulers:
      raise ValueError('Scheduler %s already exists' % name)
    self._schedulers[name] = {'type': 'Scheduler',
                              'branch': branch,
                              'treeStableTimer': treeStableTimer,
                              'builders': [],
                              'categories': categories}

  def Update(self, c):
    for builder in self._builders:
      # Update the schedulers with the builder.
      schedulers = builder['schedulers']
      if schedulers:
        for scheduler in schedulers:
          self._schedulers[scheduler]['builders'].append(builder['name'])

      # Construct the category.
      categories = []
      if builder.get('category', None):
        categories.append(builder['category'])
      if builder.get('gatekeeper', None):
        categories.extend(builder['gatekeeper'].split('|'))
      category = '|'.join(categories)

      # Append the builder to the list.
      new_builder = {'name': builder['name'],
                     'factory': self._factories[builder['factory']],
                     'category': category,
                     'auto_reboot': builder['auto_reboot']}
      if builder['builddir']:
        new_builder['builddir'] = builder['builddir']
      c['builders'].append(new_builder)

    # Process the main schedulers
    for s_name in self._schedulers:
      scheduler = self._schedulers[s_name]
      if scheduler['type'] == 'Scheduler':
        instance = Scheduler(name=s_name,
                             branch=scheduler['branch'],
                             treeStableTimer=scheduler['treeStableTimer'],
                             builderNames=scheduler['builders'],
                             categories=scheduler['categories'])
        scheduler['instance'] = instance
        c['schedulers'].append(instance)

    # Process the dependent schedulers
    for s_name in self._schedulers:
      scheduler = self._schedulers[s_name]
      if scheduler['type'] == 'Dependent':
        c['schedulers'].append(
            Dependent(s_name,
                      self._schedulers[scheduler['parent']]['instance'],
                      scheduler['builders']))

    # Process the triggerable schedulers
    for s_name in self._schedulers:
      scheduler = self._schedulers[s_name]
      if scheduler['type'] == 'Triggerable':
        c['schedulers'].append(Triggerable(s_name,
                                           scheduler['builders']))

    # Process the nightly schedulers
    for s_name in self._schedulers:
      scheduler = self._schedulers[s_name]
      if scheduler['type'] == 'Nightly':
        c['schedulers'].append(Nightly(s_name,
                                       branch=scheduler['branch'],
                                       hour=scheduler['hour'],
                                       builderNames=scheduler['builders']))
<|endoftext|>


<commit_before>#!/usr/bin/env
# -*- coding: utf-8 -*-
"""Migrates old-style unregistered users (dictionaries in Node#contributor_list)
to actual User records.
"""
import logging

from modularodm.exceptions import ValidationValueError
from website import app, models
from framework import auth
from framework.auth.decorators import Auth
from tests.base import DbTestCase
from tests.factories import DeprecatedUnregUserFactory, ProjectFactory, UserFactory

logging.getLogger('factory.generate:BaseFactory').setLevel(logging.WARNING)
logger = logging.getLogger(__name__)

def main():
    app.init_app()
    for node in models.Node.find():
        migrate_contributors(node)

def make_user(user_dict):
    if 'id' in user_dict:
        user = models.User.load(user_dict['id'])
    else:
        name, email = user_dict['nr_name'], user_dict['nr_email']
        try:
            user = models.User.create_unregistered(fullname=name,
                email=email)
            user.save()
        except ValidationValueError:
            user = auth.get_user(username=email)
            assert user is not None
    return user

def migrate_user(user_dict, node):
    user = make_user(user_dict)
    if user is None:
        return
    # Add unclaimed_record to unregistered users for a given node
    if not user.is_registered:
        # First contributor (usually the creator) will be recorded as the referrer
        # of unregistered users
        referrer = node.contributors[0]
        user.add_unclaimed_record(node=node, referrer=referrer,
            given_name=user_dict['nr_name'], email=user_dict['nr_email'])
        logger.info(u'Migrated unregistered user {0}'.format(user.username))
        try:
            user.save()
        except Exception as error:
            logging.error(user_dict)
            raise error
    return user

def migrate_contributors(node):
    node.contributors = [
        migrate_user(user_dict, node) for user_dict in node.contributor_list

    ]
    logger.info(u'Finished migrating unregistered '
        'contributors for node {0}'.format(node._primary_key))
    return node.save()

# To run tests: pytest scripts/migrate_unregistered.py
# or nosetests scripts/migrate_unregistered.py
class TestMigratingUnreg(DbTestCase):

    def test_make_user_from_reg_user(self):
        reg_user = UserFactory(is_registered=True)
        user = make_user({'id': reg_user._primary_key})
        assert user == reg_user

    def test_make_user_from_old_unreg_user(self):
        old_style = DeprecatedUnregUserFactory()
        user = make_user(old_style)
        user.save()
        assert isinstance(user, models.User)
        assert user.is_registered is False
        assert user.fullname == old_style['nr_name']
        assert user.username == old_style['nr_email']
        assert old_style['nr_email'] in user.emails

    def test_migrate_contributors(self):
        creator = UserFactory(is_registered=True)
        project = ProjectFactory(creator=creator)
        auth = Auth(project.creator)

        contrib1 = UserFactory()
        project.add_contributor(contrib1, auth=auth)
        project.save()
        # sanity checks
        assert len(project.contributor_list) == 2

        nr_contrib = DeprecatedUnregUserFactory()
        project.contributor_list.append(nr_contrib)
        project.save()
        assert len(project.contributors) == 2
        assert len(project.contributor_list) == 3
        old_length = len(project.contributor_list)

        migrate_contributors(project)
        project.save()

        assert len(project.contributor_list) == old_length
        assert len(project.contributors) == old_length

        migrated_user = project.contributors[-1]
        assert migrated_user.fullname == nr_contrib['nr_name']
        assert migrated_user.is_registered is False

        # has an unclaimed record
        assert project._primary_key in migrated_user.unclaimed_records
        rec = migrated_user.get_unclaimed_record(project._primary_key)
        assert rec['name'] == nr_contrib['nr_name']
        assert rec['referrer_id'] == creator._primary_key
        assert rec['token']
        assert rec['email'] == nr_contrib['nr_email']

    def test_migrate_nr_user_with_same_email_as_registered_user(self):
        reg_user = UserFactory(is_registered=True)

        unreg_user = DeprecatedUnregUserFactory(nr_email=reg_user.username)

        assert make_user(unreg_user) == reg_user

if __name__ == '__main__':
    main()
<commit_msg>Another edge case in migrate_unregistered<commit_after>#!/usr/bin/env
# -*- coding: utf-8 -*-
"""Migrates old-style unregistered users (dictionaries in Node#contributor_list)
to actual User records.
"""
import logging

from modularodm.exceptions import ValidationValueError
from website import app, models
from framework import auth
from framework.auth.decorators import Auth
from tests.base import DbTestCase
from tests.factories import DeprecatedUnregUserFactory, ProjectFactory, UserFactory

logging.getLogger('factory.generate:BaseFactory').setLevel(logging.WARNING)
logger = logging.getLogger(__name__)

def main():
    app.init_app()
    for node in models.Node.find():
        migrate_contributors(node)

def make_user(user_dict):
    if 'id' in user_dict:
        user = models.User.load(user_dict['id'])
    else:
        name, email = user_dict['nr_name'], user_dict['nr_email']
        try:
            user = models.User.create_unregistered(fullname=name,
                email=email)
            user.save()
        except ValidationValueError:
            user = auth.get_user(username=email)
            assert user is not None
    return user

def migrate_user(user_dict, node):
    user = make_user(user_dict)
    if user is None:
        return
    # Add unclaimed_record to unregistered users for a given node
    if not user.is_registered:
        # First contributor (usually the creator) will be recorded as the referrer
        # of unregistered users
        try:
            referrer = node.contributors[0]
        except IndexError:
            if node.creator:
                referrer = node.creator
            else:  # meh
                return
        user.add_unclaimed_record(node=node, referrer=referrer,
            given_name=user_dict['nr_name'], email=user_dict['nr_email'])
        logger.info(u'Migrated unregistered user {0}'.format(user.username))
        try:
            user.save()
        except Exception as error:
            logging.error(user_dict)
            raise error
    return user

def migrate_contributors(node):
    node.contributors = [
        migrate_user(user_dict, node) for user_dict in node.contributor_list

    ]
    logger.info(u'Finished migrating unregistered '
        'contributors for node {0}'.format(node._primary_key))
    return node.save()

# To run tests: pytest scripts/migrate_unregistered.py
# or nosetests scripts/migrate_unregistered.py
class TestMigratingUnreg(DbTestCase):

    def test_make_user_from_reg_user(self):
        reg_user = UserFactory(is_registered=True)
        user = make_user({'id': reg_user._primary_key})
        assert user == reg_user

    def test_make_user_from_old_unreg_user(self):
        old_style = DeprecatedUnregUserFactory()
        user = make_user(old_style)
        user.save()
        assert isinstance(user, models.User)
        assert user.is_registered is False
        assert user.fullname == old_style['nr_name']
        assert user.username == old_style['nr_email']
        assert old_style['nr_email'] in user.emails

    def test_migrate_contributors(self):
        creator = UserFactory(is_registered=True)
        project = ProjectFactory(creator=creator)
        auth = Auth(project.creator)

        contrib1 = UserFactory()
        project.add_contributor(contrib1, auth=auth)
        project.save()
        # sanity checks
        assert len(project.contributor_list) == 2

        nr_contrib = DeprecatedUnregUserFactory()
        project.contributor_list.append(nr_contrib)
        project.save()
        assert len(project.contributors) == 2
        assert len(project.contributor_list) == 3
        old_length = len(project.contributor_list)

        migrate_contributors(project)
        project.save()

        assert len(project.contributor_list) == old_length
        assert len(project.contributors) == old_length

        migrated_user = project.contributors[-1]
        assert migrated_user.fullname == nr_contrib['nr_name']
        assert migrated_user.is_registered is False

        # has an unclaimed record
        assert project._primary_key in migrated_user.unclaimed_records
        rec = migrated_user.get_unclaimed_record(project._primary_key)
        assert rec['name'] == nr_contrib['nr_name']
        assert rec['referrer_id'] == creator._primary_key
        assert rec['token']
        assert rec['email'] == nr_contrib['nr_email']

    def test_migrate_nr_user_with_same_email_as_registered_user(self):
        reg_user = UserFactory(is_registered=True)

        unreg_user = DeprecatedUnregUserFactory(nr_email=reg_user.username)

        assert make_user(unreg_user) == reg_user

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import pysam
import argparse

def main(args):

    header = []

    with open(args.table, 'r') as table:
        for i, line in enumerate(table):
            if i == 0:
                header = line.strip().split('\t')
                print line.strip()

            else:
                rec = {}

                for n, field in enumerate(line.strip().split('\t')):
                    rec[header[n]] = field

                print(line.strip())

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='foo')
    parser.add_argument('-t', '--table', required=True, help='TEBreak table')
    args = parser.parse_args()
    main(args)

<commit_msg>update<commit_after>#!/usr/bin/env python

import pysam
import argparse

def main(args):

    header = []

    with open(args.table, 'r') as table:
        for i, line in enumerate(table):
            if i == 0:
                header = line.strip().split('\t')
                print(line.strip())

            else:
                rec = {}

                for n, field in enumerate(line.strip().split('\t')):
                    rec[header[n]] = field

                print(line.strip())

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='foo')
    parser.add_argument('-t', '--table', required=True, help='TEBreak table')
    args = parser.parse_args()
    main(args)

<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import os.path
import logging
import subprocess
import fileinput
import tempfile
import shutil

import click
import github3

logging.basicConfig(level=logging.INFO)

# List of owner/repository in which to look for/change the package version.
REPOS_TO_CHANGE = (
    ('edx', 'credentials'),
    ('edx', 'course-discovery'),
    ('edx', 'ecommerce'),
    ('edx', 'programs'),
    ('edx', 'edx-analytics-data-api'),
    ('edx', 'edx-analytics-dashboard'),
    ('edx', 'xqueue'),
    ('edx', 'edx-platform'),
)

# Format to convert the repos above to an HTTPS url.
REPO_URL_FORMAT = 'https://github.com/{}/{}'

class GitHubApiUtils(object):
    """
    Class to query/set GitHub info.
    """
    def __init__(self, owner, repo_name):
        """
        Returns a GitHub object, possibly authed as a user.
        """
        token = os.environ.get('GITHUB_TOKEN', '')
        if len(token):
            self.gh = github3.login(token=token)
        else:
            self.gh = github3.GitHub()
        self.repo = self.gh.repository(owner, repo_name)

    def create_pull(self, *args, **kwargs):
        return self.repo.create_pull(*args, **kwargs)

@click.command()
@click.option("--module_name", help="Name of Python module which is being updated.", type=str, required=True)
@click.option("--new_version", help="Updated version of Python module.", type=str, required=True)
def bump_repos_version(module_name, new_version):
    """
    Changes the pinned version number in the requirements files of all repos
    which have the specified Python module as a dependency.

    This script assumes that GITHUB_TOKEN is set for GitHub authentication.
    """
    # Make the cloning directory and change directories into it.
    original_dir = os.getcwd()
    tmp_dir = tempfile.mkdtemp(dir=os.getcwd())

    # Iterate through each repository.
    for owner, repo_name in REPOS_TO_CHANGE:
        repo_url = REPO_URL_FORMAT.format(owner, repo_name)

        gh = GitHubApiUtils(owner, repo_name)

        os.chdir(tmp_dir)

        # Clone the repo.
        ret_code = subprocess.call(['git', 'clone', '{}.git'.format(repo_url)])
        if ret_code:
            logging.error('Failed to clone repo {}'.format(repo_url))
            continue

        # Change into the cloned repo dir.
        os.chdir(repo_name)

        # Create a branch, using the version number.
        branch_name = '{}/{}'.format(module_name, new_version)
        ret_code = subprocess.call(['git', 'checkout', '-b', branch_name])
        if ret_code:
            logging.error('Failed to create branch in repo {}'.format(repo_url))
            continue

        # Search through all TXT files to find all lines with the module name, changing the pinned version.
        files_changed = False
        for root, dirs, files in os.walk('.'):
            for file in files:
                if file.endswith('.txt') and (('requirements' in file) or ('requirements' in root)):
                    found = False
                    filepath = os.path.join(root, file)
                    with open(filepath) as f:
                        if '{}=='.format(module_name) in f.read():
                            found = True
                    if found:
                        files_changed = True
                        # Change the file in-place.
                        for line in fileinput.input(filepath, inplace=True):
                            if '{}=='.format(module_name) in line:
                                print '{}=={}'.format(module_name, new_version)
                            else:
                                print line,

        if not files_changed:
            # Module name wasn't found in the requirements files.
            logging.error("Module name '{}' not found in repo {} - skipping.".format(module_name, repo_url))
            continue

        # Add/commit the files.
        ret_code = subprocess.call(['git', 'commit', '-am', 'Updating {} requirement to version {}'.format(module_name, new_version)])
        if ret_code:
            logging.error("Failed to add and commit changed files to repo {}".format(repo_url))
            continue

        # Push the branch.
        ret_code = subprocess.call(['git', 'push', '--set-upstream', 'origin', branch_name])
        if ret_code:
            logging.error("Failed to push branch {} upstream for repo {}".format(branch_name, repo_url))
            continue

        # Create a PR with an automated message.
        rollback_branch_push = False
        try:
            response = gh.create_pull(
                title='Change {} version.'.format(module_name),
                body='Change the required version of {} to {}.\n\n@edx-ops/pipeline-team Please review and tag appropriate parties.'.format(module_name, new_version),
                head=branch_name,
                base='master'
            )
        except UnknownObjectException:
            logging.error('No GitHub PR-creating permissions - did you set GITHUB_TOKEN?')
            rollback_branch_push = True
        except:
            logging.error('Failed to create PR for repo {}!'.format(repo_url))
            rollback_branch_push = True
        else:
            logging.info('Created PR for repo {}: {}'.format(repo_url, response.html_url))

        if rollback_branch_push:
            ret_code = subprocess.call(['git', 'push', 'origin', '--delete', branch_name])
            if ret_code:
                logging.error("ROLLBACK: Failed to delete upstream branch {} for repo {}".format(branch_name, repo_url))

    shutil.rmtree(tmp_dir)

if __name__ == "__main__":
    bump_repos_version()
<commit_msg>More changes per review feedback.<commit_after>#!/usr/bin/env python
import os
import os.path
import logging
import subprocess
import fileinput
import tempfile
import shutil

import click
import github3

logging.basicConfig(level=logging.INFO)

# List of owner/repository in which to look for/change the package version.
REPOS_TO_CHANGE = (
    ('edx', 'credentials'),
    ('edx', 'course-discovery'),
    ('edx', 'ecommerce'),
    ('edx', 'programs'),
    ('edx', 'edx-analytics-data-api'),
    ('edx', 'edx-analytics-dashboard'),
    ('edx', 'xqueue'),
    ('edx', 'edx-platform'),
)

# Format to convert the repos above to an HTTPS url.
REPO_URL_FORMAT = 'https://github.com/{}/{}'

class GitHubApiUtils(object):
    """
    Class to query/set GitHub info.
    """
    def __init__(self, owner, repo_name):
        """
        Returns a GitHub object, possibly authed as a user.
        """
        token = os.environ.get('GITHUB_TOKEN', '')
        if len(token):
            self.gh = github3.login(token=token)
        else:
            self.gh = github3.GitHub()
        self.repo = self.gh.repository(owner, repo_name)

    def create_pull(self, *args, **kwargs):
        return self.repo.create_pull(*args, **kwargs)

@click.command()
@click.option("--module_name", help="Name of Python module which is being updated.", type=str, required=True)
@click.option("--new_version", help="Updated version of Python module.", type=str, required=True)
@click.option("--local_only", help="Modify local repo branch without pushing to -or- creating PR on remote.", is_flag=True, default=False)
def bump_repos_version(module_name, new_version, local_only):
    """
    Changes the pinned version number in the requirements files of all repos
    which have the specified Python module as a dependency.

    This script assumes that GITHUB_TOKEN is set for GitHub authentication.
    """
    # Make the cloning directory and change directories into it.
    tmp_dir = tempfile.mkdtemp(dir=os.getcwd())

    # Iterate through each repository.
    for owner, repo_name in REPOS_TO_CHANGE:
        repo_url = REPO_URL_FORMAT.format(owner, repo_name)

        gh = GitHubApiUtils(owner, repo_name)

        os.chdir(tmp_dir)

        # Clone the repo.
        ret_code = subprocess.call(['git', 'clone', '{}.git'.format(repo_url)])
        if ret_code:
            logging.error('Failed to clone repo {}'.format(repo_url))
            continue

        # Change into the cloned repo dir.
        os.chdir(repo_name)

        # Create a branch, using the version number.
        branch_name = '{}/{}'.format(module_name, new_version)
        ret_code = subprocess.call(['git', 'checkout', '-b', branch_name])
        if ret_code:
            logging.error('Failed to create branch in repo {}'.format(repo_url))
            continue

        # Search through all TXT files to find all lines with the module name, changing the pinned version.
        files_changed = False
        for root, _dirs, files in os.walk('.'):
            for file in files:
                if file.endswith('.txt') and (('requirements' in file) or ('requirements' in root)):
                    found = False
                    filepath = os.path.join(root, file)
                    with open(filepath) as f:
                        if '{}=='.format(module_name) in f.read():
                            found = True
                    if found:
                        files_changed = True
                        # Change the file in-place.
                        for line in fileinput.input(filepath, inplace=True):
                            if '{}=='.format(module_name) in line:
                                print '{}=={}'.format(module_name, new_version)
                            else:
                                print line,

        if not files_changed:
            # Module name wasn't found in the requirements files.
            logging.info("Module name '{}' not found in repo {} - skipping.".format(module_name, repo_url))
            continue

        # Add/commit the files.
        ret_code = subprocess.call(['git', 'commit', '-am', 'Updating {} requirement to version {}'.format(module_name, new_version)])
        if ret_code:
            logging.error("Failed to add and commit changed files to repo {}".format(repo_url))
            continue

        if local_only:
            # For local_only, don't push the branch to the remote and create the PR - leave all changes local for review.
            continue

        # Push the branch.
        ret_code = subprocess.call(['git', 'push', '--set-upstream', 'origin', branch_name])
        if ret_code:
            logging.error("Failed to push branch {} upstream for repo {}".format(branch_name, repo_url))
            continue

        # Create a PR with an automated message.
        rollback_branch_push = False
        try:
            # The GitHub "mention" below does not work via the API - unfortunately...
            response = gh.create_pull(
                title='Change {} version.'.format(module_name),
                body='Change the required version of {} to {}.\n\n@edx-ops/pipeline-team Please review and tag appropriate parties.'.format(module_name, new_version),
                head=branch_name,
                base='master'
            )
        except:
            logging.error('Failed to create PR for repo {} - did you set GITHUB_TOKEN?'.format(repo_url))
            rollback_branch_push = True
        else:
            logging.info('Created PR #{} for repo {}: {}'.format(response.number, repo_url, response.html_url))

        if rollback_branch_push:
            # Since the PR creation failed, delete the branch in the remote repo as well.
            ret_code = subprocess.call(['git', 'push', 'origin', '--delete', branch_name])
            if ret_code:
                logging.error("ROLLBACK: Failed to delete upstream branch {} for repo {}".format(branch_name, repo_url))

    if not local_only:
        # Remove the temp directory containing all the cloned repos.
        shutil.rmtree(tmp_dir)

if __name__ == "__main__":
    bump_repos_version()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Author: Bo Maryniuk <bo@suse.de>
#
# Copyright 2017 SUSE LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Import Salt Testing Libs
from __future__ import absolute_import, print_function, unicode_literals

import os

import salt.modules.ansiblegate as ansible
import salt.utils.platform
from salt.exceptions import LoaderError
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.mock import MagicMock, patch
from tests.support.unit import TestCase, skipIf

try:
    import pytest
except ImportError as import_error:
    pytest = None
NO_PYTEST = not bool(pytest)

@skipIf(NO_PYTEST, False)
@skipIf(salt.utils.platform.is_windows(), "Not supported on Windows")
class AnsiblegateTestCase(TestCase, LoaderModuleMockMixin):
    def setUp(self):
        self.resolver = ansible.AnsibleModuleResolver({})
        self.resolver._modules_map = {
            "one.two.three": os.sep + os.path.join("one", "two", "three.py"),
            "four.five.six": os.sep + os.path.join("four", "five", "six.py"),
            "three.six.one": os.sep + os.path.join("three", "six", "one.py"),
        }

    def tearDown(self):
        self.resolver = None

    def setup_loader_modules(self):
        return {ansible: {}}

    def test_ansible_module_help(self):
        """
        Test help extraction from the module
        :return:
        """

        class Module(object):
            """
            An ansible module mock.
            """

            __name__ = "foo"
            DOCUMENTATION = """
---
one:
   text here
---
two:
   text here
description:
   describe the second part
        """

        with patch.object(ansible, "_resolver", self.resolver), patch.object(
            ansible._resolver, "load_module", MagicMock(return_value=Module())
        ):
            ret = ansible.help("dummy")
            assert sorted(
                ret.get('Available sections on module "{0}"'.format(Module().__name__))
            ) == ["one", "two"]
            assert ret.get("Description") == "describe the second part"

    def test_module_resolver_modlist(self):
        """
        Test Ansible resolver modules list.
        :return:
        """
        assert self.resolver.get_modules_list() == [
            "four.five.six",
            "one.two.three",
            "three.six.one",
        ]
        for ptr in ["five", "fi", "ve"]:
            assert self.resolver.get_modules_list(ptr) == ["four.five.six"]
        for ptr in ["si", "ix", "six"]:
            assert self.resolver.get_modules_list(ptr) == [
                "four.five.six",
                "three.six.one",
            ]
        assert self.resolver.get_modules_list("one") == [
            "one.two.three",
            "three.six.one",
        ]
        assert self.resolver.get_modules_list("one.two") == ["one.two.three"]
        assert self.resolver.get_modules_list("four") == ["four.five.six"]

    def test_resolver_module_loader_failure(self):
        """
        Test Ansible module loader.
        :return:
        """
        mod = "four.five.six"
        with pytest.raises(ImportError) as import_error:
            self.resolver.load_module(mod)

        mod = "i.even.do.not.exist.at.all"
        with pytest.raises(LoaderError) as loader_error:
            self.resolver.load_module(mod)

    def test_resolver_module_loader(self):
        """
        Test Ansible module loader.
        :return:
        """
        with patch("salt.modules.ansiblegate.importlib", MagicMock()), patch(
            "salt.modules.ansiblegate.importlib.import_module", lambda x: x
        ):
            assert (
                self.resolver.load_module("four.five.six")
                == "ansible.modules.four.five.six"
            )

    def test_resolver_module_loader_import_failure(self):
        """
        Test Ansible module loader failure.
        :return:
        """
        with patch("salt.modules.ansiblegate.importlib", MagicMock()), patch(
            "salt.modules.ansiblegate.importlib.import_module", lambda x: x
        ):
            with pytest.raises(LoaderError) as loader_error:
                self.resolver.load_module("something.strange")

    def test_virtual_function(self):
        """
        Test Ansible module __virtual__ when ansible is not installed on the minion.
        :return:
        """
        with patch("salt.modules.ansiblegate.ansible", None):
            assert ansible.__virtual__() == "ansible"
<commit_msg>Add unit test for ansible caller<commit_after># -*- coding: utf-8 -*-
#
# Author: Bo Maryniuk <bo@suse.de>
#
# Copyright 2017 SUSE LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Import Salt Testing Libs
from __future__ import absolute_import, print_function, unicode_literals

import os

import salt.modules.ansiblegate as ansible
import salt.utils.platform
from salt.exceptions import LoaderError
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.mock import MagicMock, patch, MockTimedProc
from tests.support.unit import TestCase, skipIf

try:
    import pytest
except ImportError as import_error:
    pytest = None
NO_PYTEST = not bool(pytest)

from salt.ext import six

@skipIf(NO_PYTEST, False)
@skipIf(salt.utils.platform.is_windows(), "Not supported on Windows")
class AnsiblegateTestCase(TestCase, LoaderModuleMockMixin):
    def setUp(self):
        self.resolver = ansible.AnsibleModuleResolver({})
        self.resolver._modules_map = {
            "one.two.three": os.sep + os.path.join("one", "two", "three.py"),
            "four.five.six": os.sep + os.path.join("four", "five", "six.py"),
            "three.six.one": os.sep + os.path.join("three", "six", "one.py"),
        }

    def tearDown(self):
        self.resolver = None

    def setup_loader_modules(self):
        return {ansible: {}}

    def test_ansible_module_help(self):
        """
        Test help extraction from the module
        :return:
        """

        class Module(object):
            """
            An ansible module mock.
            """

            __name__ = "foo"
            DOCUMENTATION = """
---
one:
   text here
---
two:
   text here
description:
   describe the second part
        """

        with patch.object(ansible, "_resolver", self.resolver), patch.object(
            ansible._resolver, "load_module", MagicMock(return_value=Module())
        ):
            ret = ansible.help("dummy")
            assert sorted(
                ret.get('Available sections on module "{0}"'.format(Module().__name__))
            ) == ["one", "two"]
            assert ret.get("Description") == "describe the second part"

    def test_module_resolver_modlist(self):
        """
        Test Ansible resolver modules list.
        :return:
        """
        assert self.resolver.get_modules_list() == [
            "four.five.six",
            "one.two.three",
            "three.six.one",
        ]
        for ptr in ["five", "fi", "ve"]:
            assert self.resolver.get_modules_list(ptr) == ["four.five.six"]
        for ptr in ["si", "ix", "six"]:
            assert self.resolver.get_modules_list(ptr) == [
                "four.five.six",
                "three.six.one",
            ]
        assert self.resolver.get_modules_list("one") == [
            "one.two.three",
            "three.six.one",
        ]
        assert self.resolver.get_modules_list("one.two") == ["one.two.three"]
        assert self.resolver.get_modules_list("four") == ["four.five.six"]

    def test_resolver_module_loader_failure(self):
        """
        Test Ansible module loader.
        :return:
        """
        mod = "four.five.six"
        with pytest.raises(ImportError) as import_error:
            self.resolver.load_module(mod)

        mod = "i.even.do.not.exist.at.all"
        with pytest.raises(LoaderError) as loader_error:
            self.resolver.load_module(mod)

    def test_resolver_module_loader(self):
        """
        Test Ansible module loader.
        :return:
        """
        with patch("salt.modules.ansiblegate.importlib", MagicMock()), patch(
            "salt.modules.ansiblegate.importlib.import_module", lambda x: x
        ):
            assert (
                self.resolver.load_module("four.five.six")
                == "ansible.modules.four.five.six"
            )

    def test_resolver_module_loader_import_failure(self):
        """
        Test Ansible module loader failure.
        :return:
        """
        with patch("salt.modules.ansiblegate.importlib", MagicMock()), patch(
            "salt.modules.ansiblegate.importlib.import_module", lambda x: x
        ):
            with pytest.raises(LoaderError) as loader_error:
                self.resolver.load_module("something.strange")

    def test_virtual_function(self):
        """
        Test Ansible module __virtual__ when ansible is not installed on the minion.
        :return:
        """
        with patch("salt.modules.ansiblegate.ansible", None):
            assert ansible.__virtual__() == "ansible"

    def test_ansible_module_call(self):
        """
        Test Ansible module call from ansible gate module

        :return:
        """

        class Module(object):
            """
            An ansible module mock.
            """
            __name__ = "one.two.three"
            __file__ = "foofile"

            def main():
                pass

        ANSIBLE_MODULE_ARGS = "{'ANSIBLE_MODULE_ARGS': ['arg_1', {'kwarg1': 'foobar'}]}"

        proc = MagicMock(side_effect=[
            MockTimedProc(
                stdout=ANSIBLE_MODULE_ARGS.encode(),
                stderr=None),
            MockTimedProc(stdout="{'completed': true}".encode(), stderr=None)
        ])

        with patch.object(ansible, "_resolver", self.resolver), \
            patch.object(ansible._resolver, "load_module", MagicMock(return_value=Module())):
            _ansible_module_caller = ansible.AnsibleModuleCaller(ansible._resolver)
            with patch("salt.utils.timed_subprocess.TimedProc", proc):
                ret = _ansible_module_caller.call("one.two.three", "arg_1", kwarg1="foobar")
                if six.PY3:
                    proc.assert_any_call(["echo", "{'ANSIBLE_MODULE_ARGS': {'kwarg1': 'foobar', '_raw_params': 'arg_1'}}"], stdout=-1, timeout=1200)
                    proc.assert_any_call(["python3", "foofile"], stdin=ANSIBLE_MODULE_ARGS, stdout=-1, timeout=1200)
                else:
                    proc.assert_any_call(["echo", "{'ANSIBLE_MODULE_ARGS': {'_raw_params': 'arg_1', 'kwarg1': 'foobar'}}"], stdout=-1, timeout=1200)
                    proc.assert_any_call(["python", 'foofile'], stdin=ANSIBLE_MODULE_ARGS, stdout=-1, timeout=1200)
                assert ret == {"completed": True, "timeout": 1200}
<|endoftext|>


<commit_before># coding: utf-8

from __future__ import print_function, unicode_literals, absolute_import

from PIL import Image

from .base import rpartial
from .pillow import PillowTestCase

class RotateRightCase(PillowTestCase):
    def handle_args(self, name, transposition):
        self.name = name
        self.transposition = transposition

    def runner(self, im):
        im.transpose(self.transposition)

    def readable_args(self):
        return [self.name]

cases = [
    rpartial(RotateRightCase, 'Flop', Image.FLIP_LEFT_RIGHT),
    rpartial(RotateRightCase, 'Flip', Image.FLIP_TOP_BOTTOM),
    rpartial(RotateRightCase, 'Rotate 90', Image.ROTATE_90),
    rpartial(RotateRightCase, 'Rotate 180', Image.ROTATE_180),
    rpartial(RotateRightCase, 'Rotate 270', Image.ROTATE_270),
]

if hasattr(Image, 'TRANSPOSE'):
    cases.append(rpartial(RotateRightCase, 'Transpose', Image.TRANSPOSE))
<commit_msg>Transpose and Transpose180 for all Pillow versions<commit_after># coding: utf-8

from __future__ import print_function, unicode_literals, absolute_import

from PIL import Image

from .base import rpartial
from .pillow import PillowTestCase

class RotateRightCase(PillowTestCase):
    def handle_args(self, name, transposition):
        self.name = name
        self.transposition = transposition

    def runner(self, im):
        for trans in self.transposition:
            im = im.transpose(trans)

    def readable_args(self):
        return [self.name]

cases = [
    rpartial(RotateRightCase, 'Flop', [Image.FLIP_LEFT_RIGHT]),
    rpartial(RotateRightCase, 'Flip', [Image.FLIP_TOP_BOTTOM]),
    rpartial(RotateRightCase, 'Rotate 90', [Image.ROTATE_90]),
    rpartial(RotateRightCase, 'Rotate 180', [Image.ROTATE_180]),
    rpartial(RotateRightCase, 'Rotate 270', [Image.ROTATE_270]),
    rpartial(RotateRightCase, 'Transpose',
        [Image.TRANSPOSE]
        if hasattr(Image, 'TRANSPOSE')
        else [Image.ROTATE_90, Image.FLIP_LEFT_RIGHT]),
    rpartial(RotateRightCase, 'Transpose180',
        [Image.TRANSPOSE_ROTATE_180]
        if hasattr(Image, 'TRANSPOSE_ROTATE_180')
        else [Image.ROTATE_270, Image.FLIP_LEFT_RIGHT]),
]
<|endoftext|>


<commit_before>'''
********************************************************************************
* Name: New App Paste Template/Scaffold
* Author: Nathan Swain
* Copyright: (c) Brigham Young University 2013
* License: BSD 2-Clause
********************************************************************************
'''

import sys
import random
import re

from paste.script.templates import Template, var
from paste.util.template import paste_script_template_renderer
from paste.script.create_distro import Command

# Horrible hack to change the behaviour of Paste itself
# Since this module is only imported when commands are
# run, this will not affect any other paster commands.
Command._bad_chars_re = re.compile('[^a-zA-Z0-9_-]')

class TethysAppTemplate(Template):

    """
    Template to build a skeleton Tethys Apps package
    """

    _template_dir = 'tethysapp_template/'
    summary = 'Create a new Tethys app project using this scaffold.'
    template_renderer = staticmethod(paste_script_template_renderer)

    vars = [
        var('proper_name', 'e.g.: "My First App" for project name "my_first_app"'),
        var('version', 'e.g.: 0.0.1'),
        var('description', 'One-line description of the app'),
        var('author', 'Author name'),
        var('author_email', 'Author email'),
        var('url', 'URL of homepage'),
        var('license_name', 'License name'),
    ]

    # Default colors from flatuicolors.com
    default_colors = ('#2ecc71',    # Emerald
                      '#3498db',    # Peter River
                      '#34495e',    # Wet Asphalt
                      '#9b59b6',    # Amethyst
                      '#e67e22',    # Carrot
                      '#f1c40f',    # Sun Flower
                      '#e74c3c',    # Alizarin
                      '#1abc9c',    # Turquoise
    )

    def check_vars(self, vars, cmd):
        vars = Template.check_vars(self, vars, cmd)
        prefix = 'tethysapp-'

        if not vars['project'].startswith(prefix):
            print('\nError: Expected the project name to start with "{0}"'.format(prefix))
            sys.exit(1)

        # Validate project name
        project_error_regex = re.compile(r'^[a-zA-Z0-9_]+$')
        project_warning_regex = re.compile(r'^[a-zA-Z0-9_-]+$')
        project = vars['project'][len(prefix):]

        # Only letters, numbers and underscores allowed in app names
        if not project_error_regex.match(project):

            # If the only offending character is a dash, replace dashes with underscores and notify user
            if project_warning_regex.match(project):
                before = project
                project = project.replace('-', '_')
                print('\nWarning: Dashes in project name "{0}" have been replaced ' \
                      'with underscores "{1}"'.format(before, project))

            # Otherwise, throw error
            else:
                print('\nError: Invalid characters in project name "{0}". Only letters, numbers, and underscores ' \
                      '(no dashes) allowed after the "tethysapp-" prefix.'.format(project))
                sys.exit(1)

        vars['project'] = project

        # Derive the project_url from the project name
        vars['project_url'] = project.replace('_', '-').lower()

        # Derive proper_name if not provided by user
        if not vars['proper_name']:
            vars['proper_name'] = project.replace('_', ' ').title()

        # Derive the proper_no_spaces variable (used for the name of the App class)
        vars['proper_no_spaces'] = ''.join(vars['proper_name'].split())

        # Add the color variable to vars
        vars['color'] = random.choice(self.default_colors)

        return vars
<commit_msg>Attempted to handle the prefix issue, save for later.<commit_after>'''
********************************************************************************
* Name: New App Paste Template/Scaffold
* Author: Nathan Swain
* Copyright: (c) Brigham Young University 2013
* License: BSD 2-Clause
********************************************************************************
'''

import sys
import random
import re

from paste.script.templates import Template, var
from paste.util.template import paste_script_template_renderer
from paste.script.create_distro import Command

# Horrible hack to change the behaviour of Paste itself
# Since this module is only imported when commands are
# run, this will not affect any other paster commands.
Command._bad_chars_re = re.compile('[^a-zA-Z0-9_-]')

class TethysAppTemplate(Template):

    """
    Template to build a skeleton Tethys Apps package
    """

    _template_dir = 'tethysapp_template/'
    summary = 'Create a new Tethys app project using this scaffold.'
    template_renderer = staticmethod(paste_script_template_renderer)

    vars = [
        var('proper_name', 'e.g.: "My First App" for project name "my_first_app"'),
        var('version', 'e.g.: 0.0.1'),
        var('description', 'One-line description of the app'),
        var('author', 'Author name'),
        var('author_email', 'Author email'),
        var('url', 'URL of homepage'),
        var('license_name', 'License name'),
    ]

    # Default colors from flatuicolors.com
    default_colors = ('#2ecc71',    # Emerald
                      '#3498db',    # Peter River
                      '#34495e',    # Wet Asphalt
                      '#9b59b6',    # Amethyst
                      '#e67e22',    # Carrot
                      '#f1c40f',    # Sun Flower
                      '#e74c3c',    # Alizarin
                      '#1abc9c',    # Turquoise
    )

    def check_vars(self, vars, cmd):
        vars = Template.check_vars(self, vars, cmd)
        PREFIX = 'tethysapp-'
        PREFIX_NO_DASH = 'tethysapp'

        if not vars['project'].startswith(PREFIX):
            vars['project'] = '{0}{1}'.format(PREFIX, vars['project'])
            vars['egg'] = '{0}_{1}'.format(PREFIX_NO_DASH, vars['egg'])
            vars['package'] = '{0}{1}'.format(PREFIX_NO_DASH, vars['package'])
            print vars
            print('\nError: Expected the project name to start with "{0}". Please add the "{0}" '
                  'as a prefix and try again'.format(PREFIX))
            sys.exit(1)

        # Validate project name
        project_error_regex = re.compile(r'^[a-zA-Z0-9_]+$')
        project_warning_regex = re.compile(r'^[a-zA-Z0-9_-]+$')
        project = vars['project'][len(PREFIX):]

        # Only letters, numbers and underscores allowed in app names
        if not project_error_regex.match(project):

            # If the only offending character is a dash, replace dashes with underscores and notify user
            if project_warning_regex.match(project):
                before = project
                project = project.replace('-', '_')
                print('\nWarning: Dashes in project name "{0}" have been replaced ' \
                      'with underscores "{1}"'.format(before, project))

            # Otherwise, throw error
            else:
                print('\nError: Invalid characters in project name "{0}". Only letters, numbers, and underscores ' \
                      '(no dashes) allowed after the "tethysapp-" prefix.'.format(project))
                sys.exit(1)

        vars['project'] = project

        # Derive the project_url from the project name
        vars['project_url'] = project.replace('_', '-').lower()

        # Derive proper_name if not provided by user
        if not vars['proper_name']:
            vars['proper_name'] = project.replace('_', ' ').title()

        # Derive the proper_no_spaces variable (used for the name of the App class)
        vars['proper_no_spaces'] = ''.join(vars['proper_name'].split())

        # Add the color variable to vars
        vars['color'] = random.choice(self.default_colors)

        return vars
<|endoftext|>


<commit_before> # Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Common python commands used by various build scripts."""

import inspect
import os
import re
import signal
import subprocess
import sys
from terminal import Color

_STDOUT_IS_TTY = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()

class CommandResult(object):
  """An object to store various attributes of a child process."""

  def __init__(self):
    self.cmd = None
    self.error = None
    self.output = None
    self.returncode = None

class RunCommandError(Exception):
  """Error caught in RunCommand() method."""
  def __init__(self, msg, cmd):
    self.cmd = cmd
    Exception.__init__(self, msg)

  def __eq__(self, other):
    return (type(self) == type(other) and
            str(self) == str(other) and
            self.cmd == other.cmd)

  def __ne__(self, other):
    return not self.__eq__(other)

def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None,
               exit_code=False, redirect_stdout=False, redirect_stderr=False,
               cwd=None, input=None, enter_chroot=False, shell=False,
               env=None, extra_env=None, ignore_sigint=False,
               combine_stdout_stderr=False):
  """Runs a command.

  Args:
    cmd: cmd to run.  Should be input to subprocess.Popen. If a string, shell
      must be true. Otherwise the command must be an array of arguments, and
      shell must be false.
    print_cmd: prints the command before running it.
    error_ok: does not raise an exception on error.
    error_message: prints out this message when an error occurrs.
    exit_code: returns the return code of the shell command.
    redirect_stdout: returns the stdout.
    redirect_stderr: holds stderr output until input is communicated.
    cwd: the working directory to run this cmd.
    input: input to pipe into this command through stdin.
    enter_chroot: this command should be run from within the chroot.  If set,
      cwd must point to the scripts directory.
    shell: Controls whether we add a shell as a command interpreter.  See cmd
      since it has to agree as to the type.
    env: If non-None, this is the environment for the new process.  If
      enter_chroot is true then this is the environment of the enter_chroot,
      most of which gets removed from the cmd run.
    extra_env: If set, this is added to the environment for the new process.
      In enter_chroot=True case, these are specified on the post-entry
      side, and so are often more useful.  This dictionary is not used to
      clear any entries though.
    ignore_sigint: If True, we'll ignore signal.SIGINT before calling the
      child.  This is the desired behavior if we know our child will handle
      Ctrl-C.  If we don't do this, I think we and the child will both get
      Ctrl-C at the same time, which means we'll forcefully kill the child.
    combine_stdout_stderr: Combines stdout and stdin streams into stdout.

  Returns:
    A CommandResult object.

  Raises:
    Exception:  Raises generic exception on error with optional error_message.
  """
  # Set default for variables.
  stdout = None
  stderr = None
  stdin = None
  cmd_result = CommandResult()

  # Modify defaults based on parameters.
  if redirect_stdout: stdout = subprocess.PIPE
  if redirect_stderr: stderr = subprocess.PIPE
  if combine_stdout_stderr: stderr = subprocess.STDOUT
  # TODO(sosa): gpylint complains about redefining built-in 'input'.
  #   Can we rename this variable?
  if input: stdin = subprocess.PIPE

  if isinstance(cmd, basestring):
    if not shell:
      raise Exception('Cannot run a string command without a shell')
    cmd = ['/bin/sh', '-c', cmd]
    shell = False
  elif shell:
    raise Exception('Cannot run an array command with a shell')

  # If we are using enter_chroot we need to use enterchroot pass env through
  # to the final command.
  if enter_chroot:
    cmd = ['./enter_chroot.sh', '--'] + cmd
    if extra_env:
      for (key, value) in extra_env.items():
        cmd.insert(1, '%s=%s' % (key, value))
  elif extra_env:
    if env is not None:
      env = env.copy()
    else:
      env = os.environ.copy()

    env.update(extra_env)

  # Print out the command before running.
  if print_cmd:
    if cwd:
      Info('RunCommand: %r in %s' % (cmd, cwd))
    else:
      Info('RunCommand: %r' % cmd)
  cmd_result.cmd = cmd

  try:
    proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin, stdout=stdout,
                            stderr=stderr, shell=False, env=env)
    if ignore_sigint:
      old_sigint = signal.signal(signal.SIGINT, signal.SIG_IGN)
    try:
      (cmd_result.output, cmd_result.error) = proc.communicate(input)
    finally:
      if ignore_sigint:
        signal.signal(signal.SIGINT, old_sigint)

    if exit_code:
      cmd_result.returncode = proc.returncode

    if not error_ok and proc.returncode:
      msg = ('Command "%r" failed.\n' % cmd +
             (error_message or cmd_result.error or cmd_result.output or ''))
      raise RunCommandError(msg, cmd)
  # TODO(sosa): is it possible not to use the catch-all Exception here?
  except OSError, e:
    if not error_ok:
      raise RunCommandError(str(e), cmd)
    else:
      Warning(str(e))
  except Exception, e:
    if not error_ok:
      raise
    else:
      Warning(str(e))

  return cmd_result

#TODO(sjg): Remove this in favor of operation.Die
def Die(message):
  """Emits a red error message and halts execution.

  Args:
    message: The message to be emitted before exiting.
  """
  print >> sys.stderr, (
      Color(_STDOUT_IS_TTY).Color(Color.RED, '\nERROR: ' + message))
  sys.exit(1)

#TODO(sjg): Remove this in favor of operation.Warning
# pylint: disable-msg=W0622
def Warning(message):
  """Emits a yellow warning message and continues execution.

  Args:
    message: The message to be emitted.
  """
  print >> sys.stderr, (
      Color(_STDOUT_IS_TTY).Color(Color.YELLOW, '\nWARNING: ' + message))

# This command is deprecated in favor of operation.Info()
# It is left here for the moment so people are aware what happened.
# The reason is that this is not aware of the terminal output restrictions such
# as verbose, quiet and subprocess output. You should not be calling this.
def Info(message):
  """Emits a blue informational message and continues execution.

  Args:
    message: The message to be emitted.
  """
  print >> sys.stderr, (
      Color(_STDOUT_IS_TTY).Color(Color.BLUE, '\nINFO: ' + message))

def ListFiles(base_dir):
  """Recurively list files in a directory.

  Args:
    base_dir: directory to start recursively listing in.

  Returns:
    A list of files relative to the base_dir path or
    An empty list of there are no files in the directories.
  """
  directories = [base_dir]
  files_list = []
  while directories:
    directory = directories.pop()
    for name in os.listdir(directory):
      fullpath = os.path.join(directory, name)
      if os.path.isfile(fullpath):
        files_list.append(fullpath)
      elif os.path.isdir(fullpath):
        directories.append(fullpath)

  return files_list

def IsInsideChroot():
  """Returns True if we are inside chroot."""
  return os.path.exists('/etc/debian_chroot')

def GetSrcRoot():
  """Get absolute path to src/scripts/ directory.

  Assuming test script will always be run from descendent of src/scripts.

  Returns:
    A string, absolute path to src/scripts directory. None if not found.
  """
  src_root = None
  match_str = '/src/scripts/'
  test_script_path = os.path.abspath('.')

  path_list = re.split(match_str, test_script_path)
  if path_list:
    src_root = os.path.join(path_list[0], match_str.strip('/'))
    Info ('src_root = %r' % src_root)
  else:
    Info ('No %r found in %r' % (match_str, test_script_path))

  return src_root

def GetChromeosVersion(str_obj):
  """Helper method to parse output for CHROMEOS_VERSION_STRING.

  Args:
    str_obj: a string, which may contain Chrome OS version info.

  Returns:
    A string, value of CHROMEOS_VERSION_STRING environment variable set by
      chromeos_version.sh. Or None if not found.
  """
  if str_obj is not None:
    match = re.search('CHROMEOS_VERSION_STRING=([0-9_.]+)', str_obj)
    if match and match.group(1):
      Info ('CHROMEOS_VERSION_STRING = %s' % match.group(1))
      return match.group(1)

  Info ('CHROMEOS_VERSION_STRING NOT found')
  return None

def GetOutputImageDir(board, cros_version):
  """Construct absolute path to output image directory.

  Args:
    board: a string.
    cros_version: a string, Chrome OS version.

  Returns:
    a string: absolute path to output directory.
  """
  src_root = GetSrcRoot()
  rel_path = 'build/images/%s' % board
  # ASSUME: --build_attempt always sets to 1
  version_str = '-'.join([cros_version, 'a1'])
  output_dir = os.path.join(os.path.dirname(src_root), rel_path, version_str)
  Info ('output_dir = %s' % output_dir)
  return output_dir

def FindRepoDir(path=None):
  """Returns the nearest higher-level repo dir from the specified path.

  Args:
    path: The path to use. Defaults to cwd.
  """
  if path is None:
    path = os.getcwd()
  path = os.path.abspath(path)
  while path != '/':
    repo_dir = os.path.join(path, '.repo')
    if os.path.isdir(repo_dir):
      return repo_dir
    path = os.path.dirname(path)
  return None

def ReinterpretPathForChroot(path):
  """Returns reinterpreted path from outside the chroot for use inside.

  Args:
    path: The path to reinterpret.  Must be in src tree.
  """
  root_path = os.path.join(FindRepoDir(path), '..')

  path_abs_path = os.path.abspath(path)
  root_abs_path = os.path.abspath(root_path)

  # Strip the repository root from the path and strip first /.
  relative_path = path_abs_path.replace(root_abs_path, '')[1:]

  if relative_path == path_abs_path:
    raise Exception('Error: path is outside your src tree, cannot reinterpret.')

  new_path = os.path.join('/home', os.getenv('USER'), 'trunk', relative_path)
  return new_path

def GetCallerName():
  """Returns the name of the calling module with __main__."""
  top_frame = inspect.stack()[-1][0]
  return os.path.basename(top_frame.f_code.co_filename)

class RunCommandException(Exception):
  """Raised when there is an error in OldRunCommand."""
  def __init__(self, msg, cmd):
    self.cmd = cmd
    Exception.__init__(self, msg)

  def __eq__(self, other):
    return (type(self) == type(other) and
            str(self) == str(other) and
            self.cmd == other.cmd)

  def __ne__(self, other):
    return not self.__eq__(other)

def OldRunCommand(cmd, print_cmd=True, error_ok=False, error_message=None,
                  exit_code=False, redirect_stdout=False, redirect_stderr=False,
                  cwd=None, input=None, enter_chroot=False, num_retries=0):
  """Legacy run shell command.

  Arguments:
    cmd: cmd to run.  Should be input to subprocess.POpen.  If a string,
      converted to an array using split().
    print_cmd: prints the command before running it.
    error_ok: does not raise an exception on error.
    error_message: prints out this message when an error occurrs.
    exit_code: returns the return code of the shell command.
    redirect_stdout: returns the stdout.
    redirect_stderr: holds stderr output until input is communicated.
    cwd: the working directory to run this cmd.
    input: input to pipe into this command through stdin.
    enter_chroot: this command should be run from within the chroot.  If set,
      cwd must point to the scripts directory.
    num_retries: the number of retries to perform before dying

  Returns:
    If exit_code is True, returns the return code of the shell command.
    Else returns the output of the shell command.

  Raises:
    Exception:  Raises RunCommandException on error with optional error_message.
  """
  # Set default for variables.
  stdout = None
  stderr = None
  stdin = None
  output = ''

  # Modify defaults based on parameters.
  if redirect_stdout:  stdout = subprocess.PIPE
  if redirect_stderr:  stderr = subprocess.PIPE
  if input:  stdin = subprocess.PIPE
  if enter_chroot:  cmd = ['./enter_chroot.sh', '--'] + cmd

  # Print out the command before running.
  if print_cmd:
    Info('PROGRAM(%s) -> RunCommand: %r in dir %s' %
         (GetCallerName(), cmd, cwd))

  for retry_count in range(num_retries + 1):
    try:
      proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin,
                              stdout=stdout, stderr=stderr)
      (output, error) = proc.communicate(input)
      if exit_code and retry_count == num_retries:
        return proc.returncode

      if proc.returncode == 0:
        break

      raise RunCommandException('Command "%r" failed.\n' % (cmd) +
                                (error_message or error or output or ''),
                                cmd)
    except RunCommandException as e:
      if not error_ok and retry_count == num_retries:
        raise e
      else:
        Warning(str(e))
        if print_cmd:
          Info('PROGRAM(%s) -> RunCommand: retrying %r in dir %s' %
               (GetCallerName(), cmd, cwd))

  return output
<commit_msg>Improve tracing for failures that may be related to environment<commit_after> # Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Common python commands used by various build scripts."""

import inspect
import os
import re
import signal
import subprocess
import sys
from terminal import Color

_STDOUT_IS_TTY = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()

class CommandResult(object):
  """An object to store various attributes of a child process."""

  def __init__(self):
    self.cmd = None
    self.error = None
    self.output = None
    self.returncode = None

class RunCommandError(Exception):
  """Error caught in RunCommand() method."""
  def __init__(self, msg, cmd):
    self.cmd = cmd
    Exception.__init__(self, msg)

  def __eq__(self, other):
    return (type(self) == type(other) and
            str(self) == str(other) and
            self.cmd == other.cmd)

  def __ne__(self, other):
    return not self.__eq__(other)

def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None,
               exit_code=False, redirect_stdout=False, redirect_stderr=False,
               cwd=None, input=None, enter_chroot=False, shell=False,
               env=None, extra_env=None, ignore_sigint=False,
               combine_stdout_stderr=False):
  """Runs a command.

  Args:
    cmd: cmd to run.  Should be input to subprocess.Popen. If a string, shell
      must be true. Otherwise the command must be an array of arguments, and
      shell must be false.
    print_cmd: prints the command before running it.
    error_ok: does not raise an exception on error.
    error_message: prints out this message when an error occurrs.
    exit_code: returns the return code of the shell command.
    redirect_stdout: returns the stdout.
    redirect_stderr: holds stderr output until input is communicated.
    cwd: the working directory to run this cmd.
    input: input to pipe into this command through stdin.
    enter_chroot: this command should be run from within the chroot.  If set,
      cwd must point to the scripts directory.
    shell: Controls whether we add a shell as a command interpreter.  See cmd
      since it has to agree as to the type.
    env: If non-None, this is the environment for the new process.  If
      enter_chroot is true then this is the environment of the enter_chroot,
      most of which gets removed from the cmd run.
    extra_env: If set, this is added to the environment for the new process.
      In enter_chroot=True case, these are specified on the post-entry
      side, and so are often more useful.  This dictionary is not used to
      clear any entries though.
    ignore_sigint: If True, we'll ignore signal.SIGINT before calling the
      child.  This is the desired behavior if we know our child will handle
      Ctrl-C.  If we don't do this, I think we and the child will both get
      Ctrl-C at the same time, which means we'll forcefully kill the child.
    combine_stdout_stderr: Combines stdout and stdin streams into stdout.

  Returns:
    A CommandResult object.

  Raises:
    Exception:  Raises generic exception on error with optional error_message.
  """
  # Set default for variables.
  stdout = None
  stderr = None
  stdin = None
  cmd_result = CommandResult()

  # Modify defaults based on parameters.
  if redirect_stdout: stdout = subprocess.PIPE
  if redirect_stderr: stderr = subprocess.PIPE
  if combine_stdout_stderr: stderr = subprocess.STDOUT
  # TODO(sosa): gpylint complains about redefining built-in 'input'.
  #   Can we rename this variable?
  if input: stdin = subprocess.PIPE

  if isinstance(cmd, basestring):
    if not shell:
      raise Exception('Cannot run a string command without a shell')
    cmd = ['/bin/sh', '-c', cmd]
    shell = False
  elif shell:
    raise Exception('Cannot run an array command with a shell')

  # If we are using enter_chroot we need to use enterchroot pass env through
  # to the final command.
  if enter_chroot:
    cmd = ['./enter_chroot.sh', '--'] + cmd
    if extra_env:
      for (key, value) in extra_env.items():
        cmd.insert(1, '%s=%s' % (key, value))
  elif extra_env:
    if env is not None:
      env = env.copy()
    else:
      env = os.environ.copy()

    env.update(extra_env)

  # Print out the command before running.
  if print_cmd:
    if cwd:
      Info('RunCommand: %r in %s' % (cmd, cwd))
    else:
      Info('RunCommand: %r' % cmd)
  cmd_result.cmd = cmd

  try:
    proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin, stdout=stdout,
                            stderr=stderr, shell=False, env=env)
    if ignore_sigint:
      old_sigint = signal.signal(signal.SIGINT, signal.SIG_IGN)
    try:
      (cmd_result.output, cmd_result.error) = proc.communicate(input)
    finally:
      if ignore_sigint:
        signal.signal(signal.SIGINT, old_sigint)

    if exit_code:
      cmd_result.returncode = proc.returncode

    if not error_ok and proc.returncode:
      msg = ('Command "%r" with extra env %r\n' % (cmd, extra_env) +
             (error_message or cmd_result.error or cmd_result.output or ''))
      raise RunCommandError(msg, cmd)
  # TODO(sosa): is it possible not to use the catch-all Exception here?
  except OSError, e:
    if not error_ok:
      raise RunCommandError(str(e), cmd)
    else:
      Warning(str(e))
  except Exception, e:
    if not error_ok:
      raise
    else:
      Warning(str(e))

  return cmd_result

#TODO(sjg): Remove this in favor of operation.Die
def Die(message):
  """Emits a red error message and halts execution.

  Args:
    message: The message to be emitted before exiting.
  """
  print >> sys.stderr, (
      Color(_STDOUT_IS_TTY).Color(Color.RED, '\nERROR: ' + message))
  sys.exit(1)

#TODO(sjg): Remove this in favor of operation.Warning
# pylint: disable-msg=W0622
def Warning(message):
  """Emits a yellow warning message and continues execution.

  Args:
    message: The message to be emitted.
  """
  print >> sys.stderr, (
      Color(_STDOUT_IS_TTY).Color(Color.YELLOW, '\nWARNING: ' + message))

# This command is deprecated in favor of operation.Info()
# It is left here for the moment so people are aware what happened.
# The reason is that this is not aware of the terminal output restrictions such
# as verbose, quiet and subprocess output. You should not be calling this.
def Info(message):
  """Emits a blue informational message and continues execution.

  Args:
    message: The message to be emitted.
  """
  print >> sys.stderr, (
      Color(_STDOUT_IS_TTY).Color(Color.BLUE, '\nINFO: ' + message))

def ListFiles(base_dir):
  """Recurively list files in a directory.

  Args:
    base_dir: directory to start recursively listing in.

  Returns:
    A list of files relative to the base_dir path or
    An empty list of there are no files in the directories.
  """
  directories = [base_dir]
  files_list = []
  while directories:
    directory = directories.pop()
    for name in os.listdir(directory):
      fullpath = os.path.join(directory, name)
      if os.path.isfile(fullpath):
        files_list.append(fullpath)
      elif os.path.isdir(fullpath):
        directories.append(fullpath)

  return files_list

def IsInsideChroot():
  """Returns True if we are inside chroot."""
  return os.path.exists('/etc/debian_chroot')

def GetSrcRoot():
  """Get absolute path to src/scripts/ directory.

  Assuming test script will always be run from descendent of src/scripts.

  Returns:
    A string, absolute path to src/scripts directory. None if not found.
  """
  src_root = None
  match_str = '/src/scripts/'
  test_script_path = os.path.abspath('.')

  path_list = re.split(match_str, test_script_path)
  if path_list:
    src_root = os.path.join(path_list[0], match_str.strip('/'))
    Info ('src_root = %r' % src_root)
  else:
    Info ('No %r found in %r' % (match_str, test_script_path))

  return src_root

def GetChromeosVersion(str_obj):
  """Helper method to parse output for CHROMEOS_VERSION_STRING.

  Args:
    str_obj: a string, which may contain Chrome OS version info.

  Returns:
    A string, value of CHROMEOS_VERSION_STRING environment variable set by
      chromeos_version.sh. Or None if not found.
  """
  if str_obj is not None:
    match = re.search('CHROMEOS_VERSION_STRING=([0-9_.]+)', str_obj)
    if match and match.group(1):
      Info ('CHROMEOS_VERSION_STRING = %s' % match.group(1))
      return match.group(1)

  Info ('CHROMEOS_VERSION_STRING NOT found')
  return None

def GetOutputImageDir(board, cros_version):
  """Construct absolute path to output image directory.

  Args:
    board: a string.
    cros_version: a string, Chrome OS version.

  Returns:
    a string: absolute path to output directory.
  """
  src_root = GetSrcRoot()
  rel_path = 'build/images/%s' % board
  # ASSUME: --build_attempt always sets to 1
  version_str = '-'.join([cros_version, 'a1'])
  output_dir = os.path.join(os.path.dirname(src_root), rel_path, version_str)
  Info ('output_dir = %s' % output_dir)
  return output_dir

def FindRepoDir(path=None):
  """Returns the nearest higher-level repo dir from the specified path.

  Args:
    path: The path to use. Defaults to cwd.
  """
  if path is None:
    path = os.getcwd()
  path = os.path.abspath(path)
  while path != '/':
    repo_dir = os.path.join(path, '.repo')
    if os.path.isdir(repo_dir):
      return repo_dir
    path = os.path.dirname(path)
  return None

def ReinterpretPathForChroot(path):
  """Returns reinterpreted path from outside the chroot for use inside.

  Args:
    path: The path to reinterpret.  Must be in src tree.
  """
  root_path = os.path.join(FindRepoDir(path), '..')

  path_abs_path = os.path.abspath(path)
  root_abs_path = os.path.abspath(root_path)

  # Strip the repository root from the path and strip first /.
  relative_path = path_abs_path.replace(root_abs_path, '')[1:]

  if relative_path == path_abs_path:
    raise Exception('Error: path is outside your src tree, cannot reinterpret.')

  new_path = os.path.join('/home', os.getenv('USER'), 'trunk', relative_path)
  return new_path

def GetCallerName():
  """Returns the name of the calling module with __main__."""
  top_frame = inspect.stack()[-1][0]
  return os.path.basename(top_frame.f_code.co_filename)

class RunCommandException(Exception):
  """Raised when there is an error in OldRunCommand."""
  def __init__(self, msg, cmd):
    self.cmd = cmd
    Exception.__init__(self, msg)

  def __eq__(self, other):
    return (type(self) == type(other) and
            str(self) == str(other) and
            self.cmd == other.cmd)

  def __ne__(self, other):
    return not self.__eq__(other)

def OldRunCommand(cmd, print_cmd=True, error_ok=False, error_message=None,
                  exit_code=False, redirect_stdout=False, redirect_stderr=False,
                  cwd=None, input=None, enter_chroot=False, num_retries=0):
  """Legacy run shell command.

  Arguments:
    cmd: cmd to run.  Should be input to subprocess.POpen.  If a string,
      converted to an array using split().
    print_cmd: prints the command before running it.
    error_ok: does not raise an exception on error.
    error_message: prints out this message when an error occurrs.
    exit_code: returns the return code of the shell command.
    redirect_stdout: returns the stdout.
    redirect_stderr: holds stderr output until input is communicated.
    cwd: the working directory to run this cmd.
    input: input to pipe into this command through stdin.
    enter_chroot: this command should be run from within the chroot.  If set,
      cwd must point to the scripts directory.
    num_retries: the number of retries to perform before dying

  Returns:
    If exit_code is True, returns the return code of the shell command.
    Else returns the output of the shell command.

  Raises:
    Exception:  Raises RunCommandException on error with optional error_message.
  """
  # Set default for variables.
  stdout = None
  stderr = None
  stdin = None
  output = ''

  # Modify defaults based on parameters.
  if redirect_stdout:  stdout = subprocess.PIPE
  if redirect_stderr:  stderr = subprocess.PIPE
  if input:  stdin = subprocess.PIPE
  if enter_chroot:  cmd = ['./enter_chroot.sh', '--'] + cmd

  # Print out the command before running.
  if print_cmd:
    Info('PROGRAM(%s) -> RunCommand: %r in dir %s' %
         (GetCallerName(), cmd, cwd))

  for retry_count in range(num_retries + 1):
    try:
      proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin,
                              stdout=stdout, stderr=stderr)
      (output, error) = proc.communicate(input)
      if exit_code and retry_count == num_retries:
        return proc.returncode

      if proc.returncode == 0:
        break

      raise RunCommandException('Command "%r" failed.\n' % (cmd) +
                                (error_message or error or output or ''),
                                cmd)
    except RunCommandException as e:
      if not error_ok and retry_count == num_retries:
        raise e
      else:
        Warning(str(e))
        if print_cmd:
          Info('PROGRAM(%s) -> RunCommand: retrying %r in dir %s' %
               (GetCallerName(), cmd, cwd))

  return output
<|endoftext|>