<commit_before>from bs4 import BeautifulSoup
from urllib.parse import urlparse, urlunparse, urljoin
from os.path import join, abspath, dirname, relpath, isdir, isfile
from re import compile
import unittest

class HTTP404 (RuntimeError): pass

class SoupServer:
    ''' Stupidest possible directory server thing.
    '''
    def __init__(self, base):
        self.base = abspath(base)
        self.url = '/'

    def go(self, path):
        path = urljoin(self.url, path)
        local = join(self.base, relpath(path, '/'))

        if isdir(local):
            local = join(local, 'index.html')

        if not isfile(local):
            raise HTTP404(path)

        with open(local) as file:
            soup = BeautifulSoup(file, "html.parser")

        path = relpath(local, self.base)
        self.url = '/' + path
        return soup

class Tests (unittest.TestCase):

    def setUp(self):
        self.server = SoupServer(join(dirname(__file__), 'dist'))

    def test_search_index(self):
        '''
        '''
        page1 = self.server.go('/search')
        url1 = self.server.url

        head1 = page1.find_all('h1')[-1]
        self.assertEqual(head1.text, 'Mapzen Search', 'Should be looking at Mapzen Search page')

        link1 = page1.find('a', class_=compile(r'\bpagination-link\b'))
        self.assertIn('Get started', link1.text, 'First pagination link should go to Get Started')

        page2 = self.server.go(link1['href'])

        head2 = page2.find_all('h1')[-1]
        self.assertIn('Get started', head2.text, 'We should be on a page called Get Started')

        crumbs2 = page2.find('ol', class_=compile(r'\bbreadcrumb\b')).find_all('li')
        self.assertEqual(len(crumbs2), 3, 'There should be three breadcrumbs at top')
        self.assertIsNotNone(crumbs2[0].find('a'), 'The first breadcrumb is a link')
        self.assertIsNotNone(crumbs2[1].find('a'), 'The second breadcrumb is a link')
        self.assertIsNone(crumbs2[2].find('a'), 'The last breadcrumb is not a link')

        edit2 = page2.find(string=compile(r'Edit this page on GitHub')).find_parent('a')

        self.assertEqual(urlparse(edit2['href']).hostname, 'github.com', 'Should link to Github.com')
        self.assertTrue(urlparse(edit2['href']).path.endswith('.md'), 'Should link to a Markdown file')

        link2 = page2.find('a', class_=compile(r'\bpagination-link\b'))
        self.server.go(link2['href'])

        self.assertEqual(self.server.url, url1, 'We should be back where we started')

if __name__ == '__main__':
    unittest.main()<commit_msg>Expanded basic checklist script to all seven top documentation sections<commit_after>from bs4 import BeautifulSoup
from urllib.parse import urlparse, urlunparse, urljoin
from os.path import join, abspath, dirname, relpath, isdir, isfile
from re import compile
import unittest

class HTTP404 (RuntimeError): pass

class SoupServer:
    ''' Stupidest possible directory server thing.
    '''
    def __init__(self, base):
        self.base = abspath(base)
        self.url = '/'

    def go(self, path):
        path = urljoin(self.url, path)
        local = join(self.base, relpath(path, '/'))

        if isdir(local):
            local = join(local, 'index.html')

        if not isfile(local):
            raise HTTP404(path)

        with open(local) as file:
            soup = BeautifulSoup(file, "html.parser")

        path = relpath(local, self.base)
        self.url = '/' + path
        return soup

class Tests (unittest.TestCase):

    def setUp(self):
        self.server = SoupServer(join(dirname(__file__), 'dist'))

    def _test_doc_section(self, start_path, start_title, next_title):
        '''
        '''
        page1 = self.server.go(start_path)
        url1 = self.server.url

        head1 = page1.find(class_=compile(r'\bdocumentation-hero\b')).find('h1')
        self.assertEqual(head1.text, start_title, 'Should be looking at {} page'.format(start_title))

        crumbs1 = page1.find('ol', class_=compile(r'\bbreadcrumb\b')).find_all('li')
        self.assertEqual(len(crumbs1), 2, 'There should be two breadcrumbs at top')
        self.assertIsNotNone(crumbs1[0].find('a'), 'The first breadcrumb is a link')
        self.assertIsNone(crumbs1[1].find('a'), 'The last breadcrumb is not a link')
        self.assertEqual(crumbs1[1].text, start_title, 'The last breadcrumb should be {}'.format(start_title))

        link1 = page1.find('a', class_=compile(r'\bpagination-link\b'))

        if next_title is None:
            self.assertIsNone(link1, 'Should be no pagination link here')
            return

        self.assertIn(next_title, link1.text, 'First pagination link should go to {}'.format(next_title))

        page2 = self.server.go(link1['href'])

        head2 = page1.find(class_=compile(r'\bdocumentation-hero\b')).find('h1')
        self.assertIn(start_title, head2.text, 'We should be on a page called {}'.format(start_title))

        crumbs2 = page2.find('ol', class_=compile(r'\bbreadcrumb\b')).find_all('li')
        self.assertEqual(len(crumbs2), 3, 'There should be three breadcrumbs at top')
        self.assertIsNotNone(crumbs2[0].find('a'), 'The first breadcrumb is a link')
        self.assertIsNotNone(crumbs2[1].find('a'), 'The second breadcrumb is a link')
        self.assertIsNone(crumbs2[2].find('a'), 'The last breadcrumb is not a link')
        self.assertEqual(crumbs2[2].text, next_title, 'The last breadcrumb should be {}'.format(next_title))

        edit2 = page2.find(string=compile(r'Edit this page on GitHub')).find_parent('a')

        self.assertEqual(urlparse(edit2['href']).hostname, 'github.com', 'Should link to Github.com')
        self.assertTrue(urlparse(edit2['href']).path.endswith('.md'), 'Should link to a Markdown file')

        link2 = page2.find('a', class_=compile(r'\bpagination-link\b'))
        self.server.go(link2['href'])

        self.assertEqual(self.server.url, url1, 'We should be back where we started')

    def test_tangram_index(self):
        self._test_doc_section('/tangram', 'Tangram', 'Walkthrough')

    def test_search_index(self):
        self._test_doc_section('/search', 'Mapzen Search', 'Get started')

    def test_turnbyturn_index(self):
        self._test_doc_section('/turn-by-turn', 'Mapzen Turn-by-Turn', 'API reference')

    def test_vectortiles_index(self):
        self._test_doc_section('/vector-tiles', 'Vector Tile Service', 'Get started')

    def test_metroextracts_index(self):
        self._test_doc_section('/metro-extracts', 'Metro Extracts', None)

    def test_elevation_index(self):
        self._test_doc_section('/elevation', 'Elevation Service', None)

    def test_matrix_index(self):
        self._test_doc_section('/matrix', 'Time-Distance Matrix', 'API reference')

if __name__ == '__main__':
    unittest.main()<|endoftext|>


<commit_before>from __future__ import division, print_function

import os
import glob
import numpy as np
from math import ceil
from scipy.ndimage.interpolation import map_coordinates
from scipy.ndimage.filters import gaussian_filter

from keras import utils
from keras.preprocessing import image as keras_image
from keras.preprocessing.image import ImageDataGenerator

from . import patient

def load_images(data_dir, mask='both'):
    assert mask in ['inner', 'outer', 'both']

    glob_search = os.path.join(data_dir, "patient*")
    patient_dirs = sorted(glob.glob(glob_search))
    if len(patient_dirs) == 0:
        raise Exception("No patient directors found in {}".format(data_dir))

    # load all images into memory (dataset is small)
    images = []
    inner_masks = []
    outer_masks = []
    for patient_dir in patient_dirs:
        p = patient.PatientData(patient_dir)
        images += p.images
        inner_masks += p.endocardium_masks
        outer_masks += p.epicardium_masks

    # reshape to account for channel dimension
    images = np.asarray(images)[:,:,:,None]
    if mask == 'inner':
        masks = np.asarray(inner_masks)
    elif mask == 'outer':
        masks = np.asarray(outer_masks)
    elif mask == 'both':
        # mask = 2 for endocardium, 1 for cardiac wall, 0 elsewhere
        masks = np.asarray(inner_masks) + np.asarray(outer_masks)

    # one-hot encode masks
    dims = masks.shape
    classes = len(set(masks[0].flatten())) # get num classes from first image
    masks = utils.to_categorical(masks).reshape(*dims, classes)

    return images, masks

def random_elastic_deformation(image, alpha, sigma, mode='nearest',
                               random_state=None):
    """Elastic deformation of images as described in [Simard2003]_.
    .. [Simard2003] Simard, Steinkraus and Platt, "Best Practices for
       Convolutional Neural Networks applied to Visual Document Analysis", in
       Proc. of the International Conference on Document Analysis and
       Recognition, 2003.
    """
    assert len(image.shape) == 3

    if random_state is None:
        random_state = np.random.RandomState(None)

    height, width, channels = image.shape

    dx = gaussian_filter(2*random_state.rand(height, width) - 1,
                         sigma, mode="constant", cval=0) * alpha
    dy = gaussian_filter(2*random_state.rand(height, width) - 1,
                         sigma, mode="constant", cval=0) * alpha

    x, y = np.meshgrid(np.arange(height), np.arange(width), indexing='ij')
    indices = (np.repeat(np.ravel(x+dx), channels),
               np.repeat(np.ravel(y+dy), channels),
               np.tile(np.arange(channels), height*width))

    values = map_coordinates(image, indices, order=1, mode=mode)

    return values.reshape((height, width, channels))

class Iterator(object):
    def __init__(self, images, masks, batch_size,
                 rotation_range=180,
                 width_shift_range=0.1,
                 height_shift_range=0.1,
                 shear_range=0.1,
                 zoom_range=0.01,
                 fill_mode='nearest',
                 alpha=500,
                 sigma=20):
        self.images = images
        self.masks = masks
        self.batch_size = batch_size
        augment_options = {
            'rotation_range': rotation_range,
            'width_shift_range': width_shift_range,
            'height_shift_range': height_shift_range,
            'shear_range': shear_range,
            'zoom_range': zoom_range,
            'fill_mode': fill_mode,
        }
        self.idg = ImageDataGenerator(**augment_options)
        self.alpha = alpha
        self.sigma = sigma
        self.fill_mode = fill_mode
        self.i = 0

    def __next__(self):
        return self.next()

    def next(self):
        start = self.i
        end = min(start + self.batch_size, len(self.images))
        self.i += self.batch_size
        if self.i >= len(self.images):
            self.i = 0
        augmented_images = []
        augmented_masks = []
        images_masks = zip(self.images[start:end], self.masks[start:end])
        for image, mask in images_masks:
            _, _, channels = image.shape
            stacked = np.concatenate((image, mask), axis=2)
            augmented = self.idg.random_transform(stacked)
            if self.alpha != 0 and self.sigma != 0:
                augmented = random_elastic_deformation(
                    augmented, self.alpha, self.sigma, self.fill_mode)
            augmented_image = augmented[:,:,:channels]
            augmented_images.append(augmented_image)
            augmented_masks.append(np.round(augmented[:,:,channels:]))
        return np.asarray(augmented_images), np.asarray(augmented_masks)

def normalize(x, epsilon=1e-7, axis=(1,2)):
    x -= np.mean(x, axis=axis, keepdims=True)
    x /= np.std(x, axis=axis, keepdims=True) + epsilon

def create_generators(data_dir, batch_size, validation_split=0.0, mask='both',
                      shuffle=False, seed=None, normalize_images=True,
                      augment_training=False, augment_validation=False,
                      augmentation_args={}):
    images, masks = load_images(data_dir, mask)

    # split out last %(validation_split) of images as validation set
    split_index = int((1-validation_split) * len(images))

    # maybe normalize image
    if normalize_images:
        normalize(images, axis=(1,2))

    if seed is not None:
        np.random.seed(seed)

    if shuffle:
        np.random.shuffle(images)

    if augment_training:
        train_generator = Iterator(
            images[:split_index], masks[:split_index],
            batch_size, **augmentation_args)
    else:
        idg = ImageDataGenerator()
        train_generator = idg.flow(images[:split_index], masks[:split_index],
                                   batch_size=batch_size)

    train_steps_per_epoch = ceil(split_index / batch_size)

    if validation_split > 0.0:
        if augment_validation:
            val_generator = Iterator(
                images[split_index:], masks[split_index:],
                batch_size, **augmentation_args)
        else:
            idg = ImageDataGenerator()
            val_generator = idg.flow(images[split_index:],
                                     masks[split_index:],
                                     batch_size=batch_size)
    else:
        val_generator = None

    val_steps_per_epoch = ceil((len(images) - split_index) / batch_size)

    return (train_generator, train_steps_per_epoch,
            val_generator, val_steps_per_epoch)
<commit_msg>Fix type casting.<commit_after>from __future__ import division, print_function

import os
import glob
import numpy as np
from math import ceil
from scipy.ndimage.interpolation import map_coordinates
from scipy.ndimage.filters import gaussian_filter

from keras import utils
from keras.preprocessing import image as keras_image
from keras.preprocessing.image import ImageDataGenerator

from . import patient

def load_images(data_dir, mask='both'):
    assert mask in ['inner', 'outer', 'both']

    glob_search = os.path.join(data_dir, "patient*")
    patient_dirs = sorted(glob.glob(glob_search))
    if len(patient_dirs) == 0:
        raise Exception("No patient directors found in {}".format(data_dir))

    # load all images into memory (dataset is small)
    images = []
    inner_masks = []
    outer_masks = []
    for patient_dir in patient_dirs:
        p = patient.PatientData(patient_dir)
        images += p.images
        inner_masks += p.endocardium_masks
        outer_masks += p.epicardium_masks

    # reshape to account for channel dimension
    images = np.asarray(images)[:,:,:,None]
    if mask == 'inner':
        masks = np.asarray(inner_masks)
    elif mask == 'outer':
        masks = np.asarray(outer_masks)
    elif mask == 'both':
        # mask = 2 for endocardium, 1 for cardiac wall, 0 elsewhere
        masks = np.asarray(inner_masks) + np.asarray(outer_masks)

    # one-hot encode masks
    dims = masks.shape
    classes = len(set(masks[0].flatten())) # get num classes from first image
    masks = utils.to_categorical(masks).reshape(*dims, classes)

    return images, masks

def random_elastic_deformation(image, alpha, sigma, mode='nearest',
                               random_state=None):
    """Elastic deformation of images as described in [Simard2003]_.
    .. [Simard2003] Simard, Steinkraus and Platt, "Best Practices for
       Convolutional Neural Networks applied to Visual Document Analysis", in
       Proc. of the International Conference on Document Analysis and
       Recognition, 2003.
    """
    assert len(image.shape) == 3

    if random_state is None:
        random_state = np.random.RandomState(None)

    height, width, channels = image.shape

    dx = gaussian_filter(2*random_state.rand(height, width) - 1,
                         sigma, mode="constant", cval=0) * alpha
    dy = gaussian_filter(2*random_state.rand(height, width) - 1,
                         sigma, mode="constant", cval=0) * alpha

    x, y = np.meshgrid(np.arange(height), np.arange(width), indexing='ij')
    indices = (np.repeat(np.ravel(x+dx), channels),
               np.repeat(np.ravel(y+dy), channels),
               np.tile(np.arange(channels), height*width))

    values = map_coordinates(image, indices, order=1, mode=mode)

    return values.reshape((height, width, channels))

class Iterator(object):
    def __init__(self, images, masks, batch_size,
                 rotation_range=180,
                 width_shift_range=0.1,
                 height_shift_range=0.1,
                 shear_range=0.1,
                 zoom_range=0.01,
                 fill_mode='nearest',
                 alpha=500,
                 sigma=20):
        self.images = images
        self.masks = masks
        self.batch_size = batch_size
        augment_options = {
            'rotation_range': rotation_range,
            'width_shift_range': width_shift_range,
            'height_shift_range': height_shift_range,
            'shear_range': shear_range,
            'zoom_range': zoom_range,
            'fill_mode': fill_mode,
        }
        self.idg = ImageDataGenerator(**augment_options)
        self.alpha = alpha
        self.sigma = sigma
        self.fill_mode = fill_mode
        self.i = 0

    def __next__(self):
        return self.next()

    def next(self):
        start = self.i
        end = min(start + self.batch_size, len(self.images))
        self.i += self.batch_size
        if self.i >= len(self.images):
            self.i = 0
        augmented_images = []
        augmented_masks = []
        images_masks = zip(self.images[start:end], self.masks[start:end])
        for image, mask in images_masks:
            _, _, channels = image.shape
            stacked = np.concatenate((image, mask), axis=2)
            augmented = self.idg.random_transform(stacked)
            if self.alpha != 0 and self.sigma != 0:
                augmented = random_elastic_deformation(
                    augmented, self.alpha, self.sigma, self.fill_mode)
            augmented_image = augmented[:,:,:channels]
            augmented_images.append(augmented_image)
            augmented_masks.append(np.round(augmented[:,:,channels:]))
        return np.asarray(augmented_images), np.asarray(augmented_masks)

def normalize(x, epsilon=1e-7, axis=(1,2)):
    x -= np.mean(x, axis=axis, keepdims=True)
    x /= np.std(x, axis=axis, keepdims=True) + epsilon

def create_generators(data_dir, batch_size, validation_split=0.0, mask='both',
                      shuffle=False, seed=None, normalize_images=True,
                      augment_training=False, augment_validation=False,
                      augmentation_args={}):
    images, masks = load_images(data_dir, mask)

    # before: type(masks) = uint8 and type(images) = uint16
    # convert images to double-precision
    images = images.astype('float64')

    # split out last %(validation_split) of images as validation set
    split_index = int((1-validation_split) * len(images))

    # maybe normalize image
    if normalize_images:
        normalize(images, axis=(1,2))

    if seed is not None:
        np.random.seed(seed)

    if shuffle:
        np.random.shuffle(images)

    if augment_training:
        train_generator = Iterator(
            images[:split_index], masks[:split_index],
            batch_size, **augmentation_args)
    else:
        idg = ImageDataGenerator()
        train_generator = idg.flow(images[:split_index], masks[:split_index],
                                   batch_size=batch_size)

    train_steps_per_epoch = ceil(split_index / batch_size)

    if validation_split > 0.0:
        if augment_validation:
            val_generator = Iterator(
                images[split_index:], masks[split_index:],
                batch_size, **augmentation_args)
        else:
            idg = ImageDataGenerator()
            val_generator = idg.flow(images[split_index:],
                                     masks[split_index:],
                                     batch_size=batch_size)
    else:
        val_generator = None

    val_steps_per_epoch = ceil((len(images) - split_index) / batch_size)

    return (train_generator, train_steps_per_epoch,
            val_generator, val_steps_per_epoch)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

""" Recursively search and download album covers for a music library. """

import argparse
import collections
import concurrent.futures
import itertools
import logging
import os
import shutil
import sys
import time

import mutagen

import sacad

AUDIO_EXTENSIONS = ("aac",
                    "ape",
                    "flac",
                    "m4a",
                    "mp3",
                    "mp4",
                    "ogg",
                    "oga",
                    "opus",
                    "wv")

def analyze_lib(lib_dir, cover_filename):
  """ Recursively analyze library, and return a dict of path -> (artist, album). """
  work = {}
  scrobbler = itertools.cycle("|/-\\")
  stats = collections.OrderedDict(((k, 0) for k in("dirs", "files", "albums", "missing covers", "errors")))
  time_progress_shown = show_analyze_progress(stats, scrobbler)
  for rootpath, rel_dirpaths, rel_filepaths in os.walk(lib_dir):
    stats["dirs"] += 1
    metadata, time_progress_shown = analyze_dir(stats, rootpath, rel_filepaths, cover_filename, scrobbler, time_progress_shown)
    if all(metadata):
      work[rootpath] = metadata
  show_analyze_progress(stats, scrobbler, end=True)
  return work

def get_metadata(audio_filepaths):
  """ Return a tuple of album, artist from a list of audio files. """
  artist, album = None, None
  for audio_filepath in audio_filepaths:
    mf = mutagen.File(audio_filepath)
    if mf is None:
      continue
    artist = mf.get("albumartist",
                    mf.get("artist", None))
    if artist is not None:
      artist = artist[0]
    album = mf.get("album", None)
    if album is not None:
      album = album[0]
    if artist and album:
      # stop at the first file that succeeds (for performance)
      break
  return artist, album

def analyze_dir(stats, parent_dir, rel_filepaths, cover_filename, scrobbler, time_progress_shown):
  """ Analyze a directory (non recursively) to get its album metadata if it is one. """
  metadata = None, None
  audio_filepaths = []
  for rel_filepath in rel_filepaths:
    stats["files"] += 1
    time_progress_shown = show_analyze_progress(stats, scrobbler, time_progress_shown=time_progress_shown)
    try:
      ext = os.path.splitext(rel_filepath)[1][1:].lower()
    except IndexError:
      continue
    if ext in AUDIO_EXTENSIONS:
      audio_filepaths.append(os.path.join(parent_dir, rel_filepath))
  if audio_filepaths:
    stats["albums"] += 1
    if not os.path.isfile(os.path.join(parent_dir, cover_filename)):
      stats["missing covers"] += 1
      metadata = get_metadata(audio_filepaths)
      if not all(metadata):
        # failed to get metadata for this album
        stats["errors"] += 1
  return metadata, time_progress_shown

def show_analyze_progress(stats, scrobbler, *, time_progress_shown=0, end=False):
  """ Display analysis global progress. """
  now = time.time()
  if (sys.stdout.isatty() and
     (end or (now - time_progress_shown > 0.1))):  # do not refresh display at each call (for performance)
    time_progress_shown = now
    print("Analyzing library %s | %s" % (next(scrobbler) if not end else "-",
                                         "  ".join(("%u %s" % (v, k)) for k, v in stats.items())),
          end="\r" if not end else "\n")
  return time_progress_shown

def get_covers(work, args):
  """ Get missing covers. """
  with concurrent.futures.ProcessPoolExecutor() as executor:
    # post work
    futures = {}
    for path, (artist, album) in work.items():
      future = executor.submit(sacad.search_and_download,
                               album,
                               artist,
                               args.format,
                               args.size,
                               args.size_tolerance_prct,
                               args.amazon_tlds,
                               args.no_lq_sources,
                               os.path.join(path, args.filename))
      futures[future] = (artist, album)

    # follow progress
    stats = collections.OrderedDict(((k, 0) for k in("ok", "no result found")))
    for i, future in enumerate(concurrent.futures.as_completed(futures)):
      artist, album = futures[future]
      status = future.result()
      show_get_covers_progress(i,
                               len(work),
                               stats,
                               artist=artist,
                               album=album)
      if status:
        stats["ok"] += 1
      else:
        stats["no result found"] += 1
  if work:
    show_get_covers_progress(i + 1, len(work), stats, end=True)

def show_get_covers_progress(current_idx, total_count, stats, *, artist=None, album=None, end=False):
  """ Display search and download global progress. """
  if not sys.stdout.isatty():
    return
  line_width = shutil.get_terminal_size(fallback=(80, 0))[0] - 1
  print(" " * line_width, end="\r")
  print("Searching and downloading covers %u%% (%u/%u)%s | %s" % (100 * current_idx // total_count,
                                                                  current_idx,
                                                                  total_count,
                                                                  (" | Current album: '%s' '%s'" % (artist, album)) if not end else "",
                                                                  "  ".join(("%u %s" % (v, k)) for k, v in stats.items())),
        end="\r" if not end else "\n")

def cl_main():
  # parse args
  arg_parser = argparse.ArgumentParser(description="SACAD (recursive tool) v%s.%s" % (sacad.__version__,
                                                                                __doc__),
                                       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  arg_parser.add_argument("lib_dir",
                          help="Music library directory to recursively analyze")
  arg_parser.add_argument("size",
                          type=int,
                          help="Target image size")
  arg_parser.add_argument("filename",
                          help="Cover image filename")
  sacad.setup_common_args(arg_parser)
  args = arg_parser.parse_args()
  args.format = os.path.splitext(args.filename)[1][1:].lower()
  try:
    args.format = sacad.SUPPORTED_IMG_FORMATS[args.format]
  except KeyError:
    print("Unable to guess image format from extension, or unknown format: %s" % (args.format))
    exit(1)

  # silence the logger
  logging.getLogger().setLevel(logging.ERROR)

  # do the job
  work = analyze_lib(args.lib_dir, args.filename)
  get_covers(work, args)
<commit_msg>Improve reporting<commit_after>#!/usr/bin/env python3

""" Recursively search and download album covers for a music library. """

import argparse
import collections
import concurrent.futures
import itertools
import logging
import os
import shutil
import sys
import time

import mutagen

import sacad

AUDIO_EXTENSIONS = ("aac",
                    "ape",
                    "flac",
                    "m4a",
                    "mp3",
                    "mp4",
                    "ogg",
                    "oga",
                    "opus",
                    "wv")

def analyze_lib(lib_dir, cover_filename):
  """ Recursively analyze library, and return a dict of path -> (artist, album). """
  work = {}
  scrobbler = itertools.cycle("|/-\\")
  stats = collections.OrderedDict(((k, 0) for k in("dirs", "files", "albums", "missing covers", "errors")))
  time_progress_shown = show_analyze_progress(stats, scrobbler)
  for rootpath, rel_dirpaths, rel_filepaths in os.walk(lib_dir):
    stats["dirs"] += 1
    metadata, time_progress_shown = analyze_dir(stats, rootpath, rel_filepaths, cover_filename, scrobbler, time_progress_shown)
    if all(metadata):
      work[rootpath] = metadata
  show_analyze_progress(stats, scrobbler, end=True)
  return work

def get_metadata(audio_filepaths):
  """ Return a tuple of album, artist from a list of audio files. """
  artist, album = None, None
  for audio_filepath in audio_filepaths:
    mf = mutagen.File(audio_filepath)
    if mf is None:
      continue
    artist = mf.get("albumartist",
                    mf.get("artist", None))
    if artist is not None:
      artist = artist[0]
    album = mf.get("album", None)
    if album is not None:
      album = album[0]
    if artist and album:
      # stop at the first file that succeeds (for performance)
      break
  return artist, album

def analyze_dir(stats, parent_dir, rel_filepaths, cover_filename, scrobbler, time_progress_shown):
  """ Analyze a directory (non recursively) to get its album metadata if it is one. """
  metadata = None, None
  audio_filepaths = []
  for rel_filepath in rel_filepaths:
    stats["files"] += 1
    time_progress_shown = show_analyze_progress(stats, scrobbler, time_progress_shown=time_progress_shown)
    try:
      ext = os.path.splitext(rel_filepath)[1][1:].lower()
    except IndexError:
      continue
    if ext in AUDIO_EXTENSIONS:
      audio_filepaths.append(os.path.join(parent_dir, rel_filepath))
  if audio_filepaths:
    stats["albums"] += 1
    if not os.path.isfile(os.path.join(parent_dir, cover_filename)):
      stats["missing covers"] += 1
      metadata = get_metadata(audio_filepaths)
      if not all(metadata):
        # failed to get metadata for this album
        stats["errors"] += 1
  return metadata, time_progress_shown

def show_analyze_progress(stats, scrobbler, *, time_progress_shown=0, end=False):
  """ Display analysis global progress. """
  now = time.time()
  if (sys.stdout.isatty() and
     (end or (now - time_progress_shown > 0.1))):  # do not refresh display at each call (for performance)
    time_progress_shown = now
    print("Analyzing library %s | %s" % (next(scrobbler) if not end else "-",
                                         "  ".join(("%u %s" % (v, k)) for k, v in stats.items())),
          end="\r" if not end else "\n")
  return time_progress_shown

def get_covers(work, args):
  """ Get missing covers. """
  with concurrent.futures.ProcessPoolExecutor() as executor:
    # post work
    futures = {}
    for path, (artist, album) in work.items():
      future = executor.submit(sacad.search_and_download,
                               album,
                               artist,
                               args.format,
                               args.size,
                               args.size_tolerance_prct,
                               args.amazon_tlds,
                               args.no_lq_sources,
                               os.path.join(path, args.filename))
      futures[future] = (path, artist, album)

    # follow progress
    stats = collections.OrderedDict(((k, 0) for k in("ok", "errors", "no result found")))
    errors = []
    not_found = []
    for i, future in enumerate(concurrent.futures.as_completed(futures)):
      path, artist, album = futures[future]
      try:
        status = future.result()
      except:
        stats["errors"] += 1
        errors.append((path, artist, album))
      else:
        if status:
          stats["ok"] += 1
        else:
          stats["no result found"] += 1
          not_found.append((path, artist, album))
      show_get_covers_progress(i,
                               len(work),
                               stats,
                               artist=artist,
                               album=album)
  if work:
    show_get_covers_progress(i + 1, len(work), stats, end=True)
  for path, artist, album in not_found:
    print("Unable to find cover for '%s' by '%s' from '%s'" % (album, artist, path))
  for path, artist, album in errors:
    print("Error occured while searching cover for '%s' by '%s' from '%s'" % (album, artist, path))
  if errors:
    print("Please report this at https://github.com/desbma/sacad/issues")

def show_get_covers_progress(current_idx, total_count, stats, *, artist=None, album=None, end=False):
  """ Display search and download global progress. """
  if not sys.stdout.isatty():
    return
  line_width = shutil.get_terminal_size(fallback=(80, 0))[0] - 1
  print(" " * line_width, end="\r")
  print("Searching and downloading covers %u%% (%u/%u)%s | %s" % (100 * current_idx // total_count,
                                                                  current_idx,
                                                                  total_count,
                                                                  (" | Current album: '%s' '%s'" % (artist, album)) if not end else "",
                                                                  "  ".join(("%u %s" % (v, k)) for k, v in stats.items())),
        end="\r" if not end else "\n")

def cl_main():
  # parse args
  arg_parser = argparse.ArgumentParser(description="SACAD (recursive tool) v%s.%s" % (sacad.__version__,
                                                                                __doc__),
                                       formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  arg_parser.add_argument("lib_dir",
                          help="Music library directory to recursively analyze")
  arg_parser.add_argument("size",
                          type=int,
                          help="Target image size")
  arg_parser.add_argument("filename",
                          help="Cover image filename")
  sacad.setup_common_args(arg_parser)
  args = arg_parser.parse_args()
  args.format = os.path.splitext(args.filename)[1][1:].lower()
  try:
    args.format = sacad.SUPPORTED_IMG_FORMATS[args.format]
  except KeyError:
    print("Unable to guess image format from extension, or unknown format: %s" % (args.format))
    exit(1)

  # silence the logger
  logging.getLogger().setLevel(logging.ERROR)

  # do the job
  work = analyze_lib(args.lib_dir, args.filename)
  get_covers(work, args)
<|endoftext|>


<commit_before>'''
Make me some salt!
'''

# Import python libs
import os
import sys
import logging

# Import salt libs, the try block below bypasses an issue at build time so
# that modules don't cause the build to fail
from salt.version import __version__  # pylint: disable-msg=W402
from salt.utils import migrations

try:
    from salt.utils import parsers
    from salt.utils.verify import check_user, verify_env, verify_socket
    from salt.utils.verify import verify_files
except ImportError as e:
    if e.args[0] != 'No module named _msgpack':
        raise

logger = logging.getLogger(__name__)

class Master(parsers.MasterOptionParser):
    '''
    Creates a master server
    '''
    def prepare(self):
        '''
        Run the preparation sequence required to start a salt master server.

        If sub-classed, don't **ever** forget to run:

            super(YourSubClass, self).prepare()
        '''
        self.parse_args()

        try:
            if self.config['verify_env']:
                verify_env(
                    [
                        self.config['pki_dir'],
                        os.path.join(self.config['pki_dir'], 'minions'),
                        os.path.join(self.config['pki_dir'], 'minions_pre'),
                        os.path.join(self.config['pki_dir'],
                                     'minions_rejected'),
                        self.config['cachedir'],
                        os.path.join(self.config['cachedir'], 'jobs'),
                        os.path.join(self.config['cachedir'], 'proc'),
                        self.config['sock_dir'],
                        self.config['token_dir'],
                    ],
                    self.config['user'],
                    permissive=self.config['permissive_pki_access'],
                    pki_dir=self.config['pki_dir'],
                )
                logfile = self.config['log_file']
                if logfile is not None and (
                        not logfile.startswith('tcp://') or
                        not logfile.startswith('udp://') or
                        not logfile.startswith('file://')):
                    # Logfile is not using Syslog, verify
                        verify_files(
                            [logfile],
                            self.config['user']
                        )
        except OSError as err:
            sys.exit(err.errno)

        self.setup_logfile_logger()
        logger.warn('Setting up the Salt Master')

        if not verify_socket(self.config['interface'],
                             self.config['publish_port'],
                             self.config['ret_port']):
            self.exit(4, 'The ports are not available to bind\n')
        migrations.migrate_paths(self.config)

        # Late import so logging works correctly
        import salt.master
        self.master = salt.master.Master(self.config)
        self.daemonize_if_required()
        self.set_pidfile()

    def start(self):
        '''
        Start the actual master.

        If sub-classed, don't **ever** forget to run:

            super(YourSubClass, self).start()

        NOTE: Run any required code before calling `super()`.
        '''
        self.prepare()
        if check_user(self.config['user']):
            try:
                self.master.start()
            except salt.master.MasterExit:
                self.shutdown()
            finally:
                sys.exit()

    def shutdown(self):
        '''
        If sub-classed, run any shutdown operations on this method.
        '''

class Minion(parsers.MinionOptionParser):
    '''
    Create a minion server
    '''
    def prepare(self):
        '''
        Run the preparation sequence required to start a salt minion.

        If sub-classed, don't **ever** forget to run:

            super(YourSubClass, self).prepare()
        '''
        self.parse_args()

        try:
            if self.config['verify_env']:
                confd = os.path.join(
                    os.path.dirname(self.config['conf_file']),
                    'minion.d'
                )
                verify_env(
                    [
                        self.config['pki_dir'],
                        self.config['cachedir'],
                        self.config['sock_dir'],
                        self.config['extension_modules'],
                        confd,
                    ],
                    self.config['user'],
                    permissive=self.config['permissive_pki_access'],
                    pki_dir=self.config['pki_dir'],
                )
                if (not self.config['log_file'].startswith('tcp://') or
                    not self.config['log_file'].startswith('udp://') or
                    not self.config['log_file'].startswith('file://')):
                    # Logfile is not using Syslog, verify
                    verify_files(
                        [self.config['log_file']],
                        self.config['user']
                    )
                verify_files(
                    [self.config['log_file']],
                    self.config['user'])
        except OSError as err:
            sys.exit(err.errno)

        self.setup_logfile_logger()
        logger.warn(
            'Setting up the Salt Minion "{0}"'.format(
                self.config['id']
            )
        )
        migrations.migrate_paths(self.config)
        # Late import so logging works correctly
        import salt.minion
        # If the minion key has not been accepted, then Salt enters a loop
        # waiting for it, if we daemonize later then the minion could halt
        # the boot process waiting for a key to be accepted on the master.
        # This is the latest safe place to daemonize
        self.daemonize_if_required()
        self.minion = salt.minion.Minion(self.config)
        self.set_pidfile()

    def start(self):
        '''
        Start the actual minion.

        If sub-classed, don't **ever** forget to run:

            super(YourSubClass, self).start()

        NOTE: Run any required code before calling `super()`.
        '''
        self.prepare()
        try:
            if check_user(self.config['user']):
                self.minion.tune_in()
        except KeyboardInterrupt:
            logger.warn('Stopping the Salt Minion')
            self.shutdown()

    def shutdown(self):
        '''
        If sub-classed, run any shutdown operations on this method.
        '''

class Syndic(parsers.SyndicOptionParser):
    '''
    Create a syndic server
    '''

    def prepare(self):
        '''
        Run the preparation sequence required to start a salt syndic minion.

        If sub-classed, don't **ever** forget to run:

            super(YourSubClass, self).prepare()
        '''
        self.parse_args()
        try:
            if self.config['verify_env']:
                verify_env(
                    [
                        self.config['pki_dir'],
                        self.config['cachedir'],
                        self.config['sock_dir'],
                        self.config['extension_modules'],
                    ],
                    self.config['user'],
                    permissive=self.config['permissive_pki_access'],
                    pki_dir=self.config['pki_dir'],
                )
                if (not self.config['log_file'].startswith('tcp://') or
                    not self.config['log_file'].startswith('udp://') or
                    not self.config['log_file'].startswith('file://')):
                    # Logfile is not using Syslog, verify
                    verify_files(
                        [self.config['log_file']],
                        self.config['user']
                    )
        except OSError as err:
            sys.exit(err.errno)

        self.setup_logfile_logger()
        logger.warn(
            'Setting up the Salt Syndic Minion "{0}"'.format(
                self.config['id']
            )
        )

        # Late import so logging works correctly
        import salt.minion
        self.daemonize_if_required()
        self.syndic = salt.minion.Syndic(self.config)
        self.set_pidfile()

    def start(self):
        '''
        Start the actual syndic.

        If sub-classed, don't **ever** forget to run:

            super(YourSubClass, self).start()

        NOTE: Run any required code before calling `super()`.
        '''
        self.prepare()
        if check_user(self.config['user']):
            try:
                self.syndic.tune_in()
            except KeyboardInterrupt:
                logger.warn('Stopping the Salt Syndic Minion')
                self.shutdown()

    def shutdown(self):
        '''
        If sub-classed, run any shutdown operations on this method.
        '''
<commit_msg>Add a useful error message when the salt minion must exit because the IPC URI is too long.<commit_after>'''
Make me some salt!
'''

# Import python libs
import os
import sys
import logging

# Import salt libs, the try block below bypasses an issue at build time so
# that modules don't cause the build to fail
from salt.version import __version__  # pylint: disable-msg=W402
from salt.utils import migrations

try:
    from salt.utils import parsers
    from salt.utils.verify import check_user, verify_env, verify_socket
    from salt.utils.verify import verify_files
except ImportError as e:
    if e.args[0] != 'No module named _msgpack':
        raise
from salt.exceptions import SaltSystemExit

logger = logging.getLogger(__name__)

class Master(parsers.MasterOptionParser):
    '''
    Creates a master server
    '''
    def prepare(self):
        '''
        Run the preparation sequence required to start a salt master server.

        If sub-classed, don't **ever** forget to run:

            super(YourSubClass, self).prepare()
        '''
        self.parse_args()

        try:
            if self.config['verify_env']:
                verify_env(
                    [
                        self.config['pki_dir'],
                        os.path.join(self.config['pki_dir'], 'minions'),
                        os.path.join(self.config['pki_dir'], 'minions_pre'),
                        os.path.join(self.config['pki_dir'],
                                     'minions_rejected'),
                        self.config['cachedir'],
                        os.path.join(self.config['cachedir'], 'jobs'),
                        os.path.join(self.config['cachedir'], 'proc'),
                        self.config['sock_dir'],
                        self.config['token_dir'],
                    ],
                    self.config['user'],
                    permissive=self.config['permissive_pki_access'],
                    pki_dir=self.config['pki_dir'],
                )
                logfile = self.config['log_file']
                if logfile is not None and (
                        not logfile.startswith('tcp://') or
                        not logfile.startswith('udp://') or
                        not logfile.startswith('file://')):
                    # Logfile is not using Syslog, verify
                        verify_files(
                            [logfile],
                            self.config['user']
                        )
        except OSError as err:
            sys.exit(err.errno)

        self.setup_logfile_logger()
        logger.warn('Setting up the Salt Master')

        if not verify_socket(self.config['interface'],
                             self.config['publish_port'],
                             self.config['ret_port']):
            self.exit(4, 'The ports are not available to bind\n')
        migrations.migrate_paths(self.config)

        # Late import so logging works correctly
        import salt.master
        self.master = salt.master.Master(self.config)
        self.daemonize_if_required()
        self.set_pidfile()

    def start(self):
        '''
        Start the actual master.

        If sub-classed, don't **ever** forget to run:

            super(YourSubClass, self).start()

        NOTE: Run any required code before calling `super()`.
        '''
        self.prepare()
        if check_user(self.config['user']):
            try:
                self.master.start()
            except salt.master.MasterExit:
                self.shutdown()
            finally:
                sys.exit()

    def shutdown(self):
        '''
        If sub-classed, run any shutdown operations on this method.
        '''

class Minion(parsers.MinionOptionParser):
    '''
    Create a minion server
    '''
    def prepare(self):
        '''
        Run the preparation sequence required to start a salt minion.

        If sub-classed, don't **ever** forget to run:

            super(YourSubClass, self).prepare()
        '''
        self.parse_args()

        try:
            if self.config['verify_env']:
                confd = os.path.join(
                    os.path.dirname(self.config['conf_file']),
                    'minion.d'
                )
                verify_env(
                    [
                        self.config['pki_dir'],
                        self.config['cachedir'],
                        self.config['sock_dir'],
                        self.config['extension_modules'],
                        confd,
                    ],
                    self.config['user'],
                    permissive=self.config['permissive_pki_access'],
                    pki_dir=self.config['pki_dir'],
                )
                if (not self.config['log_file'].startswith('tcp://') or
                    not self.config['log_file'].startswith('udp://') or
                    not self.config['log_file'].startswith('file://')):
                    # Logfile is not using Syslog, verify
                    verify_files(
                        [self.config['log_file']],
                        self.config['user']
                    )
                verify_files(
                    [self.config['log_file']],
                    self.config['user'])
        except OSError as err:
            sys.exit(err.errno)

        self.setup_logfile_logger()
        logger.warn(
            'Setting up the Salt Minion "{0}"'.format(
                self.config['id']
            )
        )
        migrations.migrate_paths(self.config)
        # Late import so logging works correctly
        import salt.minion
        # If the minion key has not been accepted, then Salt enters a loop
        # waiting for it, if we daemonize later then the minion could halt
        # the boot process waiting for a key to be accepted on the master.
        # This is the latest safe place to daemonize
        self.daemonize_if_required()
        self.minion = salt.minion.Minion(self.config)
        self.set_pidfile()

    def start(self):
        '''
        Start the actual minion.

        If sub-classed, don't **ever** forget to run:

            super(YourSubClass, self).start()

        NOTE: Run any required code before calling `super()`.
        '''
        self.prepare()
        try:
            if check_user(self.config['user']):
                self.minion.tune_in()
        except (KeyboardInterrupt, SaltSystemExit) as e:
            logger.warn('Stopping the Salt Minion')
            if isinstance(e, KeyboardInterrupt):
                logger.warn('Exiting on Ctrl-c')
            else:
                logger.error(str(e))
        finally:
            self.shutdown()

    def shutdown(self):
        '''
        If sub-classed, run any shutdown operations on this method.
        '''

class Syndic(parsers.SyndicOptionParser):
    '''
    Create a syndic server
    '''

    def prepare(self):
        '''
        Run the preparation sequence required to start a salt syndic minion.

        If sub-classed, don't **ever** forget to run:

            super(YourSubClass, self).prepare()
        '''
        self.parse_args()
        try:
            if self.config['verify_env']:
                verify_env(
                    [
                        self.config['pki_dir'],
                        self.config['cachedir'],
                        self.config['sock_dir'],
                        self.config['extension_modules'],
                    ],
                    self.config['user'],
                    permissive=self.config['permissive_pki_access'],
                    pki_dir=self.config['pki_dir'],
                )
                if (not self.config['log_file'].startswith('tcp://') or
                    not self.config['log_file'].startswith('udp://') or
                    not self.config['log_file'].startswith('file://')):
                    # Logfile is not using Syslog, verify
                    verify_files(
                        [self.config['log_file']],
                        self.config['user']
                    )
        except OSError as err:
            sys.exit(err.errno)

        self.setup_logfile_logger()
        logger.warn(
            'Setting up the Salt Syndic Minion "{0}"'.format(
                self.config['id']
            )
        )

        # Late import so logging works correctly
        import salt.minion
        self.daemonize_if_required()
        self.syndic = salt.minion.Syndic(self.config)
        self.set_pidfile()

    def start(self):
        '''
        Start the actual syndic.

        If sub-classed, don't **ever** forget to run:

            super(YourSubClass, self).start()

        NOTE: Run any required code before calling `super()`.
        '''
        self.prepare()
        if check_user(self.config['user']):
            try:
                self.syndic.tune_in()
            except KeyboardInterrupt:
                logger.warn('Stopping the Salt Syndic Minion')
                self.shutdown()

    def shutdown(self):
        '''
        If sub-classed, run any shutdown operations on this method.
        '''
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# The pam components have been modified to be salty and have been taken from
# the pam module under this licence:
# (c) 2007 Chris AtLee <chris@atlee.ca>
# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license.php
'''
Authenticate against PAM

Provides an authenticate function that will allow the caller to authenticate
a user against the Pluggable Authentication Modules (PAM) on the system.

Implemented using ctypes, so no compilation is necessary.

.. note:: PAM authentication will not work for the ``root`` user.

    The Python interface to PAM does not support authenticating as ``root``.

'''

# Import python libs
from ctypes import CDLL, POINTER, Structure, CFUNCTYPE, cast, pointer, sizeof
from ctypes import c_void_p, c_uint, c_char_p, c_char, c_int
from ctypes.util import find_library

# Import Salt libs
from salt.utils import get_group_list

LIBPAM = CDLL(find_library('pam'))
LIBC = CDLL(find_library('c'))

CALLOC = LIBC.calloc
CALLOC.restype = c_void_p
CALLOC.argtypes = [c_uint, c_uint]

STRDUP = LIBC.strdup
STRDUP.argstypes = [c_char_p]
STRDUP.restype = POINTER(c_char)  # NOT c_char_p !!!!

# Various constants
PAM_PROMPT_ECHO_OFF = 1
PAM_PROMPT_ECHO_ON = 2
PAM_ERROR_MSG = 3
PAM_TEXT_INFO = 4

class PamHandle(Structure):
    '''
    Wrapper class for pam_handle_t
    '''
    _fields_ = [
            ('handle', c_void_p)
            ]

    def __init__(self):
        Structure.__init__(self)
        self.handle = 0

class PamMessage(Structure):
    '''
    Wrapper class for pam_message structure
    '''
    _fields_ = [
            ("msg_style", c_int),
            ("msg", c_char_p),
            ]

    def __repr__(self):
        return '<PamMessage {0} {1!r}>'.format(self.msg_style, self.msg)

class PamResponse(Structure):
    '''
    Wrapper class for pam_response structure
    '''
    _fields_ = [
            ('resp', c_char_p),
            ('resp_retcode', c_int),
            ]

    def __repr__(self):
        return '<PamResponse {0} {1!r}>'.format(self.resp_retcode, self.resp)

CONV_FUNC = CFUNCTYPE(c_int,
        c_int, POINTER(POINTER(PamMessage)),
               POINTER(POINTER(PamResponse)), c_void_p)

class PamConv(Structure):
    '''
    Wrapper class for pam_conv structure
    '''
    _fields_ = [
            ('conv', CONV_FUNC),
            ('appdata_ptr', c_void_p)
            ]

try:
    PAM_START = LIBPAM.pam_start
    PAM_START.restype = c_int
    PAM_START.argtypes = [c_char_p, c_char_p, POINTER(PamConv),
            POINTER(PamHandle)]

    PAM_AUTHENTICATE = LIBPAM.pam_authenticate
    PAM_AUTHENTICATE.restype = c_int
    PAM_AUTHENTICATE.argtypes = [PamHandle, c_int]
except Exception:
    HAS_PAM = False
else:
    HAS_PAM = True

def __virtual__():
    '''
    Only load on Linux systems
    '''
    return HAS_PAM

def authenticate(username, password, service='login'):
    '''
    Returns True if the given username and password authenticate for the
    given service.  Returns False otherwise

    ``username``: the username to authenticate

    ``password``: the password in plain text

    ``service``: the PAM service to authenticate against.
                 Defaults to 'login'
    '''
    @CONV_FUNC
    def my_conv(n_messages, messages, p_response, app_data):
        '''
        Simple conversation function that responds to any
        prompt where the echo is off with the supplied password
        '''
        # Create an array of n_messages response objects
        addr = CALLOC(n_messages, sizeof(PamResponse))
        p_response[0] = cast(addr, POINTER(PamResponse))
        for i in range(n_messages):
            if messages[i].contents.msg_style == PAM_PROMPT_ECHO_OFF:
                pw_copy = STRDUP(str(password))
                p_response.contents[i].resp = cast(pw_copy, c_char_p)
                p_response.contents[i].resp_retcode = 0
        return 0

    handle = PamHandle()
    conv = PamConv(my_conv, 0)
    retval = PAM_START(service, username, pointer(conv), pointer(handle))

    if retval != 0:
        # TODO: This is not an authentication error, something
        # has gone wrong starting up PAM
        return False

    retval = PAM_AUTHENTICATE(handle, 0)
    return retval == 0

def auth(username, password, **kwargs):
    '''
    Authenticate via pam
    '''
    return authenticate(username, password, kwargs.get('service', 'login'))

def groups(username, *args, **kwargs):
    '''
    Retrieve groups for a given user for this auth provider

    Uses system groups
    '''
    return get_group_list(username)
<commit_msg>Fix #16085 pam auth module was not closeing PamHandle. moreinfo: http://pubs.opengroup.org/onlinepubs/8329799/pam_end.htm<commit_after># -*- coding: utf-8 -*-
# The pam components have been modified to be salty and have been taken from
# the pam module under this licence:
# (c) 2007 Chris AtLee <chris@atlee.ca>
# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license.php
'''
Authenticate against PAM

Provides an authenticate function that will allow the caller to authenticate
a user against the Pluggable Authentication Modules (PAM) on the system.

Implemented using ctypes, so no compilation is necessary.

.. note:: PAM authentication will not work for the ``root`` user.

    The Python interface to PAM does not support authenticating as ``root``.

'''

# Import python libs
from ctypes import CDLL, POINTER, Structure, CFUNCTYPE, cast, pointer, sizeof
from ctypes import c_void_p, c_uint, c_char_p, c_char, c_int
from ctypes.util import find_library

# Import Salt libs
from salt.utils import get_group_list

LIBPAM = CDLL(find_library('pam'))
LIBC = CDLL(find_library('c'))

CALLOC = LIBC.calloc
CALLOC.restype = c_void_p
CALLOC.argtypes = [c_uint, c_uint]

STRDUP = LIBC.strdup
STRDUP.argstypes = [c_char_p]
STRDUP.restype = POINTER(c_char)  # NOT c_char_p !!!!

# Various constants
PAM_PROMPT_ECHO_OFF = 1
PAM_PROMPT_ECHO_ON = 2
PAM_ERROR_MSG = 3
PAM_TEXT_INFO = 4

class PamHandle(Structure):
    '''
    Wrapper class for pam_handle_t
    '''
    _fields_ = [
            ('handle', c_void_p)
            ]

    def __init__(self):
        Structure.__init__(self)
        self.handle = 0

class PamMessage(Structure):
    '''
    Wrapper class for pam_message structure
    '''
    _fields_ = [
            ("msg_style", c_int),
            ("msg", c_char_p),
            ]

    def __repr__(self):
        return '<PamMessage {0} {1!r}>'.format(self.msg_style, self.msg)

class PamResponse(Structure):
    '''
    Wrapper class for pam_response structure
    '''
    _fields_ = [
            ('resp', c_char_p),
            ('resp_retcode', c_int),
            ]

    def __repr__(self):
        return '<PamResponse {0} {1!r}>'.format(self.resp_retcode, self.resp)

CONV_FUNC = CFUNCTYPE(c_int,
        c_int, POINTER(POINTER(PamMessage)),
               POINTER(POINTER(PamResponse)), c_void_p)

class PamConv(Structure):
    '''
    Wrapper class for pam_conv structure
    '''
    _fields_ = [
            ('conv', CONV_FUNC),
            ('appdata_ptr', c_void_p)
            ]

try:
    PAM_START = LIBPAM.pam_start
    PAM_START.restype = c_int
    PAM_START.argtypes = [c_char_p, c_char_p, POINTER(PamConv),
            POINTER(PamHandle)]

    PAM_AUTHENTICATE = LIBPAM.pam_authenticate
    PAM_AUTHENTICATE.restype = c_int
    PAM_AUTHENTICATE.argtypes = [PamHandle, c_int]

    PAM_END = LIBPAM.pam_end
    PAM_END.restype = c_int
    PAM_END.argtypes = [PamHandle, c_int]
except Exception:
    HAS_PAM = False
else:
    HAS_PAM = True

def __virtual__():
    '''
    Only load on Linux systems
    '''
    return HAS_PAM

def authenticate(username, password, service='login'):
    '''
    Returns True if the given username and password authenticate for the
    given service.  Returns False otherwise

    ``username``: the username to authenticate

    ``password``: the password in plain text

    ``service``: the PAM service to authenticate against.
                 Defaults to 'login'
    '''
    @CONV_FUNC
    def my_conv(n_messages, messages, p_response, app_data):
        '''
        Simple conversation function that responds to any
        prompt where the echo is off with the supplied password
        '''
        # Create an array of n_messages response objects
        addr = CALLOC(n_messages, sizeof(PamResponse))
        p_response[0] = cast(addr, POINTER(PamResponse))
        for i in range(n_messages):
            if messages[i].contents.msg_style == PAM_PROMPT_ECHO_OFF:
                pw_copy = STRDUP(str(password))
                p_response.contents[i].resp = cast(pw_copy, c_char_p)
                p_response.contents[i].resp_retcode = 0
        return 0

    handle = PamHandle()
    conv = PamConv(my_conv, 0)
    retval = PAM_START(service, username, pointer(conv), pointer(handle))

    if retval != 0:
        # TODO: This is not an authentication error, something
        # has gone wrong starting up PAM
        PAM_END(handle, retval)
        return False

    retval = PAM_AUTHENTICATE(handle, 0)
    PAM_END(handle, 0)
    return retval == 0

def auth(username, password, **kwargs):
    '''
    Authenticate via pam
    '''
    return authenticate(username, password, kwargs.get('service', 'login'))

def groups(username, *args, **kwargs):
    '''
    Retrieve groups for a given user for this auth provider

    Uses system groups
    '''
    return get_group_list(username)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Import python libs
from __future__ import absolute_import, print_function
import os
import sys

# Import Salt libs
import salt.utils.job
from salt.ext.six import string_types
from salt.utils import parsers, print_cli
from salt.utils.args import yamlify_arg
from salt.exceptions import (
        SaltClientError,
        SaltInvocationError,
        EauthAuthenticationError
        )

# Import 3rd-party libs
import salt.ext.six as six

class SaltCMD(parsers.SaltCMDOptionParser):
    '''
    The execution of a salt command happens here
    '''

    def run(self):
        '''
        Execute the salt command line
        '''
        import salt.auth
        import salt.client
        self.parse_args()

        # Setup file logging!
        self.setup_logfile_logger()

        try:
            # We don't need to bail on config file permission errors
            # if the CLI
            # process is run with the -a flag
            skip_perm_errors = self.options.eauth != ''

            local = salt.client.get_local_client(
                self.get_config_file_path(),
                skip_perm_errors=skip_perm_errors)
        except SaltClientError as exc:
            self.exit(2, '{0}\n'.format(exc))
            return

        if self.options.batch or self.options.static:
            import salt.cli.batch
            eauth = {}
            if 'token' in self.config:
                eauth['token'] = self.config['token']

            # If using eauth and a token hasn't already been loaded into
            # kwargs, prompt the user to enter auth credentials
            if 'token' not in eauth and self.options.eauth:
                resolver = salt.auth.Resolver(self.config)
                res = resolver.cli(self.options.eauth)
                if self.options.mktoken and res:
                    tok = resolver.token_cli(
                            self.options.eauth,
                            res
                            )
                    if tok:
                        eauth['token'] = tok.get('token', '')
                if not res:
                    sys.stderr.write('ERROR: Authentication failed\n')
                    sys.exit(2)
                eauth.update(res)
                eauth['eauth'] = self.options.eauth

            if self.options.static:

                if not self.options.batch:
                    self.config['batch'] = '100%'

                batch = salt.cli.batch.Batch(self.config, eauth=eauth, quiet=True)

                ret = {}

                for res in batch.run():
                    ret.update(res)

                self._output_ret(ret, '')

            else:
                try:
                    batch = salt.cli.batch.Batch(self.config, eauth=eauth)
                except salt.exceptions.SaltClientError as exc:
                    # We will print errors to the console further down the stack
                    sys.exit(1)
                # Printing the output is already taken care of in run() itself
                for res in batch.run():
                    if self.options.failhard:
                        for ret in six.itervalues(res):
                            retcode = salt.utils.job.get_retcode(ret)
                            if retcode != 0:
                                sys.stderr.write('ERROR: Minions returned with non-zero exit code\n')
                                sys.exit(retcode)

        else:
            if self.options.timeout <= 0:
                self.options.timeout = local.opts['timeout']

            kwargs = {
                'tgt': self.config['tgt'],
                'fun': self.config['fun'],
                'arg': self.config['arg'],
                'timeout': self.options.timeout,
                'show_timeout': self.options.show_timeout,
                'show_jid': self.options.show_jid}

            if 'token' in self.config:
                try:
                    with salt.utils.fopen(os.path.join(self.config['cachedir'], '.root_key'), 'r') as fp_:
                        kwargs['key'] = fp_.readline()
                except IOError:
                    kwargs['token'] = self.config['token']

            kwargs['delimiter'] = self.options.delimiter

            if self.selected_target_option:
                kwargs['expr_form'] = self.selected_target_option
            else:
                kwargs['expr_form'] = 'glob'

            if getattr(self.options, 'return'):
                kwargs['ret'] = getattr(self.options, 'return')

            if getattr(self.options, 'return_config'):
                kwargs['ret_config'] = getattr(self.options, 'return_config')

            if getattr(self.options, 'module_executors'):
                kwargs['module_executors'] = yamlify_arg(getattr(self.options, 'module_executors'))

            if getattr(self.options, 'metadata'):
                kwargs['metadata'] = yamlify_arg(
                        getattr(self.options, 'metadata'))

            # If using eauth and a token hasn't already been loaded into
            # kwargs, prompt the user to enter auth credentials
            if 'token' not in kwargs and self.options.eauth:
                resolver = salt.auth.Resolver(self.config)
                res = resolver.cli(self.options.eauth)
                if self.options.mktoken and res:
                    tok = resolver.token_cli(
                            self.options.eauth,
                            res
                            )
                    if tok:
                        kwargs['token'] = tok.get('token', '')
                if not res:
                    sys.stderr.write('ERROR: Authentication failed\n')
                    sys.exit(2)
                kwargs.update(res)
                kwargs['eauth'] = self.options.eauth

            if self.config['async']:
                jid = local.cmd_async(**kwargs)
                print_cli('Executed command with job ID: {0}'.format(jid))
                return
            retcodes = []
            try:
                # local will be None when there was an error
                errors = []
                if local:
                    if self.options.subset:
                        cmd_func = local.cmd_subset
                        kwargs['sub'] = self.options.subset
                        kwargs['cli'] = True
                    else:
                        cmd_func = local.cmd_cli

                    if self.options.progress:
                        kwargs['progress'] = True
                        self.config['progress'] = True
                        ret = {}
                        for progress in cmd_func(**kwargs):
                            out = 'progress'
                            try:
                                self._progress_ret(progress, out)
                            except salt.exceptions.LoaderError as exc:
                                raise salt.exceptions.SaltSystemExit(exc)
                            if 'return_count' not in progress:
                                ret.update(progress)
                        self._progress_end(out)
                        self._print_returns_summary(ret)
                    elif self.config['fun'] == 'sys.doc':
                        ret = {}
                        out = ''
                        for full_ret in local.cmd_cli(**kwargs):
                            ret_, out, retcode = self._format_ret(full_ret)
                            ret.update(ret_)
                        self._output_ret(ret, out)
                    else:
                        if self.options.verbose:
                            kwargs['verbose'] = True
                        ret = {}
                        for full_ret in cmd_func(**kwargs):
                            try:
                                ret_, out, retcode = self._format_ret(full_ret)
                                retcodes.append(retcode)
                                self._output_ret(ret_, out)
                                ret.update(ret_)
                            except KeyError:
                                errors.append(full_ret)

                    # Returns summary
                    if self.config['cli_summary'] is True:
                        if self.config['fun'] != 'sys.doc':
                            if self.options.output is None:
                                self._print_returns_summary(ret)
                                self._print_errors_summary(errors)

                    # NOTE: Return code is set here based on if all minions
                    # returned 'ok' with a retcode of 0.
                    # This is the final point before the 'salt' cmd returns,
                    # which is why we set the retcode here.
                    if retcodes.count(0) < len(retcodes):
                        sys.stderr.write('ERROR: Minions returned with non-zero exit code\n')
                        sys.exit(11)

            except (SaltInvocationError, EauthAuthenticationError, SaltClientError) as exc:
                ret = str(exc)
                out = ''
                self._output_ret(ret, out)

    def _print_errors_summary(self, errors):
        if errors:
            print_cli('\n')
            print_cli('---------------------------')
            print_cli('Errors')
            print_cli('---------------------------')
            for minion in errors:
                print_cli(self._format_error(minion))

    def _print_returns_summary(self, ret):
        '''
        Display returns summary
        '''
        return_counter = 0
        not_return_counter = 0
        not_return_minions = []
        not_response_minions = []
        not_connected_minions = []
        for each_minion in ret:
            minion_ret = ret[each_minion]
            if (
                    isinstance(minion_ret, string_types)
                    and minion_ret.startswith("Minion did not return")
                    ):
                if "Not connected" in ret[each_minion]:
                    not_connected_minions.append(each_minion)
                elif "No response" in ret[each_minion]:
                    not_response_minions.append(each_minion)
                not_return_counter += 1
                not_return_minions.append(each_minion)
            else:
                return_counter += 1
        print_cli('\n')
        print_cli('-------------------------------------------')
        print_cli('Summary')
        print_cli('-------------------------------------------')
        print_cli('# of minions targeted: {0}'.format(return_counter + not_return_counter))
        print_cli('# of minions returned: {0}'.format(return_counter))
        print_cli('# of minions that did not return: {0}'.format(not_return_counter))
        if self.options.verbose:
            if not_connected_minions:
                print_cli('Minions not connected: {0}'.format(" ".join(not_connected_minions)))
            if not_response_minions:
                print_cli('Minions not responding: {0}'.format(" ".join(not_response_minions)))
        print_cli('-------------------------------------------')

    def _progress_end(self, out):
        import salt.output
        salt.output.progress_end(self.progress_bar)

    def _progress_ret(self, progress, out):
        '''
        Print progress events
        '''
        import salt.output
        # Get the progress bar
        if not hasattr(self, 'progress_bar'):
            try:
                self.progress_bar = salt.output.get_progress(self.config, out, progress)
            except Exception as exc:
                raise salt.exceptions.LoaderError('\nWARNING: Install the `progressbar` python package. '
                                                  'Requested job was still run but output cannot be displayed.\n')
        salt.output.update_progress(self.config, progress, self.progress_bar, out)

    def _output_ret(self, ret, out):
        '''
        Print the output from a single return to the terminal
        '''
        import salt.output
        # Handle special case commands
        if self.config['fun'] == 'sys.doc' and not isinstance(ret, Exception):
            self._print_docs(ret)
        else:
            # Determine the proper output method and run it
            salt.output.display_output(ret, out, self.config)
        if not ret:
            sys.stderr.write('ERROR: No return received\n')
            sys.exit(2)

    def _format_ret(self, full_ret):
        '''
        Take the full return data and format it to simple output
        '''
        ret = {}
        out = ''
        retcode = 0
        for key, data in six.iteritems(full_ret):
            ret[key] = data['ret']
            if 'out' in data:
                out = data['out']
            ret_retcode = salt.utils.job.get_retcode(data)
            if ret_retcode > retcode:
                retcode = ret_retcode
        return ret, out, retcode

    def _format_error(self, minion_error):
        for minion, error_doc in six.iteritems(minion_error):
            error = 'Minion [{0}] encountered exception \'{1}\''.format(minion, error_doc['message'])
        return error

    def _print_docs(self, ret):
        '''
        Print out the docstrings for all of the functions on the minions
        '''
        import salt.output
        docs = {}
        if not ret:
            self.exit(2, 'No minions found to gather docs from\n')
        if isinstance(ret, str):
            self.exit(2, '{0}\n'.format(ret))
        for host in ret:
            if ret[host] == 'Minion did not return. [Not connected]':
                continue
            for fun in ret[host]:
                if fun not in docs:
                    if ret[host][fun]:
                        docs[fun] = ret[host][fun]
        for fun in sorted(docs):
            salt.output.display_output(fun + ':', 'nested', self.config)
            print_cli(docs[fun])
            print_cli('')
<commit_msg>Summary requires full return information.<commit_after># -*- coding: utf-8 -*-

# Import python libs
from __future__ import absolute_import, print_function
import os
import sys

# Import Salt libs
import salt.utils.job
from salt.ext.six import string_types
from salt.utils import parsers, print_cli
from salt.utils.args import yamlify_arg
from salt.exceptions import (
        SaltClientError,
        SaltInvocationError,
        EauthAuthenticationError
        )

# Import 3rd-party libs
import salt.ext.six as six

class SaltCMD(parsers.SaltCMDOptionParser):
    '''
    The execution of a salt command happens here
    '''

    def run(self):
        '''
        Execute the salt command line
        '''
        import salt.auth
        import salt.client
        self.parse_args()

        # Setup file logging!
        self.setup_logfile_logger()

        try:
            # We don't need to bail on config file permission errors
            # if the CLI
            # process is run with the -a flag
            skip_perm_errors = self.options.eauth != ''

            local = salt.client.get_local_client(
                self.get_config_file_path(),
                skip_perm_errors=skip_perm_errors)
        except SaltClientError as exc:
            self.exit(2, '{0}\n'.format(exc))
            return

        if self.options.batch or self.options.static:
            import salt.cli.batch
            eauth = {}
            if 'token' in self.config:
                eauth['token'] = self.config['token']

            # If using eauth and a token hasn't already been loaded into
            # kwargs, prompt the user to enter auth credentials
            if 'token' not in eauth and self.options.eauth:
                resolver = salt.auth.Resolver(self.config)
                res = resolver.cli(self.options.eauth)
                if self.options.mktoken and res:
                    tok = resolver.token_cli(
                            self.options.eauth,
                            res
                            )
                    if tok:
                        eauth['token'] = tok.get('token', '')
                if not res:
                    sys.stderr.write('ERROR: Authentication failed\n')
                    sys.exit(2)
                eauth.update(res)
                eauth['eauth'] = self.options.eauth

            if self.options.static:

                if not self.options.batch:
                    self.config['batch'] = '100%'

                batch = salt.cli.batch.Batch(self.config, eauth=eauth, quiet=True)

                ret = {}

                for res in batch.run():
                    ret.update(res)

                self._output_ret(ret, '')

            else:
                try:
                    batch = salt.cli.batch.Batch(self.config, eauth=eauth)
                except salt.exceptions.SaltClientError as exc:
                    # We will print errors to the console further down the stack
                    sys.exit(1)
                # Printing the output is already taken care of in run() itself
                for res in batch.run():
                    if self.options.failhard:
                        for ret in six.itervalues(res):
                            retcode = salt.utils.job.get_retcode(ret)
                            if retcode != 0:
                                sys.stderr.write('ERROR: Minions returned with non-zero exit code\n')
                                sys.exit(retcode)

        else:
            if self.options.timeout <= 0:
                self.options.timeout = local.opts['timeout']

            kwargs = {
                'tgt': self.config['tgt'],
                'fun': self.config['fun'],
                'arg': self.config['arg'],
                'timeout': self.options.timeout,
                'show_timeout': self.options.show_timeout,
                'show_jid': self.options.show_jid}

            if 'token' in self.config:
                try:
                    with salt.utils.fopen(os.path.join(self.config['cachedir'], '.root_key'), 'r') as fp_:
                        kwargs['key'] = fp_.readline()
                except IOError:
                    kwargs['token'] = self.config['token']

            kwargs['delimiter'] = self.options.delimiter

            if self.selected_target_option:
                kwargs['expr_form'] = self.selected_target_option
            else:
                kwargs['expr_form'] = 'glob'

            if getattr(self.options, 'return'):
                kwargs['ret'] = getattr(self.options, 'return')

            if getattr(self.options, 'return_config'):
                kwargs['ret_config'] = getattr(self.options, 'return_config')

            if getattr(self.options, 'module_executors'):
                kwargs['module_executors'] = yamlify_arg(getattr(self.options, 'module_executors'))

            if getattr(self.options, 'metadata'):
                kwargs['metadata'] = yamlify_arg(
                        getattr(self.options, 'metadata'))

            # If using eauth and a token hasn't already been loaded into
            # kwargs, prompt the user to enter auth credentials
            if 'token' not in kwargs and self.options.eauth:
                resolver = salt.auth.Resolver(self.config)
                res = resolver.cli(self.options.eauth)
                if self.options.mktoken and res:
                    tok = resolver.token_cli(
                            self.options.eauth,
                            res
                            )
                    if tok:
                        kwargs['token'] = tok.get('token', '')
                if not res:
                    sys.stderr.write('ERROR: Authentication failed\n')
                    sys.exit(2)
                kwargs.update(res)
                kwargs['eauth'] = self.options.eauth

            if self.config['async']:
                jid = local.cmd_async(**kwargs)
                print_cli('Executed command with job ID: {0}'.format(jid))
                return
            retcodes = []
            try:
                # local will be None when there was an error
                errors = []
                if local:
                    if self.options.subset:
                        cmd_func = local.cmd_subset
                        kwargs['sub'] = self.options.subset
                        kwargs['cli'] = True
                    else:
                        cmd_func = local.cmd_cli

                    if self.options.progress:
                        kwargs['progress'] = True
                        self.config['progress'] = True
                        ret = {}
                        for progress in cmd_func(**kwargs):
                            out = 'progress'
                            try:
                                self._progress_ret(progress, out)
                            except salt.exceptions.LoaderError as exc:
                                raise salt.exceptions.SaltSystemExit(exc)
                            if 'return_count' not in progress:
                                ret.update(progress)
                        self._progress_end(out)
                        self._print_returns_summary(ret)
                    elif self.config['fun'] == 'sys.doc':
                        ret = {}
                        out = ''
                        for full_ret in local.cmd_cli(**kwargs):
                            ret_, out, retcode = self._format_ret(full_ret)
                            ret.update(full_ret)
                        self._output_ret(ret, out)
                    else:
                        if self.options.verbose:
                            kwargs['verbose'] = True
                        ret = {}
                        for full_ret in cmd_func(**kwargs):
                            try:
                                ret_, out, retcode = self._format_ret(full_ret)
                                retcodes.append(retcode)
                                self._output_ret(ret_, out)
                                ret.update(full_ret)
                            except KeyError:
                                errors.append(full_ret)

                    # Returns summary
                    if self.config['cli_summary'] is True:
                        if self.config['fun'] != 'sys.doc':
                            if self.options.output is None:
                                self._print_returns_summary(ret)
                                self._print_errors_summary(errors)

                    # NOTE: Return code is set here based on if all minions
                    # returned 'ok' with a retcode of 0.
                    # This is the final point before the 'salt' cmd returns,
                    # which is why we set the retcode here.
                    if retcodes.count(0) < len(retcodes):
                        sys.stderr.write('ERROR: Minions returned with non-zero exit code\n')
                        sys.exit(11)

            except (SaltInvocationError, EauthAuthenticationError, SaltClientError) as exc:
                ret = str(exc)
                out = ''
                self._output_ret(ret, out)

    def _print_errors_summary(self, errors):
        if errors:
            print_cli('\n')
            print_cli('---------------------------')
            print_cli('Errors')
            print_cli('---------------------------')
            for minion in errors:
                print_cli(self._format_error(minion))

    def _print_returns_summary(self, ret):
        '''
        Display returns summary
        '''
        return_counter = 0
        not_return_counter = 0
        not_return_minions = []
        not_response_minions = []
        not_connected_minions = []
        for each_minion in ret:
            minion_ret = ret[each_minion]
            if (
                    isinstance(minion_ret, string_types)
                    and minion_ret.startswith("Minion did not return")
                    ):
                if "Not connected" in ret[each_minion]:
                    not_connected_minions.append(each_minion)
                elif "No response" in ret[each_minion]:
                    not_response_minions.append(each_minion)
                not_return_counter += 1
                not_return_minions.append(each_minion)
            else:
                return_counter += 1
        print_cli('\n')
        print_cli('-------------------------------------------')
        print_cli('Summary')
        print_cli('-------------------------------------------')
        print_cli('# of minions targeted: {0}'.format(return_counter + not_return_counter))
        print_cli('# of minions returned: {0}'.format(return_counter))
        print_cli('# of minions that did not return: {0}'.format(not_return_counter))
        if self.options.verbose:
            if not_connected_minions:
                print_cli('Minions not connected: {0}'.format(" ".join(not_connected_minions)))
            if not_response_minions:
                print_cli('Minions not responding: {0}'.format(" ".join(not_response_minions)))
        print_cli('-------------------------------------------')

    def _progress_end(self, out):
        import salt.output
        salt.output.progress_end(self.progress_bar)

    def _progress_ret(self, progress, out):
        '''
        Print progress events
        '''
        import salt.output
        # Get the progress bar
        if not hasattr(self, 'progress_bar'):
            try:
                self.progress_bar = salt.output.get_progress(self.config, out, progress)
            except Exception as exc:
                raise salt.exceptions.LoaderError('\nWARNING: Install the `progressbar` python package. '
                                                  'Requested job was still run but output cannot be displayed.\n')
        salt.output.update_progress(self.config, progress, self.progress_bar, out)

    def _output_ret(self, ret, out):
        '''
        Print the output from a single return to the terminal
        '''
        import salt.output
        # Handle special case commands
        if self.config['fun'] == 'sys.doc' and not isinstance(ret, Exception):
            self._print_docs(ret)
        else:
            # Determine the proper output method and run it
            salt.output.display_output(ret, out, self.config)
        if not ret:
            sys.stderr.write('ERROR: No return received\n')
            sys.exit(2)

    def _format_ret(self, full_ret):
        '''
        Take the full return data and format it to simple output
        '''
        ret = {}
        out = ''
        retcode = 0
        for key, data in six.iteritems(full_ret):
            ret[key] = data['ret']
            if 'out' in data:
                out = data['out']
            ret_retcode = salt.utils.job.get_retcode(data)
            if ret_retcode > retcode:
                retcode = ret_retcode
        return ret, out, retcode

    def _format_error(self, minion_error):
        for minion, error_doc in six.iteritems(minion_error):
            error = 'Minion [{0}] encountered exception \'{1}\''.format(minion, error_doc['message'])
        return error

    def _print_docs(self, ret):
        '''
        Print out the docstrings for all of the functions on the minions
        '''
        import salt.output
        docs = {}
        if not ret:
            self.exit(2, 'No minions found to gather docs from\n')
        if isinstance(ret, str):
            self.exit(2, '{0}\n'.format(ret))
        for host in ret:
            if ret[host] == 'Minion did not return. [Not connected]':
                continue
            for fun in ret[host]:
                if fun not in docs:
                    if ret[host][fun]:
                        docs[fun] = ret[host][fun]
        for fun in sorted(docs):
            salt.output.display_output(fun + ':', 'nested', self.config)
            print_cli(docs[fun])
            print_cli('')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)`

    salt.syspaths
    ~~~~~~~~~~~~~

    Salt's defaults system paths

    This module allows defining Salt's default paths at build time by writing a
    ``_syspath.py`` file to the filesystem. This is useful, for example, for
    setting platform-specific defaults that differ from the standard Linux
    paths.

    These values are static values and must be considered as secondary to any
    paths that are set in the master/minion config files.
'''

# Import python libs
import sys
import os.path

if 'SETUP_DIRNAME' in globals():
    # This is from the exec() call in Salt's setup.py
    THIS_FILE = os.path.join(SETUP_DIRNAME, 'salt', 'syspaths.py')  # pylint: disable=E0602
else:
    THIS_FILE = __file__

try:
    # Let's try loading the system paths from the generated module at
    # installation time.
    from salt._syspaths import (  # pylint: disable=W0611,E0611,import-error
        ROOT_DIR,                 # because pylint thinks that _syspaths is an
        CONFIG_DIR,               # attribute of salt.__init__
        CACHE_DIR,
        SOCK_DIR,
        SRV_ROOT_DIR,
        BASE_FILE_ROOTS_DIR,
        BASE_PILLAR_ROOTS_DIR,
        BASE_MASTER_ROOTS_DIR,
        LOGS_DIR,
        PIDFILE_DIR,
    )
except ImportError:
    # The installation time was not generated, let's define the default values
    __platform = sys.platform.lower()
    if __platform.startswith('win'):
        ROOT_DIR = r'c:\salt' or '/'
        CONFIG_DIR = os.path.join(ROOT_DIR, 'conf')
    else:
        ROOT_DIR = '/'
        if 'freebsd' in __platform:
            CONFIG_DIR = os.path.join(ROOT_DIR, 'usr', 'local', 'etc', 'salt')
        elif 'netbsd' in __platform:
            CONFIG_DIR = os.path.join(ROOT_DIR, 'usr', 'pkg', 'etc', 'salt')
        else:
            CONFIG_DIR = os.path.join(ROOT_DIR, 'etc', 'salt')
    CACHE_DIR = os.path.join(ROOT_DIR, 'var', 'cache', 'salt')
    SOCK_DIR = os.path.join(ROOT_DIR, 'var', 'run', 'salt')
    SRV_ROOT_DIR = os.path.join(ROOT_DIR, 'srv')
    BASE_FILE_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, 'salt')
    BASE_PILLAR_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, 'pillar')
    BASE_MASTER_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, 'salt-master')
    LOGS_DIR = os.path.join(ROOT_DIR, 'var', 'log', 'salt')
    PIDFILE_DIR = os.path.join(ROOT_DIR, 'var', 'run')
    INSTALL_DIR = os.path.dirname(os.path.realpath(THIS_FILE))
    CLOUD_DIR = os.path.join(INSTALL_DIR, 'cloud')
    BOOTSTRAP = os.path.join(CLOUD_DIR, 'deploy', 'bootstrap-salt.sh')
<commit_msg>py3 for syspaths<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)`

    salt.syspaths
    ~~~~~~~~~~~~~

    Salt's defaults system paths

    This module allows defining Salt's default paths at build time by writing a
    ``_syspath.py`` file to the filesystem. This is useful, for example, for
    setting platform-specific defaults that differ from the standard Linux
    paths.

    These values are static values and must be considered as secondary to any
    paths that are set in the master/minion config files.
'''

# Import python libs
from __future__ import absolute_import
import sys
import os.path

if 'SETUP_DIRNAME' in globals():
    # This is from the exec() call in Salt's setup.py
    THIS_FILE = os.path.join(SETUP_DIRNAME, 'salt', 'syspaths.py')  # pylint: disable=E0602
else:
    THIS_FILE = __file__

try:
    # Let's try loading the system paths from the generated module at
    # installation time.
    from salt._syspaths import (  # pylint: disable=W0611,E0611,import-error
        ROOT_DIR,                 # because pylint thinks that _syspaths is an
        CONFIG_DIR,               # attribute of salt.__init__
        CACHE_DIR,
        SOCK_DIR,
        SRV_ROOT_DIR,
        BASE_FILE_ROOTS_DIR,
        BASE_PILLAR_ROOTS_DIR,
        BASE_MASTER_ROOTS_DIR,
        LOGS_DIR,
        PIDFILE_DIR,
    )
except ImportError:
    # The installation time was not generated, let's define the default values
    __platform = sys.platform.lower()
    if __platform.startswith('win'):
        ROOT_DIR = r'c:\salt' or '/'
        CONFIG_DIR = os.path.join(ROOT_DIR, 'conf')
    else:
        ROOT_DIR = '/'
        if 'freebsd' in __platform:
            CONFIG_DIR = os.path.join(ROOT_DIR, 'usr', 'local', 'etc', 'salt')
        elif 'netbsd' in __platform:
            CONFIG_DIR = os.path.join(ROOT_DIR, 'usr', 'pkg', 'etc', 'salt')
        else:
            CONFIG_DIR = os.path.join(ROOT_DIR, 'etc', 'salt')
    CACHE_DIR = os.path.join(ROOT_DIR, 'var', 'cache', 'salt')
    SOCK_DIR = os.path.join(ROOT_DIR, 'var', 'run', 'salt')
    SRV_ROOT_DIR = os.path.join(ROOT_DIR, 'srv')
    BASE_FILE_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, 'salt')
    BASE_PILLAR_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, 'pillar')
    BASE_MASTER_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, 'salt-master')
    LOGS_DIR = os.path.join(ROOT_DIR, 'var', 'log', 'salt')
    PIDFILE_DIR = os.path.join(ROOT_DIR, 'var', 'run')
    INSTALL_DIR = os.path.dirname(os.path.realpath(THIS_FILE))
    CLOUD_DIR = os.path.join(INSTALL_DIR, 'cloud')
    BOOTSTRAP = os.path.join(CLOUD_DIR, 'deploy', 'bootstrap-salt.sh')
<|endoftext|>