<commit_before>#!/usr/bin/env python
from __future__ import division

__author__ = "Jai Ram Rideout"
__copyright__ = "Copyright 2012, The QIIME project"
__credits__ = ["Jai Ram Rideout", "Zack Ellett", "Kyle Patnode"]
__license__ = "GPL"
__version__ = "1.6.0-dev"
__maintainer__ = "Jai Ram Rideout"
__email__ = "jai.rideout@gmail.com"
__status__ = "Development"

from qiime.util import (parse_command_line_parameters, get_options_lookup,
                        make_option)
from qiime.workflow.util import (call_commands_serially, no_status_updates,
                                 print_commands, print_to_stdout)

from taxcompare.multiple_assign_taxonomy import assign_taxonomy_multiple_times

options_lookup = get_options_lookup()

script_info = {}
script_info['brief_description'] = "Assigns taxonomy with multiple taxonomy assigners"
script_info['script_description'] = """
This script provides a workflow to assign taxonomy using multiple taxonomy
assigners over multiple datasets.

It accepts a list of directories as input (one for each dataset), where each
directory contains the same set of files following a common naming convention.
At a minimum, each directory must have an OTU table without taxonomic
information and a fasta file containing the representative sequences that need
to be assigned taxonomy. Additional files may be needed depending on the
taxonomy assignment method; please refer to the script option documentation
below for more details.

The script creates (under the specified output directory) a subdirectory for
each input dataset directory. Under each of these directories, a subdirectory
is created for every possible combination of taxonomy assigners and parameters.
Each of these directories will contain an OTU table with taxonomic information,
the representative set's taxonomic assignments, and a set of taxa summary files
at varying taxonomic levels.

For example, if ran the script using the RDP classifier at 0.6 and 0.8
confidences over the S16S-1 input dataset, we'd get an output directory that
looks like:

output_dir
  S16S-1
    rdp_0.6
      otu_table.biom
      ...
    rdp_0.8
      otu_table.biom
      ...

The script supports efficient reruns, meaning that it will not reprocess a
taxonomy assigner/parameter combo if the expected output subdirectory already
exists. This allows the script to be rerun in an incremental fashion (e.g. if
the script fails or we decide to add a new assigner or parameter, we don't want
to have to rerun everything from scratch).
"""

script_info['script_usage'] = []
script_info['script_usage'].append(("Multiple RDP and mothur assignments",
"This example shows how to assign taxonomy using RDP and mothur, at 0.6 and "
"0.8 confidence levels, over the S16S-1 and S16S-2 datasets. This command "
"will result in a total of eight subdirectories (four for each dataset).",
"%prog -i S16S-1,S16S-2 -o example_1_output -m rdp,mothur -r gg_97_otus_4feb2011.fasta -t greengenes_tax.txt -c 0.6,0.8"))

script_info['output_description'] = ""

script_info['required_options'] = [
    make_option('-i', '--input_dirs', type='string',
        help='Comma-separated list of input dataset directories'),
    options_lookup['output_dir'],
    make_option('-m', '--assignment_methods', type='string',
        help='Comma-separated list of taxon assignment methods to use, either '
        'blast, mothur, rdp, or rtax'),
    make_option('-r', '--reference_seqs_fp', type='existing_filepath',
        help='Path to reference sequences.  For assignment with blast, these '
        'are used to generate a blast database. For assignment with rdp and '
        'mothur, they are used as training sequences for the classifier'),
    make_option('-t', '--id_to_taxonomy_fp', type='existing_filepath',
        help='Path to tab-delimited file mapping sequences to assigned '
        'taxonomy. Each assigned taxonomy is provided as a '
        'semicolon-separated list.')
]
script_info['optional_options'] = [
    make_option('-c', '--confidences', type='string',
        help='Comma-separated list of minimum confidences to record an '
        'assignment, only used for rdp and mothur methods [default: %default]',
        default=None),
    make_option('-e', '--e_values', type='string',
        help='Comma-separated list of maximum e-values to record an '
        'assignment, only used for blast method [default: %default]',
        default=None),
    make_option('-x', '--rtax_modes', type='string',
        help='Comma-separated list of modes to run rtax in. Modes can be '
        'either "single" or "paired". If paired, paired-end reads must be '
        'available (i.e. forward and reverse reads, demultiplexed) '
        '[default: %default]', default=None),
    make_option('--input_fasta_filename', type='string',
        help='Name of fasta file containing sequences to receive taxonomy '
        'assignment. Must exist under each input dataset directory '
        '[default: %default]', default='rep_set.fna'),
    make_option('--clean_otu_table_filename', type='string',
        help='Name of OTU table BIOM file that will have taxonomic '
        'information added to it. Must exist under each input dataset '
        'directory [default: %default]', default='otu_table_mc2.biom'),
    make_option('--read_1_seqs_filename', type='string',
        help='Name of fasta file containing the first read from paired-end '
        'sequencing, prior to OTU clustering (used for RTAX only). Must exist '
        'under each input dataset directory if rtax is specified as an '
        'assignment method [default: %default]', default='seqs1.fna'),
    make_option('--read_2_seqs_filename', type='string',
        help='Name of fasta file containing a second read from paired-end '
        'sequencing, prior to OTU clustering (used for RTAX only). Must exist '
        'under each input dataset directory if rtax is specified as an '
        'assignment method and the "paired" rtax mode is specified '
        '[default: %default]', default='seqs2.fna'),
    make_option('--rtax_read_id_regexes', type='string',
        help='Comma-separated list of regular expressions that will be passed '
        'to RTAX for parsing read 1 IDs. The number (and order) of the '
        'supplied regexes must match the number (and order) of input datasets '
        'supplied via -i/--input_dirs [default: %default]', default=None),
    make_option('--rtax_amplicon_id_regexes', type='string',
        help='Comma-separated list of regular expressions that will be passed '
        'to RTAX for parsing amplicon IDs. The number (and order) of the '
        'supplied regexes must match the number (and order) of input datasets '
        'supplied via -i/--input_dirs [default: %default]', default=None),
    make_option('--rtax_header_id_regexes', type='string',
        help='Comma-separated list of regular expressions that will be passed '
        'to RTAX for parsing header IDs. The number (and order) of the '
        'supplied regexes must match the number (and order) of input datasets '
        'supplied via -i/--input_dirs [default: %default]', default=None),
    make_option('--rdp_max_memory', type='string',
        help='Maximum memory allocation, in MB, for JVM when using the rdp '
        'method. Increase for large training sets [default: %default]',
        default=4000),
    make_option('-w', '--print_only', action='store_true',
        help='Print the commands but don\'t call them -- useful for debugging '
        '[default: %default]', default=False),
    make_option('-f', '--force', action='store_true',
        help='Force overwrite of existing output directory (note: existing '
        'files in output_dir will not be removed). Subdirectories that '
        'already exist will be skipped (they are assumed to have been '
        'successfully created during a previous run of the script) '
        '[default: %default]', default=False)
]
script_info['version'] = __version__

def main():
    option_parser, opts, args = parse_command_line_parameters(**script_info)

    input_dirs = opts.input_dirs.split(',')
    assignment_methods = opts.assignment_methods.split(',')

    confidences = opts.confidences
    if confidences is not None:
        confidences = map(float, opts.confidences.split(','))

    e_values = opts.e_values
    if e_values is not None:
        e_values = map(float, opts.e_values.split(','))

    rtax_modes = opts.rtax_modes
    if rtax_modes is not None:
        rtax_modes = opts.rtax_modes.split(',')

    rtax_read_id_regexes = opts.rtax_read_id_regexes
    if rtax_read_id_regexes is not None:
        rtax_read_id_regexes = rtax_read_id_regexes.split(',')

    rtax_amplicon_id_regexes = opts.rtax_amplicon_id_regexes
    if rtax_amplicon_id_regexes is not None:
        rtax_amplicon_id_regexes = rtax_amplicon_id_regexes.split(',')

    rtax_header_id_regexes = opts.rtax_header_id_regexes
    if rtax_header_id_regexes is not None:
        rtax_header_id_regexes = rtax_header_id_regexes.split(',')

    if opts.print_only:
        command_handler = print_commands
    else:
        command_handler = call_commands_serially

    if opts.verbose:
        status_update_callback = print_to_stdout
    else:
        status_update_callback = no_status_updates

    assign_taxonomy_multiple_times(input_dirs, opts.output_dir,
        assignment_methods, opts.reference_seqs_fp,
        opts.id_to_taxonomy_fp, confidences=confidences,
        e_values=e_values, rtax_modes=rtax_modes,
        input_fasta_filename=opts.input_fasta_filename,
        clean_otu_table_filename=opts.clean_otu_table_filename,
        read_1_seqs_filename=opts.read_1_seqs_filename,
        read_2_seqs_filename=opts.read_2_seqs_filename,
        rtax_read_id_regexes=rtax_read_id_regexes,
        rtax_amplicon_id_regexes=rtax_amplicon_id_regexes,
        rtax_header_id_regexes=rtax_header_id_regexes,
        rdp_max_memory=opts.rdp_max_memory,
        command_handler=command_handler,
        status_update_callback=status_update_callback, force=opts.force)

if __name__ == "__main__":
    main()
<commit_msg>Add three new uclust-specific options<commit_after>#!/usr/bin/env python
from __future__ import division

__author__ = "Jai Ram Rideout"
__copyright__ = "Copyright 2012, The QIIME project"
__credits__ = ["Jai Ram Rideout", "Zack Ellett", "Kyle Patnode"]
__license__ = "GPL"
__version__ = "1.6.0-dev"
__maintainer__ = "Jai Ram Rideout"
__email__ = "jai.rideout@gmail.com"
__status__ = "Development"

from qiime.util import (parse_command_line_parameters, get_options_lookup,
                        make_option)
from qiime.workflow.util import (call_commands_serially, no_status_updates,
                                 print_commands, print_to_stdout)

from taxcompare.multiple_assign_taxonomy import assign_taxonomy_multiple_times

options_lookup = get_options_lookup()

script_info = {}
script_info['brief_description'] = "Assigns taxonomy with multiple taxonomy assigners"
script_info['script_description'] = """
This script provides a workflow to assign taxonomy using multiple taxonomy
assigners over multiple datasets.

It accepts a list of directories as input (one for each dataset), where each
directory contains the same set of files following a common naming convention.
At a minimum, each directory must have an OTU table without taxonomic
information and a fasta file containing the representative sequences that need
to be assigned taxonomy. Additional files may be needed depending on the
taxonomy assignment method; please refer to the script option documentation
below for more details.

The script creates (under the specified output directory) a subdirectory for
each input dataset directory. Under each of these directories, a subdirectory
is created for every possible combination of taxonomy assigners and parameters.
Each of these directories will contain an OTU table with taxonomic information,
the representative set's taxonomic assignments, and a set of taxa summary files
at varying taxonomic levels.

For example, if ran the script using the RDP classifier at 0.6 and 0.8
confidences over the S16S-1 input dataset, we'd get an output directory that
looks like:

output_dir
  S16S-1
    rdp_0.6
      otu_table.biom
      ...
    rdp_0.8
      otu_table.biom
      ...

The script supports efficient reruns, meaning that it will not reprocess a
taxonomy assigner/parameter combo if the expected output subdirectory already
exists. This allows the script to be rerun in an incremental fashion (e.g. if
the script fails or we decide to add a new assigner or parameter, we don't want
to have to rerun everything from scratch).
"""

script_info['script_usage'] = []
script_info['script_usage'].append(("Multiple RDP and mothur assignments",
"This example shows how to assign taxonomy using RDP and mothur, at 0.6 and "
"0.8 confidence levels, over the S16S-1 and S16S-2 datasets. This command "
"will result in a total of eight subdirectories (four for each dataset).",
"%prog -i S16S-1,S16S-2 -o example_1_output -m rdp,mothur -r gg_97_otus_4feb2011.fasta -t greengenes_tax.txt -c 0.6,0.8"))

script_info['output_description'] = ""

script_info['required_options'] = [
    make_option('-i', '--input_dirs', type='string',
        help='Comma-separated list of input dataset directories'),
    options_lookup['output_dir'],
    make_option('-m', '--assignment_methods', type='string',
        help='Comma-separated list of taxon assignment methods to use, either '
        'blast, mothur, rdp, or rtax'),
    make_option('-r', '--reference_seqs_fp', type='existing_filepath',
        help='Path to reference sequences.  For assignment with blast, these '
        'are used to generate a blast database. For assignment with rdp and '
        'mothur, they are used as training sequences for the classifier'),
    make_option('-t', '--id_to_taxonomy_fp', type='existing_filepath',
        help='Path to tab-delimited file mapping sequences to assigned '
        'taxonomy. Each assigned taxonomy is provided as a '
        'semicolon-separated list.')
]
script_info['optional_options'] = [
    make_option('-c', '--confidences', type='string',
        help='Comma-separated list of minimum confidences to record an '
        'assignment, only used for rdp and mothur methods [default: %default]',
        default=None),
    make_option('-e', '--e_values', type='string',
        help='Comma-separated list of maximum e-values to record an '
        'assignment, only used for blast method [default: %default]',
        default=None),
    make_option('-x', '--rtax_modes', type='string',
        help='Comma-separated list of modes to run rtax in. Modes can be '
        'either "single" or "paired". If paired, paired-end reads must be '
        'available (i.e. forward and reverse reads, demultiplexed) '
        '[default: %default]', default=None),
    make_option('--uclust_min_consensus_fractions', type='string',
        help='Comma-separated list of floats indicating minimum consensus '
        'fractions. Each value specifies the minimum fraction of database '
        'hits that must have a specific taxonomic assignment to assign that '
        'taxonomy to a query, only used for uclust method [default: %default]',
        default=None),
    make_option('--uclust_similarities', type='string',
        help='Comma-separated list of floats indicating the minimum percent '
        'similarities to consider a database match a hit, only used for '
        'uclust method [default: %default]', default=None),
    make_option('--uclust_max_accepts', type='string',
        help='Comma-separated list of integers indicating the number of '
        'database hits to consider when making an assignment, only used for '
        'uclust method [default: %default]', default=None),
    make_option('--input_fasta_filename', type='string',
        help='Name of fasta file containing sequences to receive taxonomy '
        'assignment. Must exist under each input dataset directory '
        '[default: %default]', default='rep_set.fna'),
    make_option('--clean_otu_table_filename', type='string',
        help='Name of OTU table BIOM file that will have taxonomic '
        'information added to it. Must exist under each input dataset '
        'directory [default: %default]', default='otu_table_mc2.biom'),
    make_option('--read_1_seqs_filename', type='string',
        help='Name of fasta file containing the first read from paired-end '
        'sequencing, prior to OTU clustering (used for RTAX only). Must exist '
        'under each input dataset directory if rtax is specified as an '
        'assignment method [default: %default]', default='seqs1.fna'),
    make_option('--read_2_seqs_filename', type='string',
        help='Name of fasta file containing a second read from paired-end '
        'sequencing, prior to OTU clustering (used for RTAX only). Must exist '
        'under each input dataset directory if rtax is specified as an '
        'assignment method and the "paired" rtax mode is specified '
        '[default: %default]', default='seqs2.fna'),
    make_option('--rtax_read_id_regexes', type='string',
        help='Comma-separated list of regular expressions that will be passed '
        'to RTAX for parsing read 1 IDs. The number (and order) of the '
        'supplied regexes must match the number (and order) of input datasets '
        'supplied via -i/--input_dirs [default: %default]', default=None),
    make_option('--rtax_amplicon_id_regexes', type='string',
        help='Comma-separated list of regular expressions that will be passed '
        'to RTAX for parsing amplicon IDs. The number (and order) of the '
        'supplied regexes must match the number (and order) of input datasets '
        'supplied via -i/--input_dirs [default: %default]', default=None),
    make_option('--rtax_header_id_regexes', type='string',
        help='Comma-separated list of regular expressions that will be passed '
        'to RTAX for parsing header IDs. The number (and order) of the '
        'supplied regexes must match the number (and order) of input datasets '
        'supplied via -i/--input_dirs [default: %default]', default=None),
    make_option('--rdp_max_memory', type='string',
        help='Maximum memory allocation, in MB, for JVM when using the rdp '
        'method. Increase for large training sets [default: %default]',
        default=4000),
    make_option('-w', '--print_only', action='store_true',
        help='Print the commands but don\'t call them -- useful for debugging '
        '[default: %default]', default=False),
    make_option('-f', '--force', action='store_true',
        help='Force overwrite of existing output directory (note: existing '
        'files in output_dir will not be removed). Subdirectories that '
        'already exist will be skipped (they are assumed to have been '
        'successfully created during a previous run of the script) '
        '[default: %default]', default=False)
]
script_info['version'] = __version__

def main():
    option_parser, opts, args = parse_command_line_parameters(**script_info)

    input_dirs = opts.input_dirs.split(',')
    assignment_methods = opts.assignment_methods.split(',')

    confidences = opts.confidences
    if confidences is not None:
        confidences = map(float, opts.confidences.split(','))

    e_values = opts.e_values
    if e_values is not None:
        e_values = map(float, opts.e_values.split(','))

    rtax_modes = opts.rtax_modes
    if rtax_modes is not None:
        rtax_modes = opts.rtax_modes.split(',')

    rtax_read_id_regexes = opts.rtax_read_id_regexes
    if rtax_read_id_regexes is not None:
        rtax_read_id_regexes = rtax_read_id_regexes.split(',')

    rtax_amplicon_id_regexes = opts.rtax_amplicon_id_regexes
    if rtax_amplicon_id_regexes is not None:
        rtax_amplicon_id_regexes = rtax_amplicon_id_regexes.split(',')

    rtax_header_id_regexes = opts.rtax_header_id_regexes
    if rtax_header_id_regexes is not None:
        rtax_header_id_regexes = rtax_header_id_regexes.split(',')

    if opts.print_only:
        command_handler = print_commands
    else:
        command_handler = call_commands_serially

    if opts.verbose:
        status_update_callback = print_to_stdout
    else:
        status_update_callback = no_status_updates

    assign_taxonomy_multiple_times(input_dirs, opts.output_dir,
        assignment_methods, opts.reference_seqs_fp,
        opts.id_to_taxonomy_fp, confidences=confidences,
        e_values=e_values, rtax_modes=rtax_modes,
        input_fasta_filename=opts.input_fasta_filename,
        clean_otu_table_filename=opts.clean_otu_table_filename,
        read_1_seqs_filename=opts.read_1_seqs_filename,
        read_2_seqs_filename=opts.read_2_seqs_filename,
        rtax_read_id_regexes=rtax_read_id_regexes,
        rtax_amplicon_id_regexes=rtax_amplicon_id_regexes,
        rtax_header_id_regexes=rtax_header_id_regexes,
        rdp_max_memory=opts.rdp_max_memory,
        command_handler=command_handler,
        status_update_callback=status_update_callback, force=opts.force)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from __future__ import division
import nibabel as nib
import csv
import numpy as np
import json
import math
from stat159lambda.config import REPO_HOME_PATH
from stat159lambda.config import NUM_OFFSET_VOLUMES, NUM_VOLUMES

INTEGER_LABELS = {'day-night': {'DAY': 0,
                                'NIGHT': 1,
                                'DAWN': 2},
                  'int-ext': {'INT': 0,
                              'EXT': 1}}

TUNING_SECONDS_OFFSET = 17

class SceneSlicer:
    def __init__(
            self,
            path_to_scene_csv='{0}/data/scenes.csv'.format(REPO_HOME_PATH)):
        self.path_to_scene_csv = path_to_scene_csv
        self.scene_slices = []
        self.scene_desc = {}

    def generate_scene_desc_dict_(self):
        with open(self.path_to_scene_csv, 'rt') as csvfile:
            reader = csv.DictReader(
                csvfile,
                fieldnames=['seconds', 'scene', 'day-night', 'int-ext'])
            for row in reader:
                scene_time = int(float(row['seconds']))
                self.scene_desc[scene_time] = (
                    INTEGER_LABELS['day-night'][row['day-night']],
                    INTEGER_LABELS['int-ext'][row['int-ext']])

    def generate_scene_slices_(self):
        day_night = NUM_OFFSET_VOLUMES * [None]
        int_ext = NUM_OFFSET_VOLUMES * [None]
        current_scene_start_time = TUNING_SECONDS_OFFSET
        for i in range(NUM_OFFSET_VOLUMES, NUM_VOLUMES):
            if i * 2 in self.scene_desc:
                current_scene_start_time = i * 2
            elif i * 2 - 1 in self.scene_desc:
                current_scene_start_time = i * 2 - 1
            day_night.append(self.scene_desc[current_scene_start_time][0])
            int_ext.append(self.scene_desc[current_scene_start_time][1])
        self.scene_slices = (day_night, int_ext)

    def get_scene_slices(self):
        if not self.scene_desc:
            self.generate_scene_desc_dict_()
        if not self.scene_slices:
            self.generate_scene_slices_()
        return self.scene_slices

    def get_labels_by_slice(self, slice):
        if not self.scene_slices:
            self.get_scene_slices()
        day_label = self.scene_slices[0][slice]
        int_label = self.scene_slices[1][slice]
        return (day_label, int_label)
<commit_msg>Added documention to scene_slicer.py<commit_after>from __future__ import division
import nibabel as nib
import csv
import numpy as np
import json
import math
from stat159lambda.config import REPO_HOME_PATH
from stat159lambda.config import NUM_OFFSET_VOLUMES, NUM_VOLUMES

INTEGER_LABELS = {'day-night': {'DAY': 0,
                                'NIGHT': 1,
                                'DAWN': 2},
                  'int-ext': {'INT': 0,
                              'EXT': 1}}

TUNING_SECONDS_OFFSET = 17

class SceneSlicer:
    """
    SceneSlicer class used to hold information about each scene. Each
    SceneSlicer has the attributes:
    (1) path_to_scene_csv : string
    (2) scene_slices : array
    (3) scene_desc : dictionary

    Parameters
    ----------
    path_to_scene_csv : string
    Absolute path to scenes.csv
    """
    def __init__(
            self,
            path_to_scene_csv='{0}/data/scenes.csv'.format(REPO_HOME_PATH)):
        self.path_to_scene_csv = path_to_scene_csv
        self.scene_slices = []
        self.scene_desc = {}

    def generate_scene_desc_dict_(self):
        """
        SceneSlicer method that parses through scenes.csv file and updates
        each scene instance's description in regards to its length, whether
        it's day or night, and whether it takes place in the interior or
        exterior. Uses assigned INTEGER_LABELS to indicate these features.

        Parameters
        ----------
        None

        Returns
        -------
        None
        """
        with open(self.path_to_scene_csv, 'rt') as csvfile:
            reader = csv.DictReader(
                csvfile,
                fieldnames=['seconds', 'scene', 'day-night', 'int-ext'])
            for row in reader:
                scene_time = int(float(row['seconds']))
                self.scene_desc[scene_time] = (
                    INTEGER_LABELS['day-night'][row['day-night']],
                    INTEGER_LABELS['int-ext'][row['int-ext']])

    def generate_scene_slices_(self):
        """
        SceneSlicer method that actually generates the scene slices by
        adjusting for tuning seconds offset. Sets the scene_slices attribute,
        which is a tuple of two arrays, one for day vs. night and the other for
        interior vs. exterior.

        Parameters
        ----------
        None

        Returns
        -------
        None
        """
        day_night = NUM_OFFSET_VOLUMES * [None]
        int_ext = NUM_OFFSET_VOLUMES * [None]
        current_scene_start_time = TUNING_SECONDS_OFFSET
        for i in range(NUM_OFFSET_VOLUMES, NUM_VOLUMES):
            if i * 2 in self.scene_desc:
                current_scene_start_time = i * 2
            elif i * 2 - 1 in self.scene_desc:
                current_scene_start_time = i * 2 - 1
            day_night.append(self.scene_desc[current_scene_start_time][0])
            int_ext.append(self.scene_desc[current_scene_start_time][1])
        self.scene_slices = (day_night, int_ext)

    def get_scene_slices(self):
        """
        SceneSlicer method that returns the scene_slices attribute of that
        instance. Calls generate_scene_desc_dict() and/or
        generate_scene_slices_() if those attributes has not already been set.

        Parameters
        ----------
        None

        Returns
        -------
        scene_slices : tuple
        """
        if not self.scene_desc:
            self.generate_scene_desc_dict_()
        if not self.scene_slices:
            self.generate_scene_slices_()
        return self.scene_slices

    def get_labels_by_slice(self, slice):
        """
        SceneSlicer method that isolates a particular slice of the total
        scene_slices and returns data that indicates if that scene was taken
        in day/night and interior/exterior.

        Parameters
        ----------
        slice : int
        Indicates which slice of the scene to isolate

        Returns
        -------
        labels : tuple
        """
        if not self.scene_slices:
            self.get_scene_slices()
        day_label = self.scene_slices[0][slice]
        int_label = self.scene_slices[1][slice]
        return (day_label, int_label)
<|endoftext|>


<commit_before>import datetime
import dateutil

import pandas as pd
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.feature_extraction.text import TfidfVectorizer

import warnings

# The easiest way to check against a bunch of different bad values is to convert whatever val we have into a string, then check it against a set containing the string representation of a bunch of bad values
bad_vals_as_strings = set([str(float('nan')), str(float('inf')), str(float('-inf')), 'None', 'none', 'NaN', 'nan', 'NULL', 'null', '', 'inf', '-inf'])

# clean_val will try to turn a value into a float.
# If it fails, it will attempt to strip commas and then attempt to turn it into a float again
# Additionally, it will check to make sure the value is not in a set of bad vals (nan, None, inf, etc.)
# This function will either return a clean value, or raise an error if we cannot turn the value into a float or the value is a bad val
def clean_val(val):
    if str(val) in bad_vals_as_strings:
        raise(ValueError('clean_val failed'))
    else:
        try:
            float_val = float(val)
        except:
            # This will throw a ValueError if it fails
            # remove any commas in the string, and try to turn into a float again
            cleaned_string = val.replace(',', '')
            float_val = float(cleaned_string)
        return float_val

# Same as above, except this version returns float('nan') when it fails
# This plays more nicely with df.apply, and assumes we will be handling nans appropriately when doing DataFrameVectorizer later.
def clean_val_nan_version(val):
    if str(val) in bad_vals_as_strings:
        return float('nan')
    else:
        try:
            float_val = float(val)
        except:
            # This will throw a ValueError if it fails
            # remove any commas in the string, and try to turn into a float again
            cleaned_string = val.replace(',', '')
            try:
                float_val = float(cleaned_string)
            except:
                return float('nan')
        return float_val

class BasicDataCleaning(BaseEstimator, TransformerMixin):

    def __init__(self, column_descriptions=None):
        self.column_descriptions = column_descriptions
        self.text_col_indicators = set(['text', 'nlp'])

        self.text_columns = {}
        for key, val in self.column_descriptions.items():
            if val in self.text_col_indicators:
                self.text_columns[key] = TfidfVectorizer(
                    # If we have any documents that cannot be decoded properly, just ignore them and keep going as planned with everything else
                    decode_error='ignore'
                    # Try to strip accents from characters. Using unicode is slightly slower but more comprehensive than 'ascii'
                    , strip_accents='unicode'
                    # Can also choose 'character', which will likely increase accuracy, at the cost of much more space, generally
                    , analyzer='word'
                    # Remove commonly found english words ('it', 'a', 'the') which do not typically contain much signal
                    , stop_words='english'
                    # Convert all characters to lowercase
                    , lowercase=True
                    # Only consider words that appear in fewer than max_df percent of all documents
                    # In this case, ignore all words that appear in 90% of all documents
                    , max_df=0.9
                    # Consider only the most frequently occurring 3000 words, after taking into account all the other filtering going on
                    , max_features=3000
                )

    def fit(self, X_df, y=None):

        # See if we should fit TfidfVectorizer or not
        for key in X_df.columns:
            # col_desc = self.column_descriptions.get(key, False)
            if key in self.text_columns:
                    self.text_columns[key].fit(X_df[key].astype(str, raise_on_error=False))

        return self

    def transform(self, X, y=None):

        # Convert input to DataFrame if we were given a list of dictionaries
        if isinstance(X, list):
            X = pd.DataFrame(X)

        X = X.copy()

        # All of these are values we will not want to keep for training this particular estimator.
        # Note that we have already split out the output column and saved it into it's own variable
        vals_to_drop = set(['ignore', 'output', 'regressor', 'classifier'])

        # It is much more efficient to drop a bunch of columns at once, rather than one at a time
        cols_to_drop = []

        if isinstance(X, dict):

            dict_copy = {}

            for key, val in X.items():
                col_desc = self.column_descriptions.get(key)

                if col_desc in (None, 'continuous', 'numerical', 'float', 'int'):
                    dict_copy[key] = clean_val_nan_version(val)
                elif col_desc == 'date':
                    date_feature_dict = add_date_features_dict(X, key)
                    dict_copy.update(date_feature_dict)
                elif col_desc == 'categorical':
                    dict_copy[key] = val
                # elif key in self.text_columns:
                    # Add in logic to handle nlp columns here
                elif col_desc in vals_to_drop:
                    pass
                    # del X[key]
            return dict_copy

        else:
            X.reset_index(drop=True, inplace=True)
            for key in X.columns:
                col_desc = self.column_descriptions.get(key)
                if col_desc == 'categorical':
                    # We will handle categorical data later, one-hot-encoding it inside DataFrameVectorizer
                    pass

                elif col_desc in (None, 'continuous', 'numerical', 'float', 'int'):
                    # Make sure this isn't actually a date column, and if it is, give the user good information on how to fix this
                    is_unmarked_date_col = False
                    try:
                        pd.to_datetime(X[key])
                        is_unmarked_date_col = True
                    except:
                        pass
                    # For all of our numerical columns, try to turn all of these values into floats
                    # This function handles commas inside strings that represent numbers, and returns nan if we cannot turn this value into a float. nans are ignored in DataFrameVectorizer
                    try:
                        X[key] = X[key].apply(clean_val_nan_version)
                    except TypeError as e:
                        if is_unmarked_date_col == True:
                            print('\n\n\n')
                            print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
                            print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
                            print('We have found a column that is not marked as a "date" column, but can be converted to dates.')
                            print('The name of this column is:\n')
                            print(key)
                            print('\nThis column causes errors when we try to parse it as a numerical column.')
                            print('If it is in fact a date column, please add this to the column_descriptions hash:')
                            print('{' + key + ': date}')
                            warnings.warn('UnmarkedDateColumn: Please mark the ' + key + ' column as being a date column in your column_descriptions dictionary.')
                            print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
                            print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n\n')
                        raise(e)

                elif col_desc == 'date':
                    X = add_date_features_df(X, key)

                elif key in self.text_columns:

                    col_names = self.text_columns[key].get_feature_names()

                    # Make weird characters play nice, or just ignore them :)
                    for idx, word in enumerate(col_names):
                        try:
                            col_names[idx] = str(word)
                        except:
                            col_names[idx] = 'non_ascii_word_' + str(idx)

                    col_names = ['nlp_' + key + '_' + str(word) for word in col_names]

                    nlp_matrix = self.text_columns[key].transform(X[key].astype(str, raise_on_error=False))
                    nlp_matrix = nlp_matrix.toarray()

                    text_df = pd.DataFrame(nlp_matrix)
                    text_df.columns = col_names

                    X = X.join(text_df)
                    # Once the transformed datafrane is added, remove the original text

                    X = X.drop(key, axis=1)

                elif col_desc in vals_to_drop:
                    cols_to_drop.append(key)

                else:
                    # If we have gotten here, the value is not any that we recognize
                    # This is most likely a typo that the user would want to be informed of, or a case while we're developing on auto_ml itself.
                    # In either case, it's useful to log it.
                    print('When transforming the data, we have encountered a value in column_descriptions that is not currently supported. The column has been dropped to allow the rest of the pipeline to run. Here\'s the name of the column:' )
                    print(key)
                    print('And here is the value for this column passed into column_descriptions:')
                    print(col_desc)
                    warnings.warn('UnknownValueInColumnDescriptions: Please make sure all the values you pass into column_descriptions are valid.')

        # Historically we've deleted columns here. However, we're moving this to DataFrameVectorizer as part of a broader effort to reduce duplicate computation
        # if len(cols_to_drop) > 0:
        #     X = X.drop(cols_to_drop, axis=1)
        return X

def minutes_into_day_parts(minutes_into_day):
    if minutes_into_day < 6 * 60:
        return 'late_night'
    elif minutes_into_day < 10 * 60:
        return 'morning'
    elif minutes_into_day < 11.5 * 60:
        return 'mid_morning'
    elif minutes_into_day < 14 * 60:
        return 'lunchtime'
    elif minutes_into_day < 18 * 60:
        return 'afternoon'
    elif minutes_into_day < 20.5 * 60:
        return 'dinnertime'
    elif minutes_into_day < 23.5 * 60:
        return 'early_night'
    else:
        return 'late_night'

# Note: assumes that the column is already formatted as a pandas date type
def add_date_features_df(df, date_col):
    # Pandas nicely tries to prevent you from doing stupid things, like setting values on a copy of a df, not your real one
    # However, it's a bit overzealous in this case, so we'll side-step a bunch of warnings by setting is_copy to false here
    df.is_copy = False

    df[date_col] = pd.to_datetime(df[date_col])
    df[date_col + '_day_of_week'] = df[date_col].apply(lambda x: x.weekday()).astype(int, raise_on_error=False)

    try:
        df[date_col + '_hour'] = df[date_col].apply(lambda x: x.hour).astype(int, raise_on_error=False)

        df[date_col + '_minutes_into_day'] = df[date_col].apply(lambda x: x.hour * 60 + x.minute)
    except AttributeError:
        pass

    df[date_col + '_is_weekend'] = df[date_col].apply(lambda x: x.weekday() in (5,6))
    df[date_col + '_day_part'] = df[date_col + '_minutes_into_day'].apply(minutes_into_day_parts)

    df = df.drop([date_col], axis=1)

    return df

# Same logic as above, except implemented for a single dictionary, which is much faster at prediction time when getting just a single prediction
def add_date_features_dict(row, date_col):

    date_feature_dict = {}

    # Handle cases where the val for the date_col is None
    try:
        date_val = row[date_col]
        if date_val == None:
            return date_feature_dict
        if not isinstance(date_val, (datetime.datetime, datetime.date)):
            date_val = dateutil.parser.parse(date_val)
    except:
        return date_feature_dict

    # Make a copy of all the engineered features from the date, without modifying the original object at all
    # This way the same original object can be passed into a number of different trained auto_ml predictors

    date_feature_dict[date_col + '_day_of_week'] = date_val.weekday()
    # nesting this inside a try/except block because the date might be a datetime.date, not a datetime.datetime
    try:
        date_feature_dict[date_col + '_hour'] = date_val.hour

        date_feature_dict[date_col + '_minutes_into_day'] = date_val.hour * 60 + date_val.minute
    except AttributeError:
        pass

    date_feature_dict[date_col + '_is_weekend'] = date_val.weekday() in (5,6)

    # del row[date_col]

    return date_feature_dict

<commit_msg>safer parsing of non-parseable values<commit_after>import datetime
import dateutil

import pandas as pd
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.feature_extraction.text import TfidfVectorizer

import warnings

# The easiest way to check against a bunch of different bad values is to convert whatever val we have into a string, then check it against a set containing the string representation of a bunch of bad values
bad_vals_as_strings = set([str(float('nan')), str(float('inf')), str(float('-inf')), 'None', 'none', 'NaN', 'nan', 'NULL', 'null', '', 'inf', '-inf'])

# clean_val will try to turn a value into a float.
# If it fails, it will attempt to strip commas and then attempt to turn it into a float again
# Additionally, it will check to make sure the value is not in a set of bad vals (nan, None, inf, etc.)
# This function will either return a clean value, or raise an error if we cannot turn the value into a float or the value is a bad val
def clean_val(val):
    if str(val) in bad_vals_as_strings:
        raise(ValueError('clean_val failed'))
    else:
        try:
            float_val = float(val)
        except ValueError:
            # This will throw a ValueError if it fails
            # remove any commas in the string, and try to turn into a float again
            try:
                cleaned_string = val.replace(',', '')
                float_val = float(cleaned_string)
            except TypeError:
                return None
        return float_val

# Same as above, except this version returns float('nan') when it fails
# This plays more nicely with df.apply, and assumes we will be handling nans appropriately when doing DataFrameVectorizer later.
def clean_val_nan_version(val):
    if str(val) in bad_vals_as_strings:
        return float('nan')
    else:
        try:
            float_val = float(val)
        except ValueError:
            # This will throw a ValueError if it fails
            # remove any commas in the string, and try to turn into a float again
            try:
                cleaned_string = val.replace(',', '')
            except TypeError:
                print('*************************************')
                print('We expected this value to be numeric, but were unable to convert it to a float:')
                print(val)
                print('*************************************')
                return float('nan')
            try:
                float_val = float(cleaned_string)
            except:
                return float('nan')
        return float_val

class BasicDataCleaning(BaseEstimator, TransformerMixin):

    def __init__(self, column_descriptions=None):
        self.column_descriptions = column_descriptions
        self.text_col_indicators = set(['text', 'nlp'])

        self.text_columns = {}
        for key, val in self.column_descriptions.items():
            if val in self.text_col_indicators:
                self.text_columns[key] = TfidfVectorizer(
                    # If we have any documents that cannot be decoded properly, just ignore them and keep going as planned with everything else
                    decode_error='ignore'
                    # Try to strip accents from characters. Using unicode is slightly slower but more comprehensive than 'ascii'
                    , strip_accents='unicode'
                    # Can also choose 'character', which will likely increase accuracy, at the cost of much more space, generally
                    , analyzer='word'
                    # Remove commonly found english words ('it', 'a', 'the') which do not typically contain much signal
                    , stop_words='english'
                    # Convert all characters to lowercase
                    , lowercase=True
                    # Only consider words that appear in fewer than max_df percent of all documents
                    # In this case, ignore all words that appear in 90% of all documents
                    , max_df=0.9
                    # Consider only the most frequently occurring 3000 words, after taking into account all the other filtering going on
                    , max_features=3000
                )

    def fit(self, X_df, y=None):

        # See if we should fit TfidfVectorizer or not
        for key in X_df.columns:
            # col_desc = self.column_descriptions.get(key, False)
            if key in self.text_columns:
                    self.text_columns[key].fit(X_df[key].astype(str, raise_on_error=False))

        return self

    def transform(self, X, y=None):

        # Convert input to DataFrame if we were given a list of dictionaries
        if isinstance(X, list):
            X = pd.DataFrame(X)

        X = X.copy()

        # All of these are values we will not want to keep for training this particular estimator.
        # Note that we have already split out the output column and saved it into it's own variable
        vals_to_drop = set(['ignore', 'output', 'regressor', 'classifier'])

        # It is much more efficient to drop a bunch of columns at once, rather than one at a time
        cols_to_drop = []

        if isinstance(X, dict):

            dict_copy = {}

            for key, val in X.items():
                col_desc = self.column_descriptions.get(key)

                if col_desc in (None, 'continuous', 'numerical', 'float', 'int'):
                    dict_copy[key] = clean_val_nan_version(val)
                elif col_desc == 'date':
                    date_feature_dict = add_date_features_dict(X, key)
                    dict_copy.update(date_feature_dict)
                elif col_desc == 'categorical':
                    dict_copy[key] = val
                # elif key in self.text_columns:
                    # Add in logic to handle nlp columns here
                elif col_desc in vals_to_drop:
                    pass
                    # del X[key]
            return dict_copy

        else:
            X.reset_index(drop=True, inplace=True)
            for key in X.columns:
                col_desc = self.column_descriptions.get(key)
                if col_desc == 'categorical':
                    # We will handle categorical data later, one-hot-encoding it inside DataFrameVectorizer
                    pass

                elif col_desc in (None, 'continuous', 'numerical', 'float', 'int'):
                    # Make sure this isn't actually a date column, and if it is, give the user good information on how to fix this
                    is_unmarked_date_col = False
                    try:
                        pd.to_datetime(X[key])
                        is_unmarked_date_col = True
                    except:
                        pass
                    # For all of our numerical columns, try to turn all of these values into floats
                    # This function handles commas inside strings that represent numbers, and returns nan if we cannot turn this value into a float. nans are ignored in DataFrameVectorizer
                    try:
                        X[key] = X[key].apply(clean_val_nan_version)
                    except TypeError as e:
                        if is_unmarked_date_col == True:
                            print('\n\n\n')
                            print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
                            print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
                            print('We have found a column that is not marked as a "date" column, but can be converted to dates.')
                            print('The name of this column is:\n')
                            print(key)
                            print('\nThis column causes errors when we try to parse it as a numerical column.')
                            print('If it is in fact a date column, please add this to the column_descriptions hash:')
                            print('{' + key + ': date}')
                            warnings.warn('UnmarkedDateColumn: Please mark the ' + key + ' column as being a date column in your column_descriptions dictionary.')
                            print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
                            print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n\n')
                        raise(e)

                elif col_desc == 'date':
                    X = add_date_features_df(X, key)

                elif key in self.text_columns:

                    col_names = self.text_columns[key].get_feature_names()

                    # Make weird characters play nice, or just ignore them :)
                    for idx, word in enumerate(col_names):
                        try:
                            col_names[idx] = str(word)
                        except:
                            col_names[idx] = 'non_ascii_word_' + str(idx)

                    col_names = ['nlp_' + key + '_' + str(word) for word in col_names]

                    nlp_matrix = self.text_columns[key].transform(X[key].astype(str, raise_on_error=False))
                    nlp_matrix = nlp_matrix.toarray()

                    text_df = pd.DataFrame(nlp_matrix)
                    text_df.columns = col_names

                    X = X.join(text_df)
                    # Once the transformed datafrane is added, remove the original text

                    X = X.drop(key, axis=1)

                elif col_desc in vals_to_drop:
                    cols_to_drop.append(key)

                else:
                    # If we have gotten here, the value is not any that we recognize
                    # This is most likely a typo that the user would want to be informed of, or a case while we're developing on auto_ml itself.
                    # In either case, it's useful to log it.
                    print('When transforming the data, we have encountered a value in column_descriptions that is not currently supported. The column has been dropped to allow the rest of the pipeline to run. Here\'s the name of the column:' )
                    print(key)
                    print('And here is the value for this column passed into column_descriptions:')
                    print(col_desc)
                    warnings.warn('UnknownValueInColumnDescriptions: Please make sure all the values you pass into column_descriptions are valid.')

        # Historically we've deleted columns here. However, we're moving this to DataFrameVectorizer as part of a broader effort to reduce duplicate computation
        # if len(cols_to_drop) > 0:
        #     X = X.drop(cols_to_drop, axis=1)
        return X

def minutes_into_day_parts(minutes_into_day):
    if minutes_into_day < 6 * 60:
        return 'late_night'
    elif minutes_into_day < 10 * 60:
        return 'morning'
    elif minutes_into_day < 11.5 * 60:
        return 'mid_morning'
    elif minutes_into_day < 14 * 60:
        return 'lunchtime'
    elif minutes_into_day < 18 * 60:
        return 'afternoon'
    elif minutes_into_day < 20.5 * 60:
        return 'dinnertime'
    elif minutes_into_day < 23.5 * 60:
        return 'early_night'
    else:
        return 'late_night'

# Note: assumes that the column is already formatted as a pandas date type
def add_date_features_df(df, date_col):
    # Pandas nicely tries to prevent you from doing stupid things, like setting values on a copy of a df, not your real one
    # However, it's a bit overzealous in this case, so we'll side-step a bunch of warnings by setting is_copy to false here
    df.is_copy = False

    df[date_col] = pd.to_datetime(df[date_col])
    df[date_col + '_day_of_week'] = df[date_col].apply(lambda x: x.weekday()).astype(int, raise_on_error=False)

    try:
        df[date_col + '_hour'] = df[date_col].apply(lambda x: x.hour).astype(int, raise_on_error=False)

        df[date_col + '_minutes_into_day'] = df[date_col].apply(lambda x: x.hour * 60 + x.minute)
    except AttributeError:
        pass

    df[date_col + '_is_weekend'] = df[date_col].apply(lambda x: x.weekday() in (5,6))
    df[date_col + '_day_part'] = df[date_col + '_minutes_into_day'].apply(minutes_into_day_parts)

    df = df.drop([date_col], axis=1)

    return df

# Same logic as above, except implemented for a single dictionary, which is much faster at prediction time when getting just a single prediction
def add_date_features_dict(row, date_col):

    date_feature_dict = {}

    # Handle cases where the val for the date_col is None
    try:
        date_val = row[date_col]
        if date_val == None:
            return date_feature_dict
        if not isinstance(date_val, (datetime.datetime, datetime.date)):
            date_val = dateutil.parser.parse(date_val)
    except:
        return date_feature_dict

    # Make a copy of all the engineered features from the date, without modifying the original object at all
    # This way the same original object can be passed into a number of different trained auto_ml predictors

    date_feature_dict[date_col + '_day_of_week'] = date_val.weekday()
    # nesting this inside a try/except block because the date might be a datetime.date, not a datetime.datetime
    try:
        date_feature_dict[date_col + '_hour'] = date_val.hour

        date_feature_dict[date_col + '_minutes_into_day'] = date_val.hour * 60 + date_val.minute
    except AttributeError:
        pass

    date_feature_dict[date_col + '_is_weekend'] = date_val.weekday() in (5,6)

    # del row[date_col]

    return date_feature_dict

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from datetime import datetime
import itertools
from fabric.api import env, run, runs_once, sudo, task
from fabric.utils import puts, fastprint
import six

from dockermap.utils import expand_path
from . import cli
from .api import docker_fabric
from .utils.net import get_ip4_address, get_ip6_address
from .utils.output import stdout_result

IMAGE_COLUMNS = ('Id', 'RepoTags', 'ParentId', 'Created', 'VirtualSize', 'Size')
CONTAINER_COLUMNS = ('Id', 'Names', 'Image', 'Command', 'Ports', 'Status', 'Created')
NETWORK_COLUMNS = ('Id', 'Name', 'Driver', 'Scope')

def _format_output_table(data_dict, columns, full_ids=False, full_cmd=False, short_image=False):
    def _format_port(port_dict):
        if 'PublicPort' in port_dict and 'IP' in port_dict:
            return '{IP}:{PublicPort}->{PrivatePort}/{Type}'.format(**port_dict)
        return '{PrivatePort}/{Type}'.format(**port_dict)

    def _get_column(item, column):
        data = item.get(column, '')
        if isinstance(data, list):
            if column == 'Ports':
                return map(_format_port, data)
            return data
        if column in ('Id', 'ParentId') and not full_ids:
            return data[:12],
        if column == 'Created':
            return datetime.utcfromtimestamp(data).isoformat(),
        if column == 'Command' and not full_cmd:
            return data[:25],
        if column == 'Image' and short_image:
            __, __, i_name = data.rpartition('/')
            return i_name,
        return unicode(data),

    def _max_len(col_data):
        if col_data:
            return max(map(len, col_data))
        return 0

    puts('')
    rows = [[[c] for c in columns]]
    rows.extend([_get_column(i, col) for col in columns] for i in data_dict)
    col_lens = map(max, (map(_max_len, c) for c in zip(*rows)))
    row_format = '  '.join('{{{0}:{1}}}'.format(i, l) for i, l in enumerate(col_lens))
    for row in rows:
        for c in itertools.izip_longest(*row, fillvalue=''):
            fastprint(row_format.format(*c), end='\n', flush=False)
    fastprint('', flush=True)

@task
def reset_socat(use_sudo=False):
    """
    Finds and closes all processes of `socat`.

    :param use_sudo: Use `sudo` command. As Docker-Fabric does not run `socat` with `sudo`, this is by default set to
      ``False``. Setting it to ``True`` could unintentionally remove instances from other users.
    :type use_sudo: bool
    """
    output = stdout_result('ps -o pid -C socat', quiet=True)
    pids = output.split('\n')[1:]
    puts("Removing process(es) with id(s) {0}.".format(', '.join(pids)))
    which = sudo if use_sudo else run
    which('kill {0}'.format(' '.join(pids)), quiet=True)

@task
def version():
    """
    Shows version information of the remote Docker service, similar to ``docker version``.
    """
    output = docker_fabric().version()
    col_len = max(map(len, output.keys())) + 1
    puts('')
    for k, v in six.iteritems(output):
        fastprint('{0:{1}} {2}'.format(''.join((k, ':')), col_len, v), end='\n', flush=False)
    fastprint('', flush=True)

@task
def get_ip(interface_name='docker0'):
    """
    Shows the IP4 address of a network interface.

    :param interface_name: Name of the network interface. Default is ``docker0``.
    :type interface_name: unicode
    """
    puts(get_ip4_address(interface_name))

@task
def get_ipv6(interface_name='docker0', expand=False):
    """
    Shows the IP6 address of a network interface.

    :param interface_name: Name of the network interface. Default is ``docker0``.
    :type interface_name: unicode
    :param expand: Expand the abbreviated IP6 address. Default is ``False``.
    :type expand: bool
    """
    puts(get_ip6_address(interface_name, expand=expand))

@task
def list_images(list_all=False, full_ids=False):
    """
    Lists images on the Docker remote host, similar to ``docker images``.

    :param list_all: Lists all images (e.g. dependencies). Default is ``False``, only shows named images.
    :type list_all: bool
    :param full_ids: Shows the full ids. When ``False`` (default) only shows the first 12 characters.
    :type full_ids: bool
    """
    images = docker_fabric().images(all=list_all)
    _format_output_table(images, IMAGE_COLUMNS, full_ids)

@task
def list_containers(list_all=True, short_image=True, full_ids=False, full_cmd=False):
    """
    Lists containers on the Docker remote host, similar to ``docker ps``.

    :param list_all: Shows all containers. Default is ``False``, which omits exited containers.
    :type list_all: bool
    :param short_image: Hides the repository prefix for preserving space. Default is ``True``.
    :type short_image: bool
    :param full_ids: Shows the full image ids. When ``False`` (default) only shows the first 12 characters.
    :type full_ids: bool
    :param full_cmd: Shows the full container command. When ``False`` (default) only shows the first 25 characters.
    :type full_cmd: bool
    """
    containers = docker_fabric().containers(all=list_all)
    _format_output_table(containers, CONTAINER_COLUMNS, full_ids, full_cmd, short_image)

@task
def list_networks(full_ids=False):
    """
    Lists networks on the Docker remote host, similar to ``docker network ls``.

    :param full_ids: Shows the full network ids. When ``False`` (default) only shows the first 12 characters.
    :type full_ids: bool
    """
    networks = docker_fabric().networks()
    _format_output_table(networks, NETWORK_COLUMNS, full_ids)

@task
def cleanup_containers(**kwargs):
    """
    Removes all containers that have finished running. Similar to the ``prune`` functionality in newer Docker versions.
    """
    containers = docker_fabric().cleanup_containers(**kwargs)
    if kwargs.get('list_only'):
        puts('Existing containers:')
        for c_id, c_name in containers:
            fastprint('{0}  {1}'.format(c_id, c_name), end='\n')

@task
def cleanup_images(remove_old=False, **kwargs):
    """
    Removes all images that have no name, and that are not references as dependency by any other named image. Similar
    to the ``prune`` functionality in newer Docker versions, but supports more filters.

    :param remove_old: Also remove images that do have a name, but no `latest` tag.
    :type remove_old: bool
    """
    keep_tags = env.get('docker_keep_tags')
    if keep_tags is not None:
        kwargs.setdefault('keep_tags', keep_tags)
    removed_images = docker_fabric().cleanup_images(remove_old=remove_old, **kwargs)
    if kwargs.get('list_only'):
        puts('Unused images:')
        for image_name in removed_images:
            fastprint(image_name, end='\n')

@task
def remove_all_containers(**kwargs):
    """
    Stops and removes all containers from the remote. Use with caution outside of a development environment!
    :return:
    """
    containers = docker_fabric().remove_all_containers(**kwargs)
    if kwargs.get('list_only'):
        puts('Existing containers:')
        for c_id in containers[1]:
            fastprint(c_id, end='\n')

@task
@runs_once
def save_image(image, filename=None):
    """
    Saves a Docker image from the remote to a local files. For performance reasons, uses the Docker command line client
    on the host, generates a gzip-tarball and downloads that.

    :param image: Image name or id.
    :type image: unicode
    :param filename: File name to store the local file. If not provided, will use ``<image>.tar.gz`` in the current
      working directory.
    :type filename: unicode
    """
    local_name = filename or '{0}.tar.gz'.format(image)
    cli.save_image(image, local_name)

@task
def load_image(filename, timeout=120):
    """
    Uploads an image from a local file to a Docker remote. Note that this temporarily has to extend the service timeout
    period.

    :param filename: Local file name.
    :type filename: unicode
    :param timeout: Timeout in seconds to set temporarily for the upload.
    :type timeout: int
    """
    c = docker_fabric()
    with open(expand_path(filename), 'r') as f:
        _timeout = c._timeout
        c._timeout = timeout
        try:
            c.load_image(f)
        finally:
            c._timeout = _timeout
<commit_msg>Added volume list task.<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from datetime import datetime
import itertools
from fabric.api import env, run, runs_once, sudo, task
from fabric.utils import puts, fastprint
import six

from dockermap.utils import expand_path
from . import cli
from .api import docker_fabric
from .utils.net import get_ip4_address, get_ip6_address
from .utils.output import stdout_result

IMAGE_COLUMNS = ('Id', 'RepoTags', 'ParentId', 'Created', 'VirtualSize', 'Size')
CONTAINER_COLUMNS = ('Id', 'Names', 'Image', 'Command', 'Ports', 'Status', 'Created')
NETWORK_COLUMNS = ('Id', 'Name', 'Driver', 'Scope')
VOLUME_COLUMNS = ('Name', 'Driver')

def _format_output_table(data_dict, columns, full_ids=False, full_cmd=False, short_image=False):
    def _format_port(port_dict):
        if 'PublicPort' in port_dict and 'IP' in port_dict:
            return '{IP}:{PublicPort}->{PrivatePort}/{Type}'.format(**port_dict)
        return '{PrivatePort}/{Type}'.format(**port_dict)

    def _get_column(item, column):
        data = item.get(column, '')
        if isinstance(data, list):
            if column == 'Ports':
                return map(_format_port, data)
            return data
        if column in ('Id', 'ParentId') and not full_ids:
            return data[:12],
        if column == 'Created':
            return datetime.utcfromtimestamp(data).isoformat(),
        if column == 'Command' and not full_cmd:
            return data[:25],
        if column == 'Image' and short_image:
            __, __, i_name = data.rpartition('/')
            return i_name,
        return unicode(data),

    def _max_len(col_data):
        if col_data:
            return max(map(len, col_data))
        return 0

    puts('')
    rows = [[[c] for c in columns]]
    rows.extend([_get_column(i, col) for col in columns] for i in data_dict)
    col_lens = map(max, (map(_max_len, c) for c in zip(*rows)))
    row_format = '  '.join('{{{0}:{1}}}'.format(i, l) for i, l in enumerate(col_lens))
    for row in rows:
        for c in itertools.izip_longest(*row, fillvalue=''):
            fastprint(row_format.format(*c), end='\n', flush=False)
    fastprint('', flush=True)

@task
def reset_socat(use_sudo=False):
    """
    Finds and closes all processes of `socat`.

    :param use_sudo: Use `sudo` command. As Docker-Fabric does not run `socat` with `sudo`, this is by default set to
      ``False``. Setting it to ``True`` could unintentionally remove instances from other users.
    :type use_sudo: bool
    """
    output = stdout_result('ps -o pid -C socat', quiet=True)
    pids = output.split('\n')[1:]
    puts("Removing process(es) with id(s) {0}.".format(', '.join(pids)))
    which = sudo if use_sudo else run
    which('kill {0}'.format(' '.join(pids)), quiet=True)

@task
def version():
    """
    Shows version information of the remote Docker service, similar to ``docker version``.
    """
    output = docker_fabric().version()
    col_len = max(map(len, output.keys())) + 1
    puts('')
    for k, v in six.iteritems(output):
        fastprint('{0:{1}} {2}'.format(''.join((k, ':')), col_len, v), end='\n', flush=False)
    fastprint('', flush=True)

@task
def get_ip(interface_name='docker0'):
    """
    Shows the IP4 address of a network interface.

    :param interface_name: Name of the network interface. Default is ``docker0``.
    :type interface_name: unicode
    """
    puts(get_ip4_address(interface_name))

@task
def get_ipv6(interface_name='docker0', expand=False):
    """
    Shows the IP6 address of a network interface.

    :param interface_name: Name of the network interface. Default is ``docker0``.
    :type interface_name: unicode
    :param expand: Expand the abbreviated IP6 address. Default is ``False``.
    :type expand: bool
    """
    puts(get_ip6_address(interface_name, expand=expand))

@task
def list_images(list_all=False, full_ids=False):
    """
    Lists images on the Docker remote host, similar to ``docker images``.

    :param list_all: Lists all images (e.g. dependencies). Default is ``False``, only shows named images.
    :type list_all: bool
    :param full_ids: Shows the full ids. When ``False`` (default) only shows the first 12 characters.
    :type full_ids: bool
    """
    images = docker_fabric().images(all=list_all)
    _format_output_table(images, IMAGE_COLUMNS, full_ids)

@task
def list_containers(list_all=True, short_image=True, full_ids=False, full_cmd=False):
    """
    Lists containers on the Docker remote host, similar to ``docker ps``.

    :param list_all: Shows all containers. Default is ``False``, which omits exited containers.
    :type list_all: bool
    :param short_image: Hides the repository prefix for preserving space. Default is ``True``.
    :type short_image: bool
    :param full_ids: Shows the full image ids. When ``False`` (default) only shows the first 12 characters.
    :type full_ids: bool
    :param full_cmd: Shows the full container command. When ``False`` (default) only shows the first 25 characters.
    :type full_cmd: bool
    """
    containers = docker_fabric().containers(all=list_all)
    _format_output_table(containers, CONTAINER_COLUMNS, full_ids, full_cmd, short_image)

@task
def list_networks(full_ids=False):
    """
    Lists networks on the Docker remote host, similar to ``docker network ls``.

    :param full_ids: Shows the full network ids. When ``False`` (default) only shows the first 12 characters.
    :type full_ids: bool
    """
    networks = docker_fabric().networks()
    _format_output_table(networks, NETWORK_COLUMNS, full_ids)

@task
def list_volumes():
    """
    Lists volumes on the Docker remote host, similar to ``docker volume ls``.
    """
    volumes = docker_fabric().volumes()
    _format_output_table(volumes, VOLUME_COLUMNS)

@task
def cleanup_containers(**kwargs):
    """
    Removes all containers that have finished running. Similar to the ``prune`` functionality in newer Docker versions.
    """
    containers = docker_fabric().cleanup_containers(**kwargs)
    if kwargs.get('list_only'):
        puts('Existing containers:')
        for c_id, c_name in containers:
            fastprint('{0}  {1}'.format(c_id, c_name), end='\n')

@task
def cleanup_images(remove_old=False, **kwargs):
    """
    Removes all images that have no name, and that are not references as dependency by any other named image. Similar
    to the ``prune`` functionality in newer Docker versions, but supports more filters.

    :param remove_old: Also remove images that do have a name, but no `latest` tag.
    :type remove_old: bool
    """
    keep_tags = env.get('docker_keep_tags')
    if keep_tags is not None:
        kwargs.setdefault('keep_tags', keep_tags)
    removed_images = docker_fabric().cleanup_images(remove_old=remove_old, **kwargs)
    if kwargs.get('list_only'):
        puts('Unused images:')
        for image_name in removed_images:
            fastprint(image_name, end='\n')

@task
def remove_all_containers(**kwargs):
    """
    Stops and removes all containers from the remote. Use with caution outside of a development environment!
    :return:
    """
    containers = docker_fabric().remove_all_containers(**kwargs)
    if kwargs.get('list_only'):
        puts('Existing containers:')
        for c_id in containers[1]:
            fastprint(c_id, end='\n')

@task
@runs_once
def save_image(image, filename=None):
    """
    Saves a Docker image from the remote to a local files. For performance reasons, uses the Docker command line client
    on the host, generates a gzip-tarball and downloads that.

    :param image: Image name or id.
    :type image: unicode
    :param filename: File name to store the local file. If not provided, will use ``<image>.tar.gz`` in the current
      working directory.
    :type filename: unicode
    """
    local_name = filename or '{0}.tar.gz'.format(image)
    cli.save_image(image, local_name)

@task
def load_image(filename, timeout=120):
    """
    Uploads an image from a local file to a Docker remote. Note that this temporarily has to extend the service timeout
    period.

    :param filename: Local file name.
    :type filename: unicode
    :param timeout: Timeout in seconds to set temporarily for the upload.
    :type timeout: int
    """
    c = docker_fabric()
    with open(expand_path(filename), 'r') as f:
        _timeout = c._timeout
        c._timeout = timeout
        try:
            c.load_image(f)
        finally:
            c._timeout = _timeout
<|endoftext|>


<commit_before>"""
  APF Module to handle plugins dispatch, config.
  Not named 'plugins' because there is a lib directory with that name.

  Model is plugins are kept in hierarchy by 'category':
    factory: plugins that the factory uses
    queue:   plugins that APFQueues use
    profile:  plugins looked up by label

  Beneath the catetogory, there are types
    factory:
       config:
         file
         agis
       monitor:
         apfmon
    queue:
       wmsstatus:
         condor
         panda
       batchstatus
         condor
       batchsubmit
         condor
       sched
         <various>
    authmanager:
       auth:
         x509
         ssh
         cloud

Plugins initialized with ConfigParser object and a section name as arguments. The
assumption is that the classes themselves know how to look up the info they need.
It is also assumed that Singleton/non-Singleton decision is made during class initialization.
"""

import logging
import logging.handlers
import traceback

from pprint import pprint

class NullHandler(logging.Handler):
    """
    This handler does nothing. It's intended to be used to avoid the
    "No handlers could be found for logger XXX" one-off warning. This is
    important for library code, which may contain code to log events. If a user
    of the library does not configure logging, the one-off warning might be
    produced; to avoid this, the library developer simply needs to instantiate
    a NullHandler and add it to the top-level logger of the library module or
    package.
    """
    def handle(self, record):
        pass

    def emit(self, record):
        pass

    def createLock(self):
        self.lock = None

class PluginManager(object):
    """
    Entry point for plugins creation, initialization, starting, and configuration.
    """

    def __init__(self):
        """
        Top-level object to provide plugins.
        """
        self.log = logging.getLogger('autopyfactory')
        self.log.addHandler(NullHandler())
        self.log.debug('PluginManager initialized.')

    def getpluginlist(self, parent, paths, namelist, config, section):
        """
        Provides a list of initialized plugin objects.
        parent: reference to the calling object
        paths: list of subdirectories from where to import the plugin(s)
        namelist: list of plugins to be imported
        config: config parser object to feed to the plugin
        section: the section name in the config object that is relevant for this plugin
        """
        plist = []
        for name in namelist:
            po = self._getplugin(parent, paths, name, config, section)
            plist.append(po)
        self.log.info('returning a list of plugins = %s' %plist)
        return plist

    def getplugin(self, parent, paths, name, config, section):
        """
        Provides a single initialized plugin object.
        parent: reference to the calling object
        paths: list of subdirectories from where to import the plugin(s)
        name: name of the single plugin to be imported
        config: config parser object to feed to the plugin
        section: the section name in the config object that is relevant for this plugin
        """
        p = self._getplugin(parent, paths, name, config, section)
        self.log.info('returning a plugin = %s' %p)
        return p

    def _getplugin(self, parent, paths, name, config, section):
        """
        returns an initialized plugin object.
        parent: reference to the calling object
        paths: list of subdirectories from where to import the plugin(s)
        name: name of the single plugin to be imported
        config: config parser object to feed to the plugin
        section: the section name in the config object that is relevant for this plugin
        """
        self.log.debug('Starting')
        ko = self._getpluginclass(paths, name)
        po = ko(parent, config, section)
        self.log.debug('Leaving, returning %s' %po)
        return po

    def _getpluginclass(self, paths, name):
        """
        returns a plugin class.
        The __init__() methods have not been called yet.
        paths: list of subdirectories from where to import the plugin(s)
        name: name of the single plugin to be imported
        """
        self.log.debug('Starting')
        ppath = '.'.join(paths)
        ppath = ppath + '.' + name

        try:
            self.log.debug("trying to import %s from %s" %(name, ppath))
            plugin_module = __import__(ppath, globals(), locals(), name)
        except Exception, ex:
            self.log.error(ex)
            raise ex

        plugin_class = getattr(plugin_module, name)
        self.log.debug("Retrieved plugin with class name %s" % name)
        return plugin_class

<commit_msg>removed pluginmanager file<commit_after><|endoftext|>


<commit_before>import os

import lockfile
import numpy as np

from autosklearn.constants import *
from autosklearn.metrics import sanitize_array, \
    normalize_array, regression_metrics, classification_metrics

__all__ = [
    'calculate_score',
    'get_new_run_num'
]

def calculate_score(solution, prediction, task_type, metric, num_classes,
                    all_scoring_functions=False, logger=None):
    if task_type == MULTICLASS_CLASSIFICATION:
        # This used to crash on travis-ci; special treatment to find out why
        # it crashed!
        try:
            solution_binary = np.zeros((prediction.shape[0], num_classes))
        except IndexError as e:
            if logger is not None:
                logger.error(prediction.shape)
                raise e

        for i in range(solution_binary.shape[0]):
            label = solution[i]
            solution_binary[i, label] = 1
        solution = solution_binary

    elif task_type in [BINARY_CLASSIFICATION, REGRESSION]:
        if len(solution.shape) == 1:
            solution = solution.reshape((-1, 1))

    if task_type not in TASK_TYPES:
        raise NotImplementedError(task_type)

    if solution.shape != prediction.shape:
        raise ValueError('Solution shape %s != prediction shape %s' %
                         (solution.shape, prediction.shape))

    if all_scoring_functions:
        score = dict()
        if task_type in REGRESSION_TASKS:
            cprediction = sanitize_array(prediction)
            for metric_ in REGRESSION_METRIC:
                score[metric_] = regression_metrics.calculate_score(metric_,
                                                                    solution,
                                                                    cprediction)
        else:
            csolution, cprediction = normalize_array(solution, prediction)
            for metric_ in CLASSIFICATION_METRICS:
                score[metric_] = classification_metrics.calculate_score(
                    metric_, csolution, cprediction, task_type)

    else:
        if task_type in REGRESSION_TASKS:
            cprediction = sanitize_array(prediction)
            score = regression_metrics.calculate_score(metric,
                                                       solution,
                                                       cprediction)
        else:
            csolution, cprediction = normalize_array(solution, prediction)
            score = classification_metrics.calculate_score(metric,
                                                           csolution,
                                                           cprediction,
                                                           task=task_type)
    return score

def get_new_run_num(lockdir=None):
    seed = os.environ.get('AUTOSKLEARN_SEED')
    counter_file = 'num_run'
    if seed is not None:
        counter_file = counter_file + ('_%s' % seed)
    if lockdir is None:
        lockdir = os.getcwd()
    counter_file = os.path.join(lockdir, counter_file)
    lock = lockfile.LockFile(counter_file)
    with lock:
        if not os.path.exists(counter_file):
            with open(counter_file, 'w') as fh:
                fh.write('0')
            num = 0
        else:
            with open(counter_file, 'r') as fh:
                num = int(fh.read())
            num += 1
            with open(counter_file, 'w') as fh:
                fh.write(str(num).zfill(4))

    return num
<commit_msg>More output for previous commit<commit_after>import os

import lockfile
import numpy as np

from autosklearn.constants import *
from autosklearn.metrics import sanitize_array, \
    normalize_array, regression_metrics, classification_metrics

__all__ = [
    'calculate_score',
    'get_new_run_num'
]

def calculate_score(solution, prediction, task_type, metric, num_classes,
                    all_scoring_functions=False, logger=None):
    if task_type == MULTICLASS_CLASSIFICATION:
        # This used to crash on travis-ci; special treatment to find out why
        # it crashed!
        try:
            solution_binary = np.zeros((prediction.shape[0], num_classes))
        except IndexError as e:
            if logger is not None:
                logger.error("Prediction shape: %s, solution "
                             "shape %s" % prediction.shape, solution.shape)
                raise e

        for i in range(solution_binary.shape[0]):
            label = solution[i]
            solution_binary[i, label] = 1
        solution = solution_binary

    elif task_type in [BINARY_CLASSIFICATION, REGRESSION]:
        if len(solution.shape) == 1:
            solution = solution.reshape((-1, 1))

    if task_type not in TASK_TYPES:
        raise NotImplementedError(task_type)

    if solution.shape != prediction.shape:
        raise ValueError('Solution shape %s != prediction shape %s' %
                         (solution.shape, prediction.shape))

    if all_scoring_functions:
        score = dict()
        if task_type in REGRESSION_TASKS:
            cprediction = sanitize_array(prediction)
            for metric_ in REGRESSION_METRIC:
                score[metric_] = regression_metrics.calculate_score(metric_,
                                                                    solution,
                                                                    cprediction)
        else:
            csolution, cprediction = normalize_array(solution, prediction)
            for metric_ in CLASSIFICATION_METRICS:
                score[metric_] = classification_metrics.calculate_score(
                    metric_, csolution, cprediction, task_type)

    else:
        if task_type in REGRESSION_TASKS:
            cprediction = sanitize_array(prediction)
            score = regression_metrics.calculate_score(metric,
                                                       solution,
                                                       cprediction)
        else:
            csolution, cprediction = normalize_array(solution, prediction)
            score = classification_metrics.calculate_score(metric,
                                                           csolution,
                                                           cprediction,
                                                           task=task_type)
    return score

def get_new_run_num(lockdir=None):
    seed = os.environ.get('AUTOSKLEARN_SEED')
    counter_file = 'num_run'
    if seed is not None:
        counter_file = counter_file + ('_%s' % seed)
    if lockdir is None:
        lockdir = os.getcwd()
    counter_file = os.path.join(lockdir, counter_file)
    lock = lockfile.LockFile(counter_file)
    with lock:
        if not os.path.exists(counter_file):
            with open(counter_file, 'w') as fh:
                fh.write('0')
            num = 0
        else:
            with open(counter_file, 'r') as fh:
                num = int(fh.read())
            num += 1
            with open(counter_file, 'w') as fh:
                fh.write(str(num).zfill(4))

    return num
<|endoftext|>


<commit_before>from ..queues import QueueBuiltin, QueueTtS
from ..components import _BaseComponent
from .FileCrawler import FileCrawler

import os
import sys
import multiprocessing

class BuiltinRunner(_BaseComponent):

    def __init__(self):
        super().__init__()
        self.queue_builtin = QueueBuiltin()
        self.queue_tts = QueueTtS()
        self.file_crawler = FileCrawler()

    def run(self):
        command = self.queue_builtin.get()
        command_list = command.rsplit()
        target = self.execute_order(command);
        if target is None :
            print('No file or application corresponding found : {}'.format(command))
        else :
            print('Builtin runner execute : {}'.format(command))
            p = multiprocessing.Process(target=os.startfile, args=(target,))
            p.start()
            p.join()
        self.queue_tts.put('task completed')
        self.queue_builtin.task_done()

    def execute order(self, command):
        command_list = command.rsplit(' ')
        order = command_list[0]
        if order == 'exit':
            os._exit(0)
        elif (len(command_list) > 1):
            if order == 'open' :
                return self.file_crawler.find(command_list[1], False)
            else :
                return self.file_crawler.find(command_list[1])
<commit_msg>fixing typo<commit_after>from ..queues import QueueBuiltin, QueueTtS
from ..components import _BaseComponent
from .FileCrawler import FileCrawler

import os
import sys
import multiprocessing

class BuiltinRunner(_BaseComponent):

    def __init__(self):
        super().__init__()
        self.queue_builtin = QueueBuiltin()
        self.queue_tts = QueueTtS()
        self.file_crawler = FileCrawler()

    def run(self):
        command = self.queue_builtin.get()
        command_list = command.rsplit()
        target = self.execute_order(command);
        if target is None :
            print('No file or application corresponding found : {}'.format(command))
        else :
            print('Builtin runner execute : {}'.format(command))
            p = multiprocessing.Process(target=os.startfile, args=(target,))
            p.start()
            p.join()
        self.queue_tts.put('task completed')
        self.queue_builtin.task_done()

    def execute_order(self, command):
        command_list = command.rsplit(' ')
        order = command_list[0]
        if order == 'exit':
            os._exit(0)
        elif (len(command_list) > 1):
            if order == 'open' :
                return self.file_crawler.find(command_list[1], False)
            else :
                return self.file_crawler.find(command_list[1])
<|endoftext|>


<commit_before>from ... import io, style
from ...vendor.Qt import QtCore
from ...vendor import qtawesome as qta

from ..projectmanager.model import (
    TreeModel,
    Node
)

from . import lib

class SubsetsModel(TreeModel):
    COLUMNS = ["subset",
               "family",
               "version",
               "time",
               "author",
               "frames",
               "duration",
               "handles",
               "step"]

    def __init__(self, parent=None):
        super(SubsetsModel, self).__init__(parent=parent)
        self._asset_id = None
        self._icons = {"subset": qta.icon("fa.file-o",
                                          color=style.colors.default)}

    def set_asset(self, asset_id):
        self._asset_id = asset_id
        self.refresh()

    def setData(self, index, value, role=QtCore.Qt.EditRole):

        # Trigger additional edit when `version` column changed
        # because it also updates the information in other columns
        if index.column() == 2:
            node = index.internalPointer()
            parent = node["_id"]
            version = io.find_one({"name": value,
                                   "type": "version",
                                   "parent": parent})
            self.set_version(index, version)

        return super(SubsetsModel, self).setData(index, value, role)

    def set_version(self, index, version):
        """Update the version data of the given index.

        Arguments:
            version (dict) Version document in the database. """

        assert isinstance(index, QtCore.QModelIndex)
        if not index.isValid():
            return

        node = index.internalPointer()
        assert version['parent'] == node['_id'], ("Version does not "
                                                  "belong to subset")

        # Get the data from the version
        version_data = version.get("data", dict())

        # Compute frame ranges (if data is present)
        start = version_data.get("startFrame", None)
        end = version_data.get("endFrame", None)
        handles = version_data.get("handles", None)
        if start is not None and end is not None:
            # Remove superfluous zeros from numbers (3.0 -> 3) to improve
            # readability for most frame ranges
            start_clean = ('%f' % start).rstrip('0').rstrip('.')
            end_clean = ('%f' % end).rstrip('0').rstrip('.')
            frames = "{0}-{1}".format(start_clean, end_clean)
            duration = end - start + 1
        else:
            frames = None
            duration = None

        family = version_data.get("families", [None])[0]
        family_config = lib.get(lib.FAMILY_CONFIG, family)

        node.update({
            "version": version['name'],
            "version_document": version,
            "author": version_data.get("author", None),
            "time": version_data.get("time", None),
            "family": family,
            "familyLabel": family_config.get("label", family),
            "familyIcon": family_config.get('icon', None),
            "startFrame": start,
            "endFrame": end,
            "duration": duration,
            "handles": handles,
            "frames": frames,
            "step": version_data.get("step", None)
        })

    def refresh(self):

        self.clear()
        self.beginResetModel()
        if not self._asset_id:
            self.endResetModel()
            return

        row = 0
        for subset in io.find({"type": "subset",
                               "parent": self._asset_id}):

            last_version = io.find_one({"type": "version",
                                        "parent": subset['_id']},
                                       sort=[("name", -1)])
            if not last_version:
                # No published version for the subset
                continue

            data = subset.copy()
            data['subset'] = data['name']

            node = Node()
            node.update(data)

            self.add_child(node)

            # Set the version information
            index = self.index(row, 0, parent=QtCore.QModelIndex())
            self.set_version(index, last_version)

            row += 1

        self.endResetModel()

    def data(self, index, role):

        if not index.isValid():
            return

        if role == QtCore.Qt.DisplayRole:
            if index.column() == 1:
                # Show familyLabel instead of family
                node = index.internalPointer()
                return node.get("familyLabel", None)

        if role == QtCore.Qt.DecorationRole:

            # Add icon to subset column
            if index.column() == 0:
                return self._icons['subset']

            # Add icon to family column
            if index.column() == 1:
                node = index.internalPointer()
                return node.get("familyIcon", None)

        return super(SubsetsModel, self).data(index, role)

    def flags(self, index):
        flags = QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable

        # Make the version column editable
        if index.column() == 2:  # version column
            flags |= QtCore.Qt.ItemIsEditable

        return flags

class FamiliesFilterProxyModel(QtCore.QSortFilterProxyModel):
    """Filters to specified families"""

    def __init__(self, *args, **kwargs):
        super(FamiliesFilterProxyModel, self).__init__(*args, **kwargs)
        self._families = set()

    def familyFilter(self):
        return self._families

    def setFamiliesFilter(self, values):
        """Set the families to include"""
        assert isinstance(values, (tuple, list, set))
        self._families = set(values)
        self.invalidateFilter()

    def filterAcceptsRow(self, row=0, parent=QtCore.QModelIndex()):

        if not self._families:
            return False

        model = self.sourceModel()
        index = model.index(row, 0, parent=parent)

        # Ensure index is valid
        if not index.isValid() or index is None:
            return True

        # Get the node data and validate
        node = model.data(index, TreeModel.NodeRole)
        family = node.get("family", None)

        if not family:
            return True

        # We want to keep the families which are not in the list
        return family in self._families
<commit_msg>Cosmetic<commit_after>from ... import io, style
from ...vendor.Qt import QtCore
from ...vendor import qtawesome as qta

from ..projectmanager.model import (
    TreeModel,
    Node
)

from . import lib

class SubsetsModel(TreeModel):
    COLUMNS = ["subset",
               "family",
               "version",
               "time",
               "author",
               "frames",
               "duration",
               "handles",
               "step"]

    def __init__(self, parent=None):
        super(SubsetsModel, self).__init__(parent=parent)
        self._asset_id = None
        self._icons = {"subset": qta.icon("fa.file-o",
                                          color=style.colors.default)}

    def set_asset(self, asset_id):
        self._asset_id = asset_id
        self.refresh()

    def setData(self, index, value, role=QtCore.Qt.EditRole):

        # Trigger additional edit when `version` column changed
        # because it also updates the information in other columns
        if index.column() == 2:
            node = index.internalPointer()
            parent = node["_id"]
            version = io.find_one({"name": value,
                                   "type": "version",
                                   "parent": parent})
            self.set_version(index, version)

        return super(SubsetsModel, self).setData(index, value, role)

    def set_version(self, index, version):
        """Update the version data of the given index.

        Arguments:
            version (dict) Version document in the database. """

        assert isinstance(index, QtCore.QModelIndex)
        if not index.isValid():
            return

        node = index.internalPointer()
        assert version['parent'] == node['_id'], ("Version does not "
                                                  "belong to subset")

        # Get the data from the version
        version_data = version.get("data", dict())

        # Compute frame ranges (if data is present)
        start = version_data.get("startFrame", None)
        end = version_data.get("endFrame", None)
        handles = version_data.get("handles", None)
        if start is not None and end is not None:
            # Remove superfluous zeros from numbers (3.0 -> 3) to improve
            # readability for most frame ranges
            start_clean = ('%f' % start).rstrip('0').rstrip('.')
            end_clean = ('%f' % end).rstrip('0').rstrip('.')
            frames = "{0}-{1}".format(start_clean, end_clean)
            duration = end - start + 1
        else:
            frames = None
            duration = None

        family = version_data.get("families", [None])[0]
        family_config = lib.get(lib.FAMILY_CONFIG, family)

        node.update({
            "version": version['name'],
            "version_document": version,
            "author": version_data.get("author", None),
            "time": version_data.get("time", None),
            "family": family,
            "familyLabel": family_config.get("label", family),
            "familyIcon": family_config.get('icon', None),
            "startFrame": start,
            "endFrame": end,
            "duration": duration,
            "handles": handles,
            "frames": frames,
            "step": version_data.get("step", None)
        })

    def refresh(self):

        self.clear()
        self.beginResetModel()
        if not self._asset_id:
            self.endResetModel()
            return

        row = 0
        for subset in io.find({"type": "subset",
                               "parent": self._asset_id}):

            last_version = io.find_one({"type": "version",
                                        "parent": subset["_id"]},
                                       sort=[("name", -1)])
            if not last_version:
                # No published version for the subset
                continue

            data = subset.copy()
            data["subset"] = data["name"]

            node = Node()
            node.update(data)

            self.add_child(node)

            # Set the version information
            index = self.index(row, 0, parent=QtCore.QModelIndex())
            self.set_version(index, last_version)

            row += 1

        self.endResetModel()

    def data(self, index, role):

        if not index.isValid():
            return

        if role == QtCore.Qt.DisplayRole:
            if index.column() == 1:
                # Show familyLabel instead of family
                node = index.internalPointer()
                return node.get("familyLabel", None)

        if role == QtCore.Qt.DecorationRole:

            # Add icon to subset column
            if index.column() == 0:
                return self._icons["subset"]

            # Add icon to family column
            if index.column() == 1:
                node = index.internalPointer()
                return node.get("familyIcon", None)

        return super(SubsetsModel, self).data(index, role)

    def flags(self, index):
        flags = QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable

        # Make the version column editable
        if index.column() == 2:  # version column
            flags |= QtCore.Qt.ItemIsEditable

        return flags

class FamiliesFilterProxyModel(QtCore.QSortFilterProxyModel):
    """Filters to specified families"""

    def __init__(self, *args, **kwargs):
        super(FamiliesFilterProxyModel, self).__init__(*args, **kwargs)
        self._families = set()

    def familyFilter(self):
        return self._families

    def setFamiliesFilter(self, values):
        """Set the families to include"""
        assert isinstance(values, (tuple, list, set))
        self._families = set(values)
        self.invalidateFilter()

    def filterAcceptsRow(self, row=0, parent=QtCore.QModelIndex()):

        if not self._families:
            return False

        model = self.sourceModel()
        index = model.index(row, 0, parent=parent)

        # Ensure index is valid
        if not index.isValid() or index is None:
            return True

        # Get the node data and validate
        node = model.data(index, TreeModel.NodeRole)
        family = node.get("family", None)

        if not family:
            return True

        # We want to keep the families which are not in the list
        return family in self._families
<|endoftext|>