<commit_before>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2018 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/dib-lab/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

def subparser(subparsers):
    """Define the `kevlar simlike` command-line interface."""

    desc = "Sort variants by likelihood score"
    subparser = subparsers.add_parser('simlike', description=desc,
                                      add_help=False)

    count_args = subparser.add_argument_group(
        'K-mer count files',
        'Likelihood scores are based on the abundance of alternate allele '
        'k-mers in each sample and on the abundance of reference allele '
        'k-mers in the reference genome.'
    )
    count_args.add_argument('--case', metavar='CT',
                            help='k-mer counttable for case/proband')
    count_args.add_argument('--controls', nargs='+', metavar='CT',
                            help='k-mer counttables for controls/parents/'
                            'siblings, 1 counttable per sample')
    count_args.add_argument('--refr', metavar='REFR', help='k-mer '
                            'smallcounttable for reference genome')

    thresh_args = subparser.add_argument_group(
        'K-mer count thresholds',
        'The thresholds originally used to detect novel k-mers are used at '
        'this stage to distinguish true variants from spurious predictions.'
    )
    thresh_args.add_argument('--ctrl-max', metavar='X', type=int, default=1,
                             help='maximum abundance threshold for controls; '
                             'default is 1')
    thresh_args.add_argument('--case-min', metavar='Y', type=int, default=6,
                             help='minimum abundance threshold for proband; '
                             'default is 6')

    cov_args = subparser.add_argument_group(
        'K-mer coverage',
        'Likelihood scores also depend on the estimated or observed '
        'distribution of k-mer abundances in each sample.'
    )
    cov_args.add_argument('--mu', metavar='μ', type=float, default=30.0,
                          help='mean k-mer abundance; default is 30.0')
    cov_args.add_argument('--sigma', metavar='σ', type=float, default=8.0,
                          help='standard deviation of k-mer abundance; '
                          'default is 8.0')
    cov_args.add_argument('--epsilon', metavar='ε', type=float, default=0.001,
                          help='error rate; default is 0.001')
    cov_args.add_argument('--static', dest='dynamic', action='store_false',
                          help='when computing likelihood scores for SNVs, '
                          'disable dynamic error rate (scaled by abundance of '
                          'reference allele k-mer in the reference genome)')

    filt_args = subparser.add_argument_group(
        'Heuristic filters',
        'The following heuristic filters can improve accuracy when calling '
        'de novo variants but may require tuning for your particular data set.'
    )
    filt_args.add_argument(
        '--ctrl-abund-high', metavar='H', type=int, default=4,
        help='a variant call will be filtered out if either of the control '
        'samples has >H high abundance k-mers spanning the variant (where '
        'high abundance means > --ctrl-max); by default, H=4; set H<=0 to '
        'disable the filter'
    )
    filt_args.add_argument(
        '--case-abund-low', metavar='L', type=int, default=5,
        help='a variant call will be filtered out if the case sample has L or '
        'more consecutive low abundance k-mers spanning the variant (where '
        'low abundance means < --case-min); by default, L=5; set L<=0 to '
        'disable the filter'
    )
    filt_args.add_argument(
        '--min-like-score', metavar='S', type=float, default=0.0,
        help='filter out variant predictions with likelihood scores < S; by '
        'default, S = 0.0, but it\'s often possible to improve specificity '
        'without sacrificing sensitivity by raising S to, for example, 50.0'
    )
    filt_args.add_argument(
        '--drop-outliers', action='store_true',
        help='discard terminal variant-spanning k-mers with abunance much '
        'higher than average (representing k-mers that should be in the '
        'reference genome but are not); this will increase sensitivity, but '
        'will potentially introduce many false calls as well'
    )

    misc_args = subparser.add_argument_group('Miscellaneous settings')
    misc_args.add_argument('-h', '--help', action='help',
                           help='show this help message and exit')
    misc_args.add_argument('--sample-labels', metavar='LBL', type=str,
                           nargs='+', help='list of sample labels (with case/'
                           'proband first)')
    misc_args.add_argument('-f', '--fast-mode', action='store_true',
                           help='whenever possible, stop computations '
                           'prematurely for any putative variants that have '
                           'already been filtered out')
    misc_args.add_argument('-o', '--out', metavar='OUT', default='-',
                           help='output file; default is terminal (standard '
                           'output)')
    subparser.add_argument('vcf', nargs='+')
<commit_msg>Make --case|--controls|--refr args mandatory for simlike command<commit_after>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2018 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/dib-lab/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

def subparser(subparsers):
    subparser = subparsers.add_parser(
        'simlike', description='Sort variants by likelihood score',
        add_help=False
    )

    count_args = subparser.add_argument_group(
        'K-mer count files',
        'Likelihood scores are based on the abundance of alternate allele '
        'k-mers in each sample and on the abundance of reference allele '
        'k-mers in the reference genome.'
    )
    count_args.add_argument(
        '--case', metavar='CT', required=True,
        help='k-mer counttable for case/proband'
    )
    count_args.add_argument(
        '--controls', nargs='+', metavar='CT', required=True,
        help='k-mer counttables for controls/parents/siblings, 1 counttable '
        'per sample'
    )
    count_args.add_argument(
        '--refr', metavar='REFR',  required=True,
        help='k-mer smallcounttable for reference genome'
    )

    thresh_args = subparser.add_argument_group(
        'K-mer count thresholds',
        'The thresholds originally used to detect novel k-mers are used at '
        'this stage to distinguish true variants from spurious predictions.'
    )
    thresh_args.add_argument('--ctrl-max', metavar='X', type=int, default=1,
                             help='maximum abundance threshold for controls; '
                             'default is 1')
    thresh_args.add_argument('--case-min', metavar='Y', type=int, default=6,
                             help='minimum abundance threshold for proband; '
                             'default is 6')

    cov_args = subparser.add_argument_group(
        'K-mer coverage',
        'Likelihood scores also depend on the estimated or observed '
        'distribution of k-mer abundances in each sample.'
    )
    cov_args.add_argument('--mu', metavar='μ', type=float, default=30.0,
                          help='mean k-mer abundance; default is 30.0')
    cov_args.add_argument('--sigma', metavar='σ', type=float, default=8.0,
                          help='standard deviation of k-mer abundance; '
                          'default is 8.0')
    cov_args.add_argument('--epsilon', metavar='ε', type=float, default=0.001,
                          help='error rate; default is 0.001')
    cov_args.add_argument('--static', dest='dynamic', action='store_false',
                          help='when computing likelihood scores for SNVs, '
                          'disable dynamic error rate (scaled by abundance of '
                          'reference allele k-mer in the reference genome)')

    filt_args = subparser.add_argument_group(
        'Heuristic filters',
        'The following heuristic filters can improve accuracy when calling '
        'de novo variants but may require tuning for your particular data set.'
    )
    filt_args.add_argument(
        '--ctrl-abund-high', metavar='H', type=int, default=4,
        help='a variant call will be filtered out if either of the control '
        'samples has >H high abundance k-mers spanning the variant (where '
        'high abundance means > --ctrl-max); by default, H=4; set H<=0 to '
        'disable the filter'
    )
    filt_args.add_argument(
        '--case-abund-low', metavar='L', type=int, default=5,
        help='a variant call will be filtered out if the case sample has L or '
        'more consecutive low abundance k-mers spanning the variant (where '
        'low abundance means < --case-min); by default, L=5; set L<=0 to '
        'disable the filter'
    )
    filt_args.add_argument(
        '--min-like-score', metavar='S', type=float, default=0.0,
        help='filter out variant predictions with likelihood scores < S; by '
        'default, S = 0.0, but it\'s often possible to improve specificity '
        'without sacrificing sensitivity by raising S to, for example, 50.0'
    )
    filt_args.add_argument(
        '--drop-outliers', action='store_true',
        help='discard terminal variant-spanning k-mers with abunance much '
        'higher than average (representing k-mers that should be in the '
        'reference genome but are not); this will increase sensitivity, but '
        'will potentially introduce many false calls as well'
    )

    misc_args = subparser.add_argument_group('Miscellaneous settings')
    misc_args.add_argument('-h', '--help', action='help',
                           help='show this help message and exit')
    misc_args.add_argument('--sample-labels', metavar='LBL', type=str,
                           nargs='+', help='list of sample labels (with case/'
                           'proband first)')
    misc_args.add_argument('-f', '--fast-mode', action='store_true',
                           help='whenever possible, stop computations '
                           'prematurely for any putative variants that have '
                           'already been filtered out')
    misc_args.add_argument('-o', '--out', metavar='OUT', default='-',
                           help='output file; default is terminal (standard '
                           'output)')
    subparser.add_argument('vcf', nargs='+')
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function

from argparse import ArgumentParser
from ConfigParser import ConfigParser
import json
import logging
import os
import re
import sys

from boto.s3.connection import S3Connection

from download_juju import (
    download_files,
    filter_keys,
    )
from jujuci import (
    acquire_binary,
    JobNamer,
    PackageNamer,
    )
from jujuconfig import get_juju_home
from utility import configure_logging

def parse_args(args=None):
    parser = ArgumentParser()
    default_config = os.path.join(get_juju_home(), 'juju-qa.s3cfg')
    subparsers = parser.add_subparsers(help='Sub-command to run',
                                       dest='command')
    parser_get_juju_bin = subparsers.add_parser(
        'get-juju-bin', help='Retrieve and extract juju binaries.',
        description="""
        Download the package for this client machine from s3.
        Extract the package files.  Print the location of the juju client.
        """)
    parser_get = subparsers.add_parser(
        'get', help='Download job files(s)',
        description="""
        Download a file from a job for this revision build.

        Within the revision build, the most recent version will be selected.
        """)
    parser_get_summary = subparsers.add_parser(
        'get-summary', help='Summarize this test',
        description="""
        Print a summary of what is being tested.
        """
        )
    for subparser in [parser_get_juju_bin, parser_get, parser_get_summary]:
        subparser.add_argument(
            'revision_build', type=int,
            help='The revision-build to use.')
    parser_get.add_argument('job', help='The job to get files from',)
    parser_get.add_argument(
        'file_pattern', help='The file pattern to use for selecting files',)
    parser_get_summary.add_argument(
        'env', help='The name of the environment used for this test.')
    for subparser in [parser_get_juju_bin, parser_get]:
        subparser.add_argument(
            'workspace', nargs='?', default='.',
            help='The directory to download into')
    for subparser in [parser_get_juju_bin, parser_get, parser_get_summary]:
        subparser.add_argument(
            '--config', default=default_config,
            help=('s3cmd config file for credentials.  Default to '
                  'juju-qa.s3cfg in juju home.'))
        subparser.add_argument(
            '--verbose', '-v', default=0, action='count',
            help='Increase verbosity')
    return parser.parse_args(args)

def get_s3_credentials(s3cfg_path):
    config = ConfigParser()
    with open(s3cfg_path) as fp:
        config.readfp(fp)
    access_key = config.get('default', 'access_key')
    secret_key = config.get('default', 'secret_key')
    return access_key, secret_key

def get_job_path(revision_build, job):
    return 'juju-ci/products/version-{}/{}'.format(revision_build, job)

class PackageNotFound(Exception):
    """Raised when a package cannot be found."""

def find_package_key(bucket, revision_build):
    namer = JobNamer.factory()
    job = namer.get_build_binary_job()
    prefix = get_job_path(revision_build, job)
    keys = bucket.list(prefix)
    suffix = PackageNamer.factory().get_release_package_suffix()
    filtered = [
        (k, f) for k, f in filter_keys(keys, suffix)
        if f.startswith('juju-core_')]
    if len(filtered) == 0:
        raise PackageNotFound('Package could not be found.')
    return sorted(filtered, key=lambda x: int(
        re.search(r'build-(\d+)/', x[0].name).group(1)))[-1]

def fetch_juju_binary(bucket, revision_build, workspace):
    package_key, filename = find_package_key(bucket, revision_build)
    logging.info('Selected: %s', package_key.name)
    download_files([package_key], workspace)
    package_path = os.path.join(workspace, filename)
    logging.info('Extracting: %s', package_path)
    return acquire_binary(package_path, workspace)

def find_file_keys(bucket, revision_build, job, file_regex):
    prefix = get_job_path(revision_build, job)
    keys = bucket.list(prefix)
    by_build = {}
    for key in keys:
        match = re.search('^/build-(\d+)', key.name[len(prefix):])
        build = int(match.group(1))
        by_build.setdefault(build, []).append(key)
    # We can't use last successful build, because we don't know what builds
    # are successful, so use last build and require it to be successful.
    last_build = max(by_build.keys())
    build_keys = by_build[last_build]
    filtered = []
    full_prefix = '{}/build-{}/'.format(prefix, last_build)
    for key in build_keys:
        path = key.name[len(full_prefix):]
        if re.match(file_regex, path):
            filtered.append(key)
    return filtered

def fetch_files(bucket, revision_build, job, file_pattern, workspace):
    file_keys = find_file_keys(bucket, revision_build, job, file_pattern)
    out_files = [os.path.join(workspace, k.name.split('/')[-1])
                 for k in file_keys]
    for key in file_keys:
        logging.info('Selected: %s', key.name)
    download_files(file_keys, workspace)
    return out_files

def main():
    args = parse_args()
    log_level = logging.WARNING - args.verbose * (
        logging.WARNING - logging.INFO)
    configure_logging(log_level)
    if args.command == 'get-juju-bin':
        return get_juju_bin(args)
    elif args.command == 'get':
        return cmd_get(args)
    elif args.command == 'get-summary':
        return cmd_get_summary(args)
    else:
        raise Exception('{} not implemented.'.format(args.command))

def get_qa_data_bucket(config):
    credentials = get_s3_credentials(config)
    conn = S3Connection(*credentials)
    return conn.get_bucket('juju-qa-data')

def get_juju_bin(args):
    bucket = get_qa_data_bucket(args.config)
    print(fetch_juju_binary(bucket, args.revision_build, args.workspace))

def cmd_get(args):
    bucket = get_qa_data_bucket(args.config)
    for path in fetch_files(bucket, args.revision_build, args.job,
                            args.file_pattern, args.workspace):
        print(path)

def get_summary(bucket, revision_build, env):
    """Return a textual summary of a test.

    revision-build is the build number of the build-revision that is under
    test.

    env is the name of the environmnet being used to test.
    """
    template = (
        'Testing {branch} {short_revision_id} on {env} for {revision_build}')
    job_path = get_job_path(revision_build, 'build-revision')
    buildvars_path = 'build-{}/buildvars.json'.format(revision_build)
    full_path = '/'.join([job_path, buildvars_path])
    key = bucket.get_key(full_path)
    buildvars = json.loads(key.get_contents_as_string())
    short_revision_id = buildvars['revision_id'][0:7]
    return template.format(
        short_revision_id=short_revision_id, env=env, **buildvars)

def cmd_get_summary(args):
    """Print a textual summary of a test."""
    bucket = get_qa_data_bucket(args.config)
    print(get_summary(bucket, args.revision_build, args.env))

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Provide juju-qa-data as constant.<commit_after>#!/usr/bin/env python
from __future__ import print_function

from argparse import ArgumentParser
from ConfigParser import ConfigParser
import json
import logging
import os
import re
import sys

from boto.s3.connection import S3Connection

from download_juju import (
    download_files,
    filter_keys,
    )
from jujuci import (
    acquire_binary,
    JobNamer,
    PackageNamer,
    )
from jujuconfig import get_juju_home
from utility import configure_logging

JUJU_QA_DATA = 'juju-qa-data'

def parse_args(args=None):
    parser = ArgumentParser()
    default_config = os.path.join(get_juju_home(), 'juju-qa.s3cfg')
    subparsers = parser.add_subparsers(help='Sub-command to run',
                                       dest='command')
    parser_get_juju_bin = subparsers.add_parser(
        'get-juju-bin', help='Retrieve and extract juju binaries.',
        description="""
        Download the package for this client machine from s3.
        Extract the package files.  Print the location of the juju client.
        """)
    parser_get = subparsers.add_parser(
        'get', help='Download job files(s)',
        description="""
        Download a file from a job for this revision build.

        Within the revision build, the most recent version will be selected.
        """)
    parser_get_summary = subparsers.add_parser(
        'get-summary', help='Summarize this test',
        description="""
        Print a summary of what is being tested.
        """
        )
    for subparser in [parser_get_juju_bin, parser_get, parser_get_summary]:
        subparser.add_argument(
            'revision_build', type=int,
            help='The revision-build to use.')
    parser_get.add_argument('job', help='The job to get files from',)
    parser_get.add_argument(
        'file_pattern', help='The file pattern to use for selecting files',)
    parser_get_summary.add_argument(
        'env', help='The name of the environment used for this test.')
    for subparser in [parser_get_juju_bin, parser_get]:
        subparser.add_argument(
            'workspace', nargs='?', default='.',
            help='The directory to download into')
    for subparser in [parser_get_juju_bin, parser_get, parser_get_summary]:
        subparser.add_argument(
            '--config', default=default_config,
            help=('s3cmd config file for credentials.  Default to '
                  'juju-qa.s3cfg in juju home.'))
        subparser.add_argument(
            '--verbose', '-v', default=0, action='count',
            help='Increase verbosity')
    return parser.parse_args(args)

def get_s3_credentials(s3cfg_path):
    config = ConfigParser()
    with open(s3cfg_path) as fp:
        config.readfp(fp)
    access_key = config.get('default', 'access_key')
    secret_key = config.get('default', 'secret_key')
    return access_key, secret_key

def get_job_path(revision_build, job):
    return 'juju-ci/products/version-{}/{}'.format(revision_build, job)

class PackageNotFound(Exception):
    """Raised when a package cannot be found."""

def find_package_key(bucket, revision_build):
    namer = JobNamer.factory()
    job = namer.get_build_binary_job()
    prefix = get_job_path(revision_build, job)
    keys = bucket.list(prefix)
    suffix = PackageNamer.factory().get_release_package_suffix()
    filtered = [
        (k, f) for k, f in filter_keys(keys, suffix)
        if f.startswith('juju-core_')]
    if len(filtered) == 0:
        raise PackageNotFound('Package could not be found.')
    return sorted(filtered, key=lambda x: int(
        re.search(r'build-(\d+)/', x[0].name).group(1)))[-1]

def fetch_juju_binary(bucket, revision_build, workspace):
    package_key, filename = find_package_key(bucket, revision_build)
    logging.info('Selected: %s', package_key.name)
    download_files([package_key], workspace)
    package_path = os.path.join(workspace, filename)
    logging.info('Extracting: %s', package_path)
    return acquire_binary(package_path, workspace)

def find_file_keys(bucket, revision_build, job, file_regex):
    prefix = get_job_path(revision_build, job)
    keys = bucket.list(prefix)
    by_build = {}
    for key in keys:
        match = re.search('^/build-(\d+)', key.name[len(prefix):])
        build = int(match.group(1))
        by_build.setdefault(build, []).append(key)
    # We can't use last successful build, because we don't know what builds
    # are successful, so use last build and require it to be successful.
    last_build = max(by_build.keys())
    build_keys = by_build[last_build]
    filtered = []
    full_prefix = '{}/build-{}/'.format(prefix, last_build)
    for key in build_keys:
        path = key.name[len(full_prefix):]
        if re.match(file_regex, path):
            filtered.append(key)
    return filtered

def fetch_files(bucket, revision_build, job, file_pattern, workspace):
    file_keys = find_file_keys(bucket, revision_build, job, file_pattern)
    out_files = [os.path.join(workspace, k.name.split('/')[-1])
                 for k in file_keys]
    for key in file_keys:
        logging.info('Selected: %s', key.name)
    download_files(file_keys, workspace)
    return out_files

def main():
    args = parse_args()
    log_level = logging.WARNING - args.verbose * (
        logging.WARNING - logging.INFO)
    configure_logging(log_level)
    if args.command == 'get-juju-bin':
        return get_juju_bin(args)
    elif args.command == 'get':
        return cmd_get(args)
    elif args.command == 'get-summary':
        return cmd_get_summary(args)
    else:
        raise Exception('{} not implemented.'.format(args.command))

def get_qa_data_bucket(config):
    credentials = get_s3_credentials(config)
    conn = S3Connection(*credentials)
    return conn.get_bucket(JUJU_QA_DATA)

def get_juju_bin(args):
    bucket = get_qa_data_bucket(args.config)
    print(fetch_juju_binary(bucket, args.revision_build, args.workspace))

def cmd_get(args):
    bucket = get_qa_data_bucket(args.config)
    for path in fetch_files(bucket, args.revision_build, args.job,
                            args.file_pattern, args.workspace):
        print(path)

def get_summary(bucket, revision_build, env):
    """Return a textual summary of a test.

    revision-build is the build number of the build-revision that is under
    test.

    env is the name of the environmnet being used to test.
    """
    template = (
        'Testing {branch} {short_revision_id} on {env} for {revision_build}')
    job_path = get_job_path(revision_build, 'build-revision')
    buildvars_path = 'build-{}/buildvars.json'.format(revision_build)
    full_path = '/'.join([job_path, buildvars_path])
    key = bucket.get_key(full_path)
    buildvars = json.loads(key.get_contents_as_string())
    short_revision_id = buildvars['revision_id'][0:7]
    return template.format(
        short_revision_id=short_revision_id, env=env, **buildvars)

def cmd_get_summary(args):
    """Print a textual summary of a test."""
    bucket = get_qa_data_bucket(args.config)
    print(get_summary(bucket, args.revision_build, args.env))

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>from s4v3 import *
from matplotlib.pyplot as plt # import matplotlib's pyplot and assign it to a variable name. I didn't know you could do this, but it's pretty cool for unweildly and long library names or function. However, it's probably not great for universal readability since I'm changing a well known name into a name that I specifically made up
# upon further reflection this "as" statement is seen in the matplotlib.pylot tutorial, so maybe it's more standard

# more on matplotlib.pylot from http://matplotlib.org/users/pyplot_tutorial.html
# matplotlib.pyplot is a collection of command style functions that make matplotlib work like MATLAB. Each pyplot function makes some change to a figure: e.g., create a figure, create a plotting area in a figure, plot some lines in a plotting area, decorate the plot with labels, etc.... matplotlib.pyplot is stateful, in that it keeps track of the current figure and plotting area, and the plotting functions are directed to the current axes

def create_line_chart(data_sample, title, exported_figure_filename):
	fig = plt.figure() # create a figure object, don't know what this is... but according to the documentation a figure is the "top level container for all plot elements". So it's kinda like a list as it contains all the data that is going to go into our diagram.
	ax = fig.add_subplot(1, 1, 1) # create an axis object within the figure object. This calls the add_subplot function which I'm not familiar with.

	prices = (sorted(map(float, sample))) # create a variable prices and assign it to the sorted, map (what is a map?), in floats, of the sample... did they mean the data_sample? there is no sample defined...
	# map(function, iterable, ...) apply the function to every item of iterable. I'm not sure how float is a function... nevermind it's a built in function of python. float() = return a floating point number constructed from a number or a string. ... so, in this case, we are going to apply the float() function to every item in the sample, or sample_data, ... basically map() is going to make every item in the sample (data sample?) a float. Then... what is sorted going to do... I mean, it's probably going to sort them, but how? What's the default sort parameter for sorted()... it doesn't explicitly say, but it seems to be smallest to largest with smallest at the front of the list.

	x_axis_ticks = list(range(len(sample))) # create a variable named x axis ticks (marks?) and assign it the value of the length of the sample (again, data_sample maybe?) as a range as a list?
	# range() = This is a versatile function to create lists containing arithmetic progressions. It is most often used in for loops. this is taking the length of the sample as it's arg and it's going to create a list that's the length of the sample. it's not a list of the sample it's simply a list of integers that range from 0 to the len(sample) so something like x = [0, 1, 2, 3, 4, ... len]
	# it looks like list() also just creates a list...
	# Ok, so here's what I think is happening: we're creating a variable that is the number of increments along the x axis, number of graph markers. It's going to create a list, that's the range, of the length of the data sample, iterating on increments of 1 integer at a time.

	ax.plot(x_axis_ticks, prices, linewidth=2) # here it looks like we're ploting something on our axis object with the x_axis_ticks, the prices as arguments and with an argument for formatting that sets the graph linewidth to 2)

	ax.set_title(title) # now we're using a function I don't know, probably from the matplotlib that sets the title of the graph. the title arg is pulled from the function args

	ax.set_xlim([0, len(sample)]) # ok, more guess-work here, but it looks like we're setting a limit to the x axis... but really it looks more like a range from 0 to the end of the sample length... the strange thing is that if the sample doesn't increment by one, then the lim might not be right. Like what happens if the sample goes from 10 - 786 but it only has 10 items in the sample. Doesn't len() return the number of items, not the max value?

	ax.set_xlabel('Tie Price ($)') # finally something straightforward. Set the label of the x axis to Tie Price.

	ax.set_ylabel('Number of Ties') # the label of the y axis

	fig.savefig(exported_figure_filename)

create_line_chart([x[2] for x in gucci_ties] # use list comprehension to iterate through the gucci_ties list and chart the results in this case it looks like we're plotting the values from the third row of the data sample (i.e. x[2]), so basically we're making a list of the values from the third row/column of the data sample, but instead of returning the list or assigning it to a variable we're feeding it directly into the creat line chart function as the data sample ... all of that just for the first function argument.

# list comprehension ... explained? ... start by placing everything inside the brackets, like the list yo, [].  Then place the thing you wanted to append to the list at the beginning of the statement. then use a for loop to iterate throught the set of values.

# more at http://www.secnetix.de/olli/Python/list_comprehensions.hawk
<commit_msg>Fix typos<commit_after>from s4v3 import *
import matplotlib.pyplot as plt

# import matplotlib's pyplot and assign it to a variable name. I didn't know you could do this, but it's pretty cool for unweildly and long library names or function. However, it's probably not great for universal readability since I'm changing a well known name into a name that I specifically made up
# upon further reflection this "as" statement is seen in the matplotlib.pylot tutorial, so maybe it's more standard

# more on matplotlib.pylot from http://matplotlib.org/users/pyplot_tutorial.html
# matplotlib.pyplot is a collection of command style functions that make matplotlib work like MATLAB. Each pyplot function makes some change to a figure: e.g., create a figure, create a plotting area in a figure, plot some lines in a plotting area, decorate the plot with labels, etc.... matplotlib.pyplot is stateful, in that it keeps track of the current figure and plotting area, and the plotting functions are directed to the current axes

def create_line_chart(data_sample, title, exported_figure_filename):
	fig = plt.figure() # create a figure object, don't know what this is... but according to the documentation a figure is the "top level container for all plot elements". So it's kinda like a list as it contains all the data that is going to go into our diagram.
	ax = fig.add_subplot(1, 1, 1) # create an axis object within the figure object. This calls the add_subplot function which I'm not familiar with.

	prices = (sorted(map(float, data_sample))) # create a variable prices and assign it to the sorted, map (what is a map?), in floats, of the sample... did they mean the data_sample? there is no sample defined...
	# map(function, iterable, ...) apply the function to every item of iterable. I'm not sure how float is a function... nevermind it's a built in function of python. float() = return a floating point number constructed from a number or a string. ... so, in this case, we are going to apply the float() function to every item in the sample, or sample_data, ... basically map() is going to make every item in the sample (data sample?) a float. Then... what is sorted going to do... I mean, it's probably going to sort them, but how? What's the default sort parameter for sorted()... it doesn't explicitly say, but it seems to be smallest to largest with smallest at the front of the list.

	x_axis_ticks = list(range(len(data_sample))) # create a variable named x axis ticks (marks?) and assign it the value of the length of the sample (again, data_sample maybe?) as a range as a list?
	# range() = This is a versatile function to create lists containing arithmetic progressions. It is most often used in for loops. this is taking the length of the sample as it's arg and it's going to create a list that's the length of the sample. it's not a list of the sample it's simply a list of integers that range from 0 to the len(sample) so something like x = [0, 1, 2, 3, 4, ... len]
	# it looks like list() also just creates a list...
	# Ok, so here's what I think is happening: we're creating a variable that is the number of increments along the x axis, number of graph markers. It's going to create a list, that's the range, of the length of the data sample, iterating on increments of 1 integer at a time.

	ax.plot(x_axis_ticks, prices, linewidth=2) # here it looks like we're ploting something on our axis object with the x_axis_ticks, the prices as arguments and with an argument for formatting that sets the graph linewidth to 2)

	ax.set_title(title) # now we're using a function I don't know, probably from the matplotlib that sets the title of the graph. the title arg is pulled from the function args

	ax.set_xlim([0, len(data_sample)]) # ok, more guess-work here, but it looks like we're setting a limit to the x axis... but really it looks more like a range from 0 to the end of the sample length... the strange thing is that if the sample doesn't increment by one, then the lim might not be right. Like what happens if the sample goes from 10 - 786 but it only has 10 items in the sample. Doesn't len() return the number of items, not the max value?

	ax.set_xlabel('Tie Price ($)') # finally something straightforward. Set the label of the x axis to Tie Price.

	ax.set_ylabel('Number of Ties') # the label of the y axis

	fig.savefig(exported_figure_filename)

create_line_chart([x[2] for x in gucci_ties[1:]], "Distribution of Prices for Gucci Ties", "_charts/s5-line-gucci.png")
	 # use list comprehension to iterate through the gucci_ties list and chart the results in this case it looks like we're plotting the values from the third row of the data sample (i.e. x[2]), so basically we're making a list of the values from the third row/column of the data sample, but instead of returning the list or assigning it to a variable we're feeding it directly into the creat line chart function as the data sample ... all of that just for the first function argument.
	 # Ooo... looks like it returns PNG files.
	 # Don't forget to slice past the header row in gucci_ties

# list comprehension ... explained? ... start by placing everything inside the brackets, like the list yo, [].  Then place the thing you wanted to append to the list at the beginning of the statement. then use a for loop to iterate throught the set of values.

# more at http://www.secnetix.de/olli/Python/list_comprehensions.hawk
<|endoftext|>


<commit_before>import simplejson

from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.db import models
from django.db.models.query import QuerySet
from django.utils.translation import ugettext as _

from django_restricted_resource.managers import RestrictedResourceManager
from django_restricted_resource.models import RestrictedResource
from django_restricted_resource.utils import filter_bogus_users

from linaro_django_xmlrpc.models import AuthToken

class JSONDataError(ValueError):
    """Error raised when JSON is syntactically valid but ill-formed."""

class Tag(models.Model):

    name = models.SlugField(unique=True)

    description = models.TextField(null=True, blank=True)

    def __unicode__(self):
        return self.name

def validate_job_json(data):
    try:
        ob = simplejson.loads(data)
    except ValueError, e:
        raise ValidationError(str(e))
    else:
        if not isinstance(ob, dict):
            raise ValidationError(
                "job json must be an object, not %s" % type(ob).__name__)

class DeviceType(models.Model):
    """
    A class of device, for example a pandaboard or a snowball.
    """

    name = models.SlugField(primary_key=True)

    def __unicode__(self):
        return self.name

    health_check_job = models.TextField(
        null=True, blank=True, default=None, validators=[validate_job_json])

class Device(models.Model):
    """
    A device that we can run tests on.
    """

    OFFLINE = 0
    IDLE = 1
    RUNNING = 2
    OFFLINING = 3

    STATUS_CHOICES = (
        (OFFLINE, 'Offline'),
        (IDLE, 'Idle'),
        (RUNNING, 'Running'),
        (OFFLINING, 'Going offline'),
    )

    # A device health shows a device is ready to test or not
    HEALTH_UNKNOWN, HEALTH_PASS, HEALTH_FAIL = range(3)
    HEALTH_CHOICES = (
        (HEALTH_UNKNOWN, 'Unknown'),
        (HEALTH_PASS, 'Pass'),
        (HEALTH_FAIL, 'Fail'),
    )

    hostname = models.CharField(
        verbose_name = _(u"Hostname"),
        max_length = 200,
        primary_key = True,
    )

    device_type = models.ForeignKey(
        DeviceType, verbose_name=_(u"Device type"))

    current_job = models.ForeignKey(
        "TestJob", blank=True, unique=True, null=True, related_name='+')

    tags = models.ManyToManyField(Tag, blank=True)

    status = models.IntegerField(
        choices = STATUS_CHOICES,
        default = IDLE,
        verbose_name = _(u"Device status"),
    )

    health_status = models.IntegerField(
        choices = HEALTH_CHOICES,
        default = HEALTH_UNKNOWN,
        verbose_name = _(u"Device Health"),
    )

    last_health_report_job = models.ForeignKey(
            "TestJob", blank=True, unique=True, null=True, related_name='+')

    def __unicode__(self):
        return self.hostname

    @models.permalink
    def get_absolute_url(self):
        return ("lava.scheduler.device.detail", [self.pk])

    @models.permalink
    def get_device_health_url(self):
        return ("lava.scheduler.labhealth.detail", [self.pk])

    def recent_jobs(self, user):
        return TestJob.objects.jobs_for_user(
            user).filter(actual_device=self).order_by('-start_time')

    def can_admin(self, user):
        return user.has_perm('lava_scheduler_app.change_device')

    def put_into_maintenance_mode(self, user, reason):
        if self.status in [self.RUNNING, self.OFFLINING]:
            new_status = self.OFFLINING
        else:
            new_status = self.OFFLINE
        DeviceStateTransition.objects.create(
            created_by=user, device=self, old_state=self.status,
            new_state=new_status, message=reason, job=None).save()
        self.status = new_status
        self.save()

    def put_into_online_mode(self, user, reason):
        if self.status not in [Device.OFFLINE, Device.OFFLINING]:
            return
        new_status = self.IDLE
        DeviceStateTransition.objects.create(
            created_by=user, device=self, old_state=self.status,
            new_state=new_status, message=reason, job=None).save()
        self.status = new_status
        self.health_status = Device.HEALTH_UNKNOWN
        self.save()

    #@classmethod
    #def find_devices_by_type(cls, device_type):
    #    return device_type.device_set.all()

class TestJob(RestrictedResource):
    """
    A test job is a test process that will be run on a Device.
    """

    SUBMITTED = 0
    RUNNING = 1
    COMPLETE = 2
    INCOMPLETE = 3
    CANCELED = 4
    CANCELING = 5

    STATUS_CHOICES = (
        (SUBMITTED, 'Submitted'),
        (RUNNING, 'Running'),
        (COMPLETE, 'Complete'),
        (INCOMPLETE, 'Incomplete'),
        (CANCELED, 'Canceled'),
        (CANCELING, 'Canceling'),
    )

    id = models.AutoField(primary_key=True)

    submitter = models.ForeignKey(
        User,
        verbose_name = _(u"Submitter"),
        related_name = '+',
    )

    submit_token = models.ForeignKey(AuthToken, null=True, blank=True)

    description = models.CharField(
        verbose_name = _(u"Description"),
        max_length = 200,
        null = True,
        blank = True,
        default = None
    )

    health_check = models.BooleanField(default=False)

    # Only one of these two should be non-null.
    requested_device = models.ForeignKey(
        Device, null=True, default=None, related_name='+', blank=True)
    requested_device_type = models.ForeignKey(
        DeviceType, null=True, default=None, related_name='+', blank=True)

    tags = models.ManyToManyField(Tag, blank=True)

    # This is set once the job starts.
    actual_device = models.ForeignKey(
        Device, null=True, default=None, related_name='+', blank=True)

    #priority = models.IntegerField(
    #    verbose_name = _(u"Priority"),
    #    default=0)
    submit_time = models.DateTimeField(
        verbose_name = _(u"Submit time"),
        auto_now = False,
        auto_now_add = True
    )
    start_time = models.DateTimeField(
        verbose_name = _(u"Start time"),
        auto_now = False,
        auto_now_add = False,
        null = True,
        blank = True,
        editable = False
    )
    end_time = models.DateTimeField(
        verbose_name = _(u"End time"),
        auto_now = False,
        auto_now_add = False,
        null = True,
        blank = True,
        editable = False
    )
    status = models.IntegerField(
        choices = STATUS_CHOICES,
        default = SUBMITTED,
        verbose_name = _(u"Status"),
    )
    definition = models.TextField(
        editable = False,
    )
    log_file = models.FileField(
        upload_to='lava-logs', default=None, null=True, blank=True)

    results_link = models.CharField(
        max_length=400, default=None, null=True, blank=True)

    def __unicode__(self):
        r = "%s test job" % self.get_status_display()
        if self.requested_device:
            r += " for %s" % (self.requested_device.hostname,)
        return r

    @models.permalink
    def get_absolute_url(self):
        return ("lava.scheduler.job.detail", [self.pk])

    @classmethod
    def from_json_and_user(cls, json_data, user):
        job_data = simplejson.loads(json_data)
        if 'target' in job_data:
            target = Device.objects.get(hostname=job_data['target'])
            device_type = None
        elif 'device_type' in job_data:
            target = None
            device_type = DeviceType.objects.get(name=job_data['device_type'])
        else:
            raise JSONDataError(
                "Neither 'target' nor 'device_type' found in job data.")
        job_name = job_data.get('job_name', '')

        is_check = job_data.get('health_check', False)

        tags = []
        for tag_name in job_data.get('device_tags', []):
            try:
                tags.append(Tag.objects.get(name=tag_name))
            except Tag.DoesNotExist:
                raise JSONDataError("tag %r does not exist" % tag_name)
        job = TestJob(
            definition=json_data, submitter=user, requested_device=target,
            requested_device_type=device_type, description=job_name,
            health_check=is_check, user=user)
        job.save()
        for tag in tags:
            job.tags.add(tag)
        return job

    def can_cancel(self, user):
        return user.is_superuser or user == self.submitter

    def cancel(self):
        if self.status == TestJob.RUNNING:
            self.status = TestJob.CANCELING
        else:
            self.status = TestJob.CANCELED
        self.save()

    def clean(self):
        if self.owner is None and self.group is None:
            self.owner = self.submitter
        super(TestJob, self).clean()

class DeviceStateTransition(models.Model):
    created_on = models.DateTimeField(auto_now_add=True)
    created_by = models.ForeignKey(User, null=True, blank=True)
    device = models.ForeignKey(Device, related_name='transitions')
    job = models.ForeignKey(TestJob, null=True, blank=True)
    old_state = models.IntegerField(choices=Device.STATUS_CHOICES)
    new_state = models.IntegerField(choices=Device.STATUS_CHOICES)
    message = models.TextField(null=True, blank=True)
<commit_msg>restore recent_jobs to working state<commit_after>import simplejson

from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.db import models
from django.db.models.query import QuerySet
from django.utils.translation import ugettext as _

from django_restricted_resource.managers import RestrictedResourceManager
from django_restricted_resource.models import RestrictedResource
from django_restricted_resource.utils import filter_bogus_users

from linaro_django_xmlrpc.models import AuthToken

class JSONDataError(ValueError):
    """Error raised when JSON is syntactically valid but ill-formed."""

class Tag(models.Model):

    name = models.SlugField(unique=True)

    description = models.TextField(null=True, blank=True)

    def __unicode__(self):
        return self.name

def validate_job_json(data):
    try:
        ob = simplejson.loads(data)
    except ValueError, e:
        raise ValidationError(str(e))
    else:
        if not isinstance(ob, dict):
            raise ValidationError(
                "job json must be an object, not %s" % type(ob).__name__)

class DeviceType(models.Model):
    """
    A class of device, for example a pandaboard or a snowball.
    """

    name = models.SlugField(primary_key=True)

    def __unicode__(self):
        return self.name

    health_check_job = models.TextField(
        null=True, blank=True, default=None, validators=[validate_job_json])

class Device(models.Model):
    """
    A device that we can run tests on.
    """

    OFFLINE = 0
    IDLE = 1
    RUNNING = 2
    OFFLINING = 3

    STATUS_CHOICES = (
        (OFFLINE, 'Offline'),
        (IDLE, 'Idle'),
        (RUNNING, 'Running'),
        (OFFLINING, 'Going offline'),
    )

    # A device health shows a device is ready to test or not
    HEALTH_UNKNOWN, HEALTH_PASS, HEALTH_FAIL = range(3)
    HEALTH_CHOICES = (
        (HEALTH_UNKNOWN, 'Unknown'),
        (HEALTH_PASS, 'Pass'),
        (HEALTH_FAIL, 'Fail'),
    )

    hostname = models.CharField(
        verbose_name = _(u"Hostname"),
        max_length = 200,
        primary_key = True,
    )

    device_type = models.ForeignKey(
        DeviceType, verbose_name=_(u"Device type"))

    current_job = models.ForeignKey(
        "TestJob", blank=True, unique=True, null=True, related_name='+')

    tags = models.ManyToManyField(Tag, blank=True)

    status = models.IntegerField(
        choices = STATUS_CHOICES,
        default = IDLE,
        verbose_name = _(u"Device status"),
    )

    health_status = models.IntegerField(
        choices = HEALTH_CHOICES,
        default = HEALTH_UNKNOWN,
        verbose_name = _(u"Device Health"),
    )

    last_health_report_job = models.ForeignKey(
            "TestJob", blank=True, unique=True, null=True, related_name='+')

    def __unicode__(self):
        return self.hostname

    @models.permalink
    def get_absolute_url(self):
        return ("lava.scheduler.device.detail", [self.pk])

    @models.permalink
    def get_device_health_url(self):
        return ("lava.scheduler.labhealth.detail", [self.pk])

    def recent_jobs(self):
        return TestJob.objects.select_related(
            "actual_device",
            "requested_device",
            "requested_device_type",
            "submitter",
            "user",
            "group",
        ).filter(
            actual_device=self
        ).order_by(
            '-start_time'
        )

    def can_admin(self, user):
        return user.has_perm('lava_scheduler_app.change_device')

    def put_into_maintenance_mode(self, user, reason):
        if self.status in [self.RUNNING, self.OFFLINING]:
            new_status = self.OFFLINING
        else:
            new_status = self.OFFLINE
        DeviceStateTransition.objects.create(
            created_by=user, device=self, old_state=self.status,
            new_state=new_status, message=reason, job=None).save()
        self.status = new_status
        self.save()

    def put_into_online_mode(self, user, reason):
        if self.status not in [Device.OFFLINE, Device.OFFLINING]:
            return
        new_status = self.IDLE
        DeviceStateTransition.objects.create(
            created_by=user, device=self, old_state=self.status,
            new_state=new_status, message=reason, job=None).save()
        self.status = new_status
        self.health_status = Device.HEALTH_UNKNOWN
        self.save()

    #@classmethod
    #def find_devices_by_type(cls, device_type):
    #    return device_type.device_set.all()

class TestJob(RestrictedResource):
    """
    A test job is a test process that will be run on a Device.
    """

    SUBMITTED = 0
    RUNNING = 1
    COMPLETE = 2
    INCOMPLETE = 3
    CANCELED = 4
    CANCELING = 5

    STATUS_CHOICES = (
        (SUBMITTED, 'Submitted'),
        (RUNNING, 'Running'),
        (COMPLETE, 'Complete'),
        (INCOMPLETE, 'Incomplete'),
        (CANCELED, 'Canceled'),
        (CANCELING, 'Canceling'),
    )

    id = models.AutoField(primary_key=True)

    submitter = models.ForeignKey(
        User,
        verbose_name = _(u"Submitter"),
        related_name = '+',
    )

    submit_token = models.ForeignKey(AuthToken, null=True, blank=True)

    description = models.CharField(
        verbose_name = _(u"Description"),
        max_length = 200,
        null = True,
        blank = True,
        default = None
    )

    health_check = models.BooleanField(default=False)

    # Only one of these two should be non-null.
    requested_device = models.ForeignKey(
        Device, null=True, default=None, related_name='+', blank=True)
    requested_device_type = models.ForeignKey(
        DeviceType, null=True, default=None, related_name='+', blank=True)

    tags = models.ManyToManyField(Tag, blank=True)

    # This is set once the job starts.
    actual_device = models.ForeignKey(
        Device, null=True, default=None, related_name='+', blank=True)

    #priority = models.IntegerField(
    #    verbose_name = _(u"Priority"),
    #    default=0)
    submit_time = models.DateTimeField(
        verbose_name = _(u"Submit time"),
        auto_now = False,
        auto_now_add = True
    )
    start_time = models.DateTimeField(
        verbose_name = _(u"Start time"),
        auto_now = False,
        auto_now_add = False,
        null = True,
        blank = True,
        editable = False
    )
    end_time = models.DateTimeField(
        verbose_name = _(u"End time"),
        auto_now = False,
        auto_now_add = False,
        null = True,
        blank = True,
        editable = False
    )
    status = models.IntegerField(
        choices = STATUS_CHOICES,
        default = SUBMITTED,
        verbose_name = _(u"Status"),
    )
    definition = models.TextField(
        editable = False,
    )
    log_file = models.FileField(
        upload_to='lava-logs', default=None, null=True, blank=True)

    results_link = models.CharField(
        max_length=400, default=None, null=True, blank=True)

    def __unicode__(self):
        r = "%s test job" % self.get_status_display()
        if self.requested_device:
            r += " for %s" % (self.requested_device.hostname,)
        return r

    @models.permalink
    def get_absolute_url(self):
        return ("lava.scheduler.job.detail", [self.pk])

    @classmethod
    def from_json_and_user(cls, json_data, user):
        job_data = simplejson.loads(json_data)
        if 'target' in job_data:
            target = Device.objects.get(hostname=job_data['target'])
            device_type = None
        elif 'device_type' in job_data:
            target = None
            device_type = DeviceType.objects.get(name=job_data['device_type'])
        else:
            raise JSONDataError(
                "Neither 'target' nor 'device_type' found in job data.")
        job_name = job_data.get('job_name', '')

        is_check = job_data.get('health_check', False)

        tags = []
        for tag_name in job_data.get('device_tags', []):
            try:
                tags.append(Tag.objects.get(name=tag_name))
            except Tag.DoesNotExist:
                raise JSONDataError("tag %r does not exist" % tag_name)
        job = TestJob(
            definition=json_data, submitter=user, requested_device=target,
            requested_device_type=device_type, description=job_name,
            health_check=is_check, user=user)
        job.save()
        for tag in tags:
            job.tags.add(tag)
        return job

    def can_cancel(self, user):
        return user.is_superuser or user == self.submitter

    def cancel(self):
        if self.status == TestJob.RUNNING:
            self.status = TestJob.CANCELING
        else:
            self.status = TestJob.CANCELED
        self.save()

    def clean(self):
        if self.owner is None and self.group is None:
            self.owner = self.submitter
        super(TestJob, self).clean()

class DeviceStateTransition(models.Model):
    created_on = models.DateTimeField(auto_now_add=True)
    created_by = models.ForeignKey(User, null=True, blank=True)
    device = models.ForeignKey(Device, related_name='transitions')
    job = models.ForeignKey(TestJob, null=True, blank=True)
    old_state = models.IntegerField(choices=Device.STATUS_CHOICES)
    new_state = models.IntegerField(choices=Device.STATUS_CHOICES)
    message = models.TextField(null=True, blank=True)
<|endoftext|>


<commit_before>def system_boot_time():
    '''
    Get the system boot time.

    CLI Example::

        salt 'os:Windows' sccm.system_boot_time
    '''
    return __salt__['cmd.run']('systeminfo | select-string "System Boot Time"', shell='powershell')

def hardware_inventory():
    '''
    Invoke the hardware inventory cycle

    CLI Example::

        salt 'os:Windows' sccm.hardware_inventory
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000001}"', shell='powershell')

def software_inventory():
    '''
    Invoke the software inventory cycle

    CLI Example::

        salt 'os:Windows' sccm.software_inventory
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000002}"', shell='powershell')

def discovery_data_record():
    '''
    Invoke the discovery data record

    CLI Example::

        salt 'os:Windows' sccm.discovery_data_record
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000003}"', shell='powershell')

def request_machine_policy():
    '''
    Request the machine policy

    CLI Example::

        salt 'os:Windows' sccm.request_machine_policy
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000021}"', shell='powershell')

def evaluate_machine_policy():
    '''
    Evaluate the machine policy

    CLI Example::

        salt 'os:Windows' sccm.evaluate_machine_policy
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000022}"', shell='powershell')

def file_collection():
    '''
    Invoke file collection cycle

    CLI Example::

        salt 'os:Windows' sccm.file_collection
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000010}"', shell='powershell')

def sw_metering_usage_report():
    '''
    Software metering generating usage report

    CLI Example::

        salt 'os:Windows' sccm.sw_metering_usage_report
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000031}"', shell='powershell')

def windows_installer_source_list():
    '''
    Windows installer source list

    CLI Example::

        salt 'os:Windows' sccm.windows_installer_source_list
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000032}"', shell='powershell')

def software_updates_scan():
    '''
    Invoke software updates scan

    CLI Example::

        salt 'os:Windows' sccm.software_updates_scan
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000113}"', shell='powershell')

def software_updates_store():
    '''
    Invoke software updates store

    CLI Example::

        salt 'os:Windows' sccm.software_updates_store
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000114}"', shell='powershell')

def software_updates_deployment():
    '''
    Invoke software updates deployment

    CLI Example::

        salt 'os:Windows' sccm.software_updates_deployment
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000108}"', shell='powershell')

# sccm updates
def update_count():
    '''
    Get the number of available updates from ConfigMgr

    CLI Example::

        salt 'os:Windows' sccm.update_count
    '''
    return __salt__['cmd.run']('($updates | measure-object).Count'), shell='powershell')

def list_updates():
    '''
    Get the list of available updates

    CLI Example::

        salt 'os:Windows' sccm.updates
    '''
    return __salt__['cmd.run']('gcim -namespace root\ccm\clientsdk
-query \'Select * from CCM_SoftwareUpdate\'', shell="powershell")

def install_updates():
    '''
    Install updates with deadlines

    CLI Example::

        salt 'os:Windows' sccm.install_updates
    '''
    return __salt__['cmd.run']('iwmi -namespace root\ccm\clientsdk -Class CCM_SoftwareUpdatesManager -name InstallUpdates([System.Management.MangementObject[]](gwmi -namespace root\ccm\clientsdk -query \'Select * from CCM_SoftwareUpdate\'))', shell='powershell')

def reboot_pending():
    '''
    Is a reboot pending from ConfigMgr

    CLI Example::

        salt 'os:Windows' sccm.reboot_pending
    '''
    return __salt__['cmd.run']('(icim -namespace root\ccm\clientsdk -ClassName CCM_ClientUtilities -Name DetermineIfRebootPending).RebootPending', shell='powershell')
<commit_msg>Fix broken python<commit_after>def system_boot_time():
    '''
    Get the system boot time.

    CLI Example::

        salt 'os:Windows' sccm.system_boot_time
    '''
    return __salt__['cmd.run']('systeminfo | select-string "System Boot Time"', shell='powershell')

def hardware_inventory():
    '''
    Invoke the hardware inventory cycle

    CLI Example::

        salt 'os:Windows' sccm.hardware_inventory
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000001}"', shell='powershell')

def software_inventory():
    '''
    Invoke the software inventory cycle

    CLI Example::

        salt 'os:Windows' sccm.software_inventory
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000002}"', shell='powershell')

def discovery_data_record():
    '''
    Invoke the discovery data record

    CLI Example::

        salt 'os:Windows' sccm.discovery_data_record
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000003}"', shell='powershell')

def request_machine_policy():
    '''
    Request the machine policy

    CLI Example::

        salt 'os:Windows' sccm.request_machine_policy
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000021}"', shell='powershell')

def evaluate_machine_policy():
    '''
    Evaluate the machine policy

    CLI Example::

        salt 'os:Windows' sccm.evaluate_machine_policy
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000022}"', shell='powershell')

def file_collection():
    '''
    Invoke file collection cycle

    CLI Example::

        salt 'os:Windows' sccm.file_collection
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000010}"', shell='powershell')

def sw_metering_usage_report():
    '''
    Software metering generating usage report

    CLI Example::

        salt 'os:Windows' sccm.sw_metering_usage_report
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000031}"', shell='powershell')

def windows_installer_source_list():
    '''
    Windows installer source list

    CLI Example::

        salt 'os:Windows' sccm.windows_installer_source_list
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000032}"', shell='powershell')

def software_updates_scan():
    '''
    Invoke software updates scan

    CLI Example::

        salt 'os:Windows' sccm.software_updates_scan
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000113}"', shell='powershell')

def software_updates_store():
    '''
    Invoke software updates store

    CLI Example::

        salt 'os:Windows' sccm.software_updates_store
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000114}"', shell='powershell')

def software_updates_deployment():
    '''
    Invoke software updates deployment

    CLI Example::

        salt 'os:Windows' sccm.software_updates_deployment
    '''
    return __salt__['cmd.run']('Invoke-WmiMethod -Namespace root\CCM -Class SMS_Client -Name TriggerSchedule -ArgumentList "{00000000-0000-0000-0000-000000000108}"', shell='powershell')

def update_count():
    '''
    Get the number of available updates from ConfigMgr

    CLI Example::

        salt 'os:Windows' sccm.update_count
    '''
    return __salt__['cmd.run']('(gcim -namespace root\ccm\clientsdk -query \'Select * from CCM_SoftwareUpdate\' | measure-object).Count', shell='powershell')

def list_updates():
    '''
    Get the list of available updates

    CLI Example::

        salt 'os:Windows' sccm.updates
    '''
    return __salt__['cmd.run']('gcim -namespace root\ccm\clientsdk -query \'Select * from CCM_SoftwareUpdate\'', shell="powershell")

def install_updates():
    '''
    Install updates with deadlines

    CLI Example::

        salt 'os:Windows' sccm.install_updates
    '''
    return __salt__['cmd.run']('iwmi -namespace root\ccm\clientsdk -Class CCM_SoftwareUpdatesManager -name InstallUpdates([System.Management.MangementObject[]](gwmi -namespace root\ccm\clientsdk -query \'Select * from CCM_SoftwareUpdate\'))', shell='powershell')

def reboot_pending():
    '''
    Is a reboot pending from ConfigMgr

    CLI Example::

        salt 'os:Windows' sccm.reboot_pending
    '''
    return __salt__['cmd.run']('(icim -namespace root\ccm\clientsdk -ClassName CCM_ClientUtilities -Name DetermineIfRebootPending).RebootPending', shell='powershell')
<|endoftext|>


<commit_before>"""
StanhopeFramers Tables
"""
import io
import subprocess

import pandas
from stanhope import utils

pandas.set_option('display.max_rows', 999)
pandas.set_option('display.width', 999)
pandas.set_option('display.max_colwidth', 999)

class Table(object):
    def __init__(self, *tables):
        self.tables = tables or (type(self).__name__,)
        self.frame = None

    @staticmethod
    def read(table):
        cmd = ['mdb-export', '/data/StanhopeFramers.mdb', table]
        out = subprocess.check_output(cmd)
        return io.BytesIO(out)

    def load(self):
        frame = pandas.DataFrame()
        for table in self.tables:
            page = pandas.read_csv(self.read(table), **self.READ_CSV)
            # page['Table'] = table
            frame = frame.append(page)
        self.frame = frame
        return frame

class Customers(Table):
    READ_CSV = {
        'converters': {
            'Customer Number': utils.upper,
            'Credit': utils.boolean,
            'Tax Exempt': utils.boolean,
            'Deceased': utils.boolean},
        'parse_dates': [
            'Date',
            'Last Order',
            'Last Update']}

    def accounts(self):
        frame = self.frame.copy()

        # Copy legacy record
        frame['Legacy Customer Record'] = \
            frame.apply(utils.legacy_customer_record, axis=1)

        # Drop unused columns
        frame.drop(['Address',
                    'City',
                    'Date',
                    'Deceased',
                    'Email',
                    'Last Order',
                    'Last Update',
                    'State',
                    'Telephone',
                    'Zip'],
                   axis=1,
                   inplace=True)

        # Rename columns
        frame.rename(inplace=True,
                     columns={'Customer Number': 'Legacy Customer Number',
                              'Name': 'Account',
                              'Comment': 'Comments'})

        # Set account name
        frame.loc[:, 'Account'] = \
            frame['Account'].apply(utils.replace_newline)\
                            .combine_first(frame['Legacy Customer Number'])

        # Massage fields
        frame.loc[~frame['Category'].isnull(), 'Category'] = \
            frame.loc[~frame['Category'].isnull(), 'Category'].apply(
                utils.account_category)
        frame.loc[~frame['Source'].isnull(), 'Source'] = \
            frame.loc[~frame['Source'].isnull(), 'Source'].apply(utils.source)

        # Return
        return frame

    def contacts(self):
        frame = self.frame.copy()

        # Drop unused columns
        frame.drop(['Category',
                    'Comment',
                    'Credit',
                    'Date',
                    'Last Order',
                    'Last Update',
                    'Source',
                    'Tax Exempt'],
                   axis=1,
                   inplace=True)

        # Rename columns
        frame.rename(inplace=True,
                     columns={'Customer Number': 'Account Link',
                              'Name': 'Contact'})

        # Massage fields
        frame.loc[:, 'Contact'] = \
            frame['Contact'].apply(utils.replace_newline)\
                            .combine_first(frame['Account Link'])
        frame.loc[:, 'Contact'] = frame['Contact'].apply(utils.replace_newline)
        frame.loc[:, 'Address'] = frame['Address'].apply(utils.replace_newline)
        frame.loc[:, 'City'] = frame['City'].apply(utils.replace_newline)
        frame.loc[:, 'State'] = frame['State'].apply(utils.replace_newline)
        frame.loc[:, 'Zip'] = frame['Zip'].apply(utils.replace_newline)
        frame.loc[:, 'Telephone'] = \
            frame['Telephone'].apply(utils.replace_newline)

        # Return
        return frame

class FrameOrders(Table):
    READ_CSV = {
        'converters': {'CustomerNo': utils.upper, 'OrderNo': utils.upper},
        'parse_dates': ['DateCompleted', 'DueDate', 'OrderDate']
    }

    def orders(self):
        frame = self.frame.copy()

        # Copy legacy record
        frame['Legacy Order Record'] = \
            frame.apply(utils.legacy_order_record, axis=1)

        # Add Legacy Order ID
        frame['Legacy Order ID'] = frame.apply(utils.legacy_order_id, axis=1)

        # Drop unused columns
        frame.drop(['Artist',
                    'BinNo',
                    'Comments',
                    'DateCompleted',
                    'Fitting',
                    'Frame Height',
                    'Frame Width',
                    'FrameMfg',
                    'FrameNo',
                    'Glazing',
                    'Joining',
                    'Mat',
                    'MatColor',
                    'MatMfg',
                    'Matting',
                    'MattingSize',
                    'ProductionComments',
                    'Qty',
                    'SalesCatgy',
                    'SalesType',
                    'TotalSale'],
                   axis=1,
                   inplace=True)

        # Rename columns
        frame.rename(inplace=True,
                     columns={'OrderNo': 'Order Number',
                              'OrderDate': 'Order Date',
                              'DueDate': 'Due Date',
                              'CustomerNo': 'Account Link',
                              'Status': 'Order Status',
                              'Location': 'Order Location',
                              'SalesPers': 'Salesperson Link',
                              'Delivery': 'Delivery Location',
                              'Cust-Client': 'Client'})

        # Massage fields
        frame.loc[:, 'Delivery Location'] = \
            frame['Delivery Location'].apply(utils.delivery_location)
        frame.loc[:, 'Discount'] = frame['Discount'].apply(utils.discount)
        frame.loc[:, 'Order Location'] = \
            frame['Order Location'].apply(utils.order_location)
        frame.loc[:, 'Order Status'] = \
            frame['Order Status'].apply(utils.status)
        frame.loc[:, 'Salesperson Link'] = \
            frame['Salesperson Link'].apply(utils.salesperson)
        frame.loc[frame['Discount'].isnull(), 'Discount'] = 'No Discount'
        frame.loc[:, 'Delivery Location'] = \
            frame['Delivery Location'].combine_first(frame['Order Location'])
        frame.loc[:, 'Client'] = frame['Client'].fillna('None')

        # Return
        return frame

    def treatments(self):
        frame = self.frame.copy()

        # Add Legacy Order ID
        frame['Order Link'] = frame.apply(utils.legacy_order_id, axis=1)

        # Drop unused columns
        frame.drop(['Artist',
                    'Cust-Client',
                    'DateCompleted',
                    'Delivery',
                    'Discount',
                    'DueDate',
                    'Fitting',
                    'FrameMfg',
                    'Glazing',
                    'Location',
                    'Mat',
                    'Matting',
                    'OrderDate',
                    'OrderNo',
                    'SalesCatgy',
                    'SalesPers',
                    'Status'],
                   axis=1,
                   inplace=True)

        # Rename fields
        frame.rename(inplace=True,
                     columns={'BinNo': 'Bin Number',
                              'Comments': 'Description',
                              'CustomerNo': 'Account Link',
                              'FrameNo': 'Frame Style',
                              'Joining': 'Frame Join',
                              'MatColor': 'Mat Color',
                              'MatMfg': 'Mat Manufacturer',
                              'MattingSize': 'Mat Size',
                              'ProductionComments': 'Production Comments',
                              'Qty': 'Quantity',
                              'SalesType': 'Type',
                              'TotalSale': 'Price'})

        # Massage fields
        frame.loc[:, 'Frame Join'] = frame['Frame Join'].apply(utils.join)
        frame.loc[:, 'Mat Manufacturer'] = \
            frame['Mat Manufacturer'].apply(utils.matmfg)
        frame.loc[:, 'Type'] = frame['Type'].apply(utils.sales_type)

        # Add dimensions
        frame['Frame Width Inches'] = \
            frame['Frame Width'].apply(utils.inches)
        frame['Frame Width Fraction'] = \
            frame['Frame Width'].apply(utils.fraction)
        frame['Frame Height Inches'] = \
            frame['Frame Height'].apply(utils.inches)
        frame['Frame Height Fraction'] = \
            frame['Frame Height'].apply(utils.fraction)
        frame.drop(['Frame Width', 'Frame Height'], axis=1, inplace=True)

        # Return
        return frame
<commit_msg>Deceased -> Removed<commit_after>"""
StanhopeFramers Tables
"""
import io
import subprocess

import pandas
from stanhope import utils

pandas.set_option('display.max_rows', 999)
pandas.set_option('display.width', 999)
pandas.set_option('display.max_colwidth', 999)

class Table(object):
    def __init__(self, *tables):
        self.tables = tables or (type(self).__name__,)
        self.frame = None

    @staticmethod
    def read(table):
        cmd = ['mdb-export', '/data/StanhopeFramers.mdb', table]
        out = subprocess.check_output(cmd)
        return io.BytesIO(out)

    def load(self):
        frame = pandas.DataFrame()
        for table in self.tables:
            page = pandas.read_csv(self.read(table), **self.READ_CSV)
            # page['Table'] = table
            frame = frame.append(page)
        self.frame = frame
        return frame

class Customers(Table):
    READ_CSV = {
        'converters': {
            'Customer Number': utils.upper,
            'Credit': utils.boolean,
            'Tax Exempt': utils.boolean,
            'Deceased': utils.boolean},
        'parse_dates': [
            'Date',
            'Last Order',
            'Last Update']}

    def accounts(self):
        frame = self.frame.copy()

        # Copy legacy record
        frame['Legacy Customer Record'] = \
            frame.apply(utils.legacy_customer_record, axis=1)

        # Drop unused columns
        frame.drop(['Address',
                    'City',
                    'Date',
                    'Deceased',
                    'Email',
                    'Last Order',
                    'Last Update',
                    'State',
                    'Telephone',
                    'Zip'],
                   axis=1,
                   inplace=True)

        # Rename columns
        frame.rename(inplace=True,
                     columns={'Customer Number': 'Legacy Customer Number',
                              'Name': 'Account',
                              'Comment': 'Comments'})

        # Set account name
        frame.loc[:, 'Account'] = \
            frame['Account'].apply(utils.replace_newline)\
                            .combine_first(frame['Legacy Customer Number'])

        # Massage fields
        frame.loc[~frame['Category'].isnull(), 'Category'] = \
            frame.loc[~frame['Category'].isnull(), 'Category'].apply(
                utils.account_category)
        frame.loc[~frame['Source'].isnull(), 'Source'] = \
            frame.loc[~frame['Source'].isnull(), 'Source'].apply(utils.source)

        # Return
        return frame

    def contacts(self):
        frame = self.frame.copy()

        # Drop unused columns
        frame.drop(['Category',
                    'Comment',
                    'Credit',
                    'Date',
                    'Last Order',
                    'Last Update',
                    'Source',
                    'Tax Exempt'],
                   axis=1,
                   inplace=True)

        # Rename columns
        frame.rename(inplace=True,
                     columns={'Customer Number': 'Account Link',
                              'Name': 'Contact',
                              'Deceased': 'Removed'})

        # Massage fields
        frame.loc[:, 'Contact'] = \
            frame['Contact'].apply(utils.replace_newline)\
                            .combine_first(frame['Account Link'])
        frame.loc[:, 'Contact'] = frame['Contact'].apply(utils.replace_newline)
        frame.loc[:, 'Address'] = frame['Address'].apply(utils.replace_newline)
        frame.loc[:, 'City'] = frame['City'].apply(utils.replace_newline)
        frame.loc[:, 'State'] = frame['State'].apply(utils.replace_newline)
        frame.loc[:, 'Zip'] = frame['Zip'].apply(utils.replace_newline)
        frame.loc[:, 'Telephone'] = \
            frame['Telephone'].apply(utils.replace_newline)

        # Return
        return frame

class FrameOrders(Table):
    READ_CSV = {
        'converters': {'CustomerNo': utils.upper, 'OrderNo': utils.upper},
        'parse_dates': ['DateCompleted', 'DueDate', 'OrderDate']
    }

    def orders(self):
        frame = self.frame.copy()

        # Copy legacy record
        frame['Legacy Order Record'] = \
            frame.apply(utils.legacy_order_record, axis=1)

        # Add Legacy Order ID
        frame['Legacy Order ID'] = frame.apply(utils.legacy_order_id, axis=1)

        # Drop unused columns
        frame.drop(['Artist',
                    'BinNo',
                    'Comments',
                    'DateCompleted',
                    'Fitting',
                    'Frame Height',
                    'Frame Width',
                    'FrameMfg',
                    'FrameNo',
                    'Glazing',
                    'Joining',
                    'Mat',
                    'MatColor',
                    'MatMfg',
                    'Matting',
                    'MattingSize',
                    'ProductionComments',
                    'Qty',
                    'SalesCatgy',
                    'SalesType',
                    'TotalSale'],
                   axis=1,
                   inplace=True)

        # Rename columns
        frame.rename(inplace=True,
                     columns={'OrderNo': 'Order Number',
                              'OrderDate': 'Order Date',
                              'DueDate': 'Due Date',
                              'CustomerNo': 'Account Link',
                              'Status': 'Order Status',
                              'Location': 'Order Location',
                              'SalesPers': 'Salesperson Link',
                              'Delivery': 'Delivery Location',
                              'Cust-Client': 'Client'})

        # Massage fields
        frame.loc[:, 'Delivery Location'] = \
            frame['Delivery Location'].apply(utils.delivery_location)
        frame.loc[:, 'Discount'] = frame['Discount'].apply(utils.discount)
        frame.loc[:, 'Order Location'] = \
            frame['Order Location'].apply(utils.order_location)
        frame.loc[:, 'Order Status'] = \
            frame['Order Status'].apply(utils.status)
        frame.loc[:, 'Salesperson Link'] = \
            frame['Salesperson Link'].apply(utils.salesperson)
        frame.loc[frame['Discount'].isnull(), 'Discount'] = 'No Discount'
        frame.loc[:, 'Delivery Location'] = \
            frame['Delivery Location'].combine_first(frame['Order Location'])
        frame.loc[:, 'Client'] = frame['Client'].fillna('None')

        # Return
        return frame

    def treatments(self):
        frame = self.frame.copy()

        # Add Legacy Order ID
        frame['Order Link'] = frame.apply(utils.legacy_order_id, axis=1)

        # Drop unused columns
        frame.drop(['Artist',
                    'Cust-Client',
                    'DateCompleted',
                    'Delivery',
                    'Discount',
                    'DueDate',
                    'Fitting',
                    'FrameMfg',
                    'Glazing',
                    'Location',
                    'Mat',
                    'Matting',
                    'OrderDate',
                    'OrderNo',
                    'SalesCatgy',
                    'SalesPers',
                    'Status'],
                   axis=1,
                   inplace=True)

        # Rename fields
        frame.rename(inplace=True,
                     columns={'BinNo': 'Bin Number',
                              'Comments': 'Description',
                              'CustomerNo': 'Account Link',
                              'FrameNo': 'Frame Style',
                              'Joining': 'Frame Join',
                              'MatColor': 'Mat Color',
                              'MatMfg': 'Mat Manufacturer',
                              'MattingSize': 'Mat Size',
                              'ProductionComments': 'Production Comments',
                              'Qty': 'Quantity',
                              'SalesType': 'Type',
                              'TotalSale': 'Price'})

        # Massage fields
        frame.loc[:, 'Frame Join'] = frame['Frame Join'].apply(utils.join)
        frame.loc[:, 'Mat Manufacturer'] = \
            frame['Mat Manufacturer'].apply(utils.matmfg)
        frame.loc[:, 'Type'] = frame['Type'].apply(utils.sales_type)

        # Add dimensions
        frame['Frame Width Inches'] = \
            frame['Frame Width'].apply(utils.inches)
        frame['Frame Width Fraction'] = \
            frame['Frame Width'].apply(utils.fraction)
        frame['Frame Height Inches'] = \
            frame['Frame Height'].apply(utils.inches)
        frame['Frame Height Fraction'] = \
            frame['Frame Height'].apply(utils.fraction)
        frame.drop(['Frame Width', 'Frame Height'], axis=1, inplace=True)

        # Return
        return frame
<|endoftext|>


<commit_before>import os
import sys
import numpy
import pandas
import csv
import feature_reduction
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from nltk.corpus import stopwords
from nltk import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.stem.snowball import EnglishStemmer
from sklearn.pipeline import Pipeline

trainingData = pandas.DataFrame({'text': [], 'class': []})
with open('training.csv') as csvfile:
	reader = csv.DictReader(csvfile, delimiter=';')
	for row in reader:
		if row['polarity'] in ('0', '4'):
			trainingData = trainingData.append(pandas.DataFrame({'text': [feature_reduction.reduce(row['tweet'])], 'class': [row['polarity']]}, index=[row['id']]))

#stanford test
testData = pandas.DataFrame({'text': [], 'class': []})
with open('testing.csv') as csvfile:
	reader = csv.DictReader(csvfile, fieldnames=('polarity', 'id', 'date', 'query', 'user', 'tweet'))
	for row in reader:
		if row['polarity'] in ('0', '4'):
			testData = testData.append(pandas.DataFrame({'text': [feature_reduction.reduce(row['tweet'])], 'class': [row['polarity']]}, index=[row['id']]))

class StemTokenizer(object):
	def __init__(self):
		self.stemmer = EnglishStemmer()
	def __call__(self, doc):
		return [self.stemmer.stem(t) for t in word_tokenize(doc)]

stop_words = stopwords.words('english')
stop_words.extend([feature_reduction.user_token.lower(), feature_reduction.url_token.lower()]);

pipe = Pipeline([
					('counter', CountVectorizer(ngram_range=(1, 1), stop_words=stop_words, tokenizer=StemTokenizer())),
					#('normalizer', TfidfTransformer(smooth_idf=True, sublinear_tf=False, use_idf=True)), #should we want to use a TfidfTransformer
					('classifier', MultinomialNB(fit_prior=False, alpha=1.0))
				])

#applies fit and transform to first parameter at all stages but the last, where only fit is applied
pipe.fit(numpy.asarray(trainingData['text']), numpy.asarray(trainingData['class']))

#applies transform at all stages but the last one where predict is applied
predictions = pipe.predict(numpy.asarray(testData['text']))

print("Processed predictions: " + str(len(predictions)))
count = 0
for (i, sentiment) in enumerate(numpy.asarray(testData['class'])):
	#print(str(predictions[i]) + ": " + sentiment)
	if predictions[i] == sentiment:
		count = count + 1

print("count: " + str(count))
print("predictions: " + str(len(predictions)))
print(count / float(len(predictions)))
#print(predictions)<commit_msg>Added basic SGDClassifier<commit_after>import os
import sys
import numpy
import pandas
import csv
import feature_reduction
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import SGDClassifier
from nltk.corpus import stopwords
from nltk import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.stem.snowball import EnglishStemmer
from sklearn.pipeline import Pipeline

trainingData = pandas.DataFrame({'text': [], 'class': []})
with open('training.csv') as csvfile:
	reader = csv.DictReader(csvfile, delimiter=';')
	for row in reader:
		if row['polarity'] in ('0', '4'):
			trainingData = trainingData.append(pandas.DataFrame({'text': [feature_reduction.reduce(row['tweet'])], 'class': [row['polarity']]}, index=[row['id']]))

#stanford test
testData = pandas.DataFrame({'text': [], 'class': []})
with open('testing.csv') as csvfile:
	reader = csv.DictReader(csvfile, fieldnames=('polarity', 'id', 'date', 'query', 'user', 'tweet'))
	for row in reader:
		if row['polarity'] in ('0', '4'):
			testData = testData.append(pandas.DataFrame({'text': [feature_reduction.reduce(row['tweet'])], 'class': [row['polarity']]}, index=[row['id']]))

class StemTokenizer(object):
	def __init__(self):
		self.stemmer = EnglishStemmer()
	def __call__(self, doc):
		return [self.stemmer.stem(t) for t in word_tokenize(doc)]

stop_words = stopwords.words('english')
stop_words.extend([feature_reduction.user_token.lower(), feature_reduction.url_token.lower()]);

pipe = Pipeline([
					('counter', CountVectorizer(ngram_range=(1, 1), stop_words=stop_words, tokenizer=StemTokenizer())),
					#('normalizer', TfidfTransformer(smooth_idf=True, sublinear_tf=False, use_idf=True)), #should we want to use a TfidfTransformer
					('classifier', MultinomialNB(fit_prior=False, alpha=1.0))
					#('classifier', SGDClassifier())
				])

#applies fit and transform to first parameter at all stages but the last, where only fit is applied
pipe.fit(numpy.asarray(trainingData['text']), numpy.asarray(trainingData['class']))

#applies transform at all stages but the last one where predict is applied
predictions = pipe.predict(numpy.asarray(testData['text']))

print("Processed predictions: " + str(len(predictions)))
count = 0
for (i, sentiment) in enumerate(numpy.asarray(testData['class'])):
	#print(str(predictions[i]) + ": " + sentiment)
	if predictions[i] == sentiment:
		count = count + 1

print("count: " + str(count))
print("predictions: " + str(len(predictions)))
print(count / float(len(predictions)))
#print(predictions)<|endoftext|>