<commit_before>"""
Author: Michel Rouly
Author: Joshua Wells
Date:   2014-04-28
"""

import sys
import logging
from time import time

from sklearn import metrics
from sklearn import cluster

from vectorize_data import vectorize_data

# Run clustering algorithm {{{
def run_clustering( clusterer, data, labels ):
    """
    Cluster: Using a predefined and parameterized clustering algorithm, fit
    some dataset and perform metrics given a set of ground-truth labels.

        clusterer: the clustering algorithm, from sklearn
        data:      array-like dataset input
        labels:    vector of ground-truth labels

    """

    # Time the operation
    t0 = time()
    clusterer.fit(data)
    t1 = time()

    # Perform metrics
    runtime         = (t1 - t0)
    homogeneity     = metrics.homogeneity_score(   labels, clusterer.labels_ )
    completeness    = metrics.completeness_score(  labels, clusterer.labels_ )
    v_measure       = metrics.v_measure_score(     labels, clusterer.labels_ )
    adjusted_rand   = metrics.adjusted_rand_score( labels, clusterer.labels_ )
    adjusted_mutual = metrics.adjusted_mutual_info_score( labels,
                                                          clusterer.labels_ )

    # Output to logs
    logging.info(" |-        Execution time: %fs"   % runtime)
    logging.info(" |-           Homogeneity: %0.3f" % homogeneity)
    logging.info(" |-          Completeness: %0.3f" % completeness)
    logging.info(" |-             V-measure: %0.3f" % v_measure)
    logging.info(" |-   Adjusted Rand-Index: %.3f"  % adjusted_rand)
    logging.info(" |-  Adjusted Mutual Info: %.3f"  % adjusted_mutual)
# }}}

# Perform KMeans {{{
def do_kmeans( data, labels ):
    """
    Do KMeans: perform KMeans clustering on an input corpus. Input is
    expected to be a dictionary of categories to tf-idf vectors.
    """

    km = cluster.MiniBatchKMeans(
        n_clusters=3,       # expected number of clusters
        init="k-means++",   # initialization method (smart)
        n_init=5,           # number of random retries
        #init_size=1000,
        batch_size=1000
    )

    logging.info("Beginning KMeans clustering.")

    run_clustering( km, data, labels )
# }}}

# Perform Affinity Propagation {{{
def do_affinity_propagation( data, labels ):
    """
    Do Affinity Propagation: perform Affinity Propagation clustering on an
    input corpus.  Input is expected to be a dictionary of categories to
    tf-idf vectors.
    """

    ap = cluster.AffinityPropagation(
        damping=0.5,            # damping factor
        convergence_iter=15,    # number of no-change iterations to converge
        affinity="euclidean",   # similarity metric
    )

    logging.info("Beginning Affinity Propagation clustering.")

    run_clustering( ap, data, labels )
# }}}

# Perform Mean Shift {{{
def do_mean_shift( data, labels ):
    """
    Do Mean Shift: perform Mean Shift clustering on an input corpus.  Input
    is expected to be a dictionary of categories to tf-idf vectors.
    """

    ms = cluster.MeanShift(
        #bandwidth=,
        #seeds=,
        min_bin_freq=1,     # only use bins with at least min frequency
        cluster_all=True,   # use all points
    )

    logging.info("Beginning Mean Shift clustering.")
    logging.warn("Meanshift is not a scalable clustering algorithm.")

    data = data.toarray()
    run_clustering( ms, data, labels )
# }}}

# Perform Spectral Clustering {{{
def do_spectral( data, labels ):
    """
    Do Spectral: perform Spectral clustering on an input corpus.  Input is
    expected to be a dictionary of categories to tf-idf vectors.
    """

    sc = cluster.SpectralClustering(
        n_clusters=2,               # expected number of clusters
        eigen_solver="arpack",      # eigenvalue decomposition strategy
        assign_labels="discretize", # label assignment strategy
    )

    logging.info("Beginning Spectral clustering.")

    run_clustering( sc, data, labels )
# }}}

# Perform Ward's Hierarchical Clustering {{{
def do_wards( data, labels ):
    """
    Do Ward's Hierarchical: perform Ward's Hierarchical clustering on an
    input corpus.  Input is expected to be a dictionary of categories to
    tf-idf vectors.
    """

    wh = cluster.Ward(
        n_clusters=3,               # expected number of clusters
        connectivity=None,          # no connectivity matrix
    )

    logging.info("Beginning Ward's Hierarhical clustering.")

    data = data.toarray()
    run_clustering( wh, data, labels )
# }}}

# TODO: Perform DBSCAN {{{
def do_dbscan( data, labels ):
    """
    Do DBSCAN: perform DBSCAN clustering on an input corpus.
    Input is expected to be a dictionary of categories to tf-idf vectors.
    """

    #db = cluster.

    logging.info("Beginning DBSCAN clustering.")
    logging.error("DBSCAN is not yet implemented.")

    return
# }}}

# TODO: Perform Gaussian {{{
def do_gaussian( data, labels ):
    """
    Do Gaussian: perform Gaussian Mixture Modeling clustering on an input
    corpus.  Input is expected to be a dictionary of categories to tf-idf
    vectors.
    """

    logging.info("Beginning Gaussian Mixture Modeling clustering.")
    logging.error("Gaussian Mixter Modeling is not yet implemented.")

    return
# }}}

# Executable (Main) {{{
if __name__ == "__main__":

    # turn on logging
    logging.basicConfig(level=logging.DEBUG,
                        format='%(levelname)s: %(message)s')

    # algorithm options
    options = "kmeans|ap|meanshift|spectral|wards|dbscan|gaussian"

    # ensure we have at least the minimum required params
    if len( sys.argv ) < 4:
        logging.error( "Usage: python cluster.py [s3|disk] data_dir [%s]" %
                options )
        sys.exit( 1 )

    # pull in parameters from the command line
    in_protocol = sys.argv[1]
    data_dir = sys.argv[2]
    requested_algorithms = sys.argv[3:]

    # available algorithms
    known_algorithms = ["kmeans",
                        "ap",
                        "meanshift",
                        "spectral",
                        "wards",
                        "dbscan",
                        "gaussian"]

    # read and vectorize the data
    (labels, data) = vectorize_data( in_protocol=in_protocol,
                                    data_dir=data_dir )

    # loop over all requested algorithms
    for algorithm in requested_algorithms:

        # verify that requested algorithm is known
        if algorithm not in known_algorithms:
            logging.error( "Algorithm \"%s\" is not recognized." % algorithm )

        if algorithm == "kmeans":
            do_kmeans( data, labels )

        if algorithm == "ap":
            do_affinity_propagation( data, labels )

        if algorithm == "meanshift":
            do_mean_shift( data, labels )

        if algorithm == "spectral":
            do_spectral( data, labels )

        if algorithm == "wards":
            do_wards( data, labels )

        if algorithm == "dbscan":
            do_dbscan( data, labels )

        if algorithm == "gaussian":
            do_gaussian( data, labels )
# }}}

<commit_msg>Implemented DBSCAN<commit_after>"""
Author: Michel Rouly
Author: Joshua Wells
Date:   2014-04-28
"""

import sys
import logging
from time import time

from sklearn import metrics
from sklearn import cluster

from vectorize_data import vectorize_data

# Run clustering algorithm {{{
def run_clustering( clusterer, data, labels ):
    """
    Cluster: Using a predefined and parameterized clustering algorithm, fit
    some dataset and perform metrics given a set of ground-truth labels.

        clusterer: the clustering algorithm, from sklearn
        data:      array-like dataset input
        labels:    vector of ground-truth labels

    """

    # Time the operation
    t0 = time()
    clusterer.fit(data)
    t1 = time()

    # Perform metrics
    runtime         = (t1 - t0)
    homogeneity     = metrics.homogeneity_score(   labels, clusterer.labels_ )
    completeness    = metrics.completeness_score(  labels, clusterer.labels_ )
    v_measure       = metrics.v_measure_score(     labels, clusterer.labels_ )
    adjusted_rand   = metrics.adjusted_rand_score( labels, clusterer.labels_ )
    adjusted_mutual = metrics.adjusted_mutual_info_score( labels,
                                                          clusterer.labels_ )

    # Output to logs
    logging.info("  |-        Execution time: %fs"   % runtime)
    logging.info("  |-           Homogeneity: %0.3f" % homogeneity)
    logging.info("  |-          Completeness: %0.3f" % completeness)
    logging.info("  |-             V-measure: %0.3f" % v_measure)
    logging.info("  |-   Adjusted Rand-Index: %.3f"  % adjusted_rand)
    logging.info("  |-  Adjusted Mutual Info: %.3f"  % adjusted_mutual)
# }}}

# Perform KMeans {{{
def do_kmeans( data, labels ):
    """
    Do KMeans: perform KMeans clustering on an input corpus. Input is
    expected to be a dictionary of categories to tf-idf vectors.
    """

    km = cluster.MiniBatchKMeans(
        n_clusters=3,       # expected number of clusters
        init="k-means++",   # initialization method (smart)
        n_init=5,           # number of random retries
        #init_size=1000,
        batch_size=1000
    )

    logging.info("Beginning KMeans clustering.")

    run_clustering( km, data, labels )
# }}}

# Perform Affinity Propagation {{{
def do_affinity_propagation( data, labels ):
    """
    Do Affinity Propagation: perform Affinity Propagation clustering on an
    input corpus.  Input is expected to be a dictionary of categories to
    tf-idf vectors.
    """

    ap = cluster.AffinityPropagation(
        damping=0.5,            # damping factor
        convergence_iter=15,    # number of no-change iterations to converge
        affinity="euclidean",   # similarity metric
    )

    logging.info("Beginning Affinity Propagation clustering.")

    run_clustering( ap, data, labels )
# }}}

# Perform Mean Shift {{{
def do_mean_shift( data, labels ):
    """
    Do Mean Shift: perform Mean Shift clustering on an input corpus.  Input
    is expected to be a dictionary of categories to tf-idf vectors.
    """

    ms = cluster.MeanShift(
        #bandwidth=,
        #seeds=,
        min_bin_freq=1,     # only use bins with at least min frequency
        cluster_all=True,   # use all points
    )

    logging.info("Beginning Mean Shift clustering.")
    logging.warn("Meanshift is not a scalable clustering algorithm.")

    data = data.toarray()
    run_clustering( ms, data, labels )
# }}}

# Perform Spectral Clustering {{{
def do_spectral( data, labels ):
    """
    Do Spectral: perform Spectral clustering on an input corpus.  Input is
    expected to be a dictionary of categories to tf-idf vectors.
    """

    sc = cluster.SpectralClustering(
        n_clusters=2,               # expected number of clusters
        eigen_solver="arpack",      # eigenvalue decomposition strategy
        assign_labels="discretize", # label assignment strategy
    )

    logging.info("Beginning Spectral clustering.")

    run_clustering( sc, data, labels )
# }}}

# Perform Ward's Hierarchical Clustering {{{
def do_wards( data, labels ):
    """
    Do Ward's Hierarchical: perform Ward's Hierarchical clustering on an
    input corpus.  Input is expected to be a dictionary of categories to
    tf-idf vectors.
    """

    wh = cluster.Ward(
        n_clusters=3,               # expected number of clusters
        connectivity=None,          # no connectivity matrix
    )

    logging.info("Beginning Ward's Hierarhical clustering.")

    data = data.toarray()
    run_clustering( wh, data, labels )
# }}}

# Perform DBSCAN {{{
def do_dbscan( data, labels ):
    """
    Do DBSCAN: perform DBSCAN clustering on an input corpus.
    Input is expected to be a dictionary of categories to tf-idf vectors.
    """

    db = cluster.DBSCAN(
        eps=0.01,          # max distance between two neighbours
        min_samples=1,  # number of neighbors for a core point
        #metric=,       #
        #random_state=, #
    )

    logging.info("Beginning DBSCAN clustering.")

    data = data.toarray()
    run_clustering( db, data, labels )
# }}}

# TODO: Perform Gaussian {{{
def do_gaussian( data, labels ):
    """
    Do Gaussian: perform Gaussian Mixture Modeling clustering on an input
    corpus.  Input is expected to be a dictionary of categories to tf-idf
    vectors.
    """

    logging.info("Beginning Gaussian Mixture Modeling clustering.")
    logging.error("Gaussian Mixter Modeling is not yet implemented.")

    return
# }}}

# Executable (Main) {{{
if __name__ == "__main__":

    # turn on logging
    logging.basicConfig(level=logging.DEBUG,
                        format='%(levelname)s: %(message)s')

    # algorithm options
    options = "kmeans|ap|meanshift|spectral|wards|dbscan|gaussian"

    # ensure we have at least the minimum required params
    if len( sys.argv ) < 4:
        logging.error( "Usage: python cluster.py [s3|disk] data_dir [%s]" %
                options )
        sys.exit( 1 )

    # pull in parameters from the command line
    in_protocol = sys.argv[1]
    data_dir = sys.argv[2]
    requested_algorithms = sys.argv[3:]

    # available algorithms
    known_algorithms = ["kmeans",
                        "ap",
                        "meanshift",
                        "spectral",
                        "wards",
                        "dbscan",
                        "gaussian"]

    # read and vectorize the data
    (labels, data) = vectorize_data( in_protocol=in_protocol,
                                    data_dir=data_dir )

    # loop over all requested algorithms
    for algorithm in requested_algorithms:

        # verify that requested algorithm is known
        if algorithm not in known_algorithms:
            logging.error( "Algorithm \"%s\" is not recognized." % algorithm )

        if algorithm == "kmeans":
            do_kmeans( data, labels )

        if algorithm == "ap":
            do_affinity_propagation( data, labels )

        if algorithm == "meanshift":
            do_mean_shift( data, labels )

        if algorithm == "spectral":
            do_spectral( data, labels )

        if algorithm == "wards":
            do_wards( data, labels )

        if algorithm == "dbscan":
            do_dbscan( data, labels )

        if algorithm == "gaussian":
            do_gaussian( data, labels )
# }}}

<|endoftext|>


<commit_before>from rapidconnect import RapidConnect
from os import path
from pprint import pprint
import json

app_dir = path.dirname(__file__)
image_dir = path.join(app_dir, 'images')

# rapidAPI Initialize
rapid = RapidConnect('WasteZero', '158eb1cd-10fa-409b-b652-d0f9cd3e9ab3')

# Opening text files and spliting them into words
with open('compost.txt', 'r+') as f:
    compost = f.readlines()
    compost = [x.lower() for x in compost]
    compost = [x.strip() for x in compost]

with open('recyclables.txt', 'r+') as f:
    recycable = f.readlines()
    recycable = [x.lower() for x in recycable]
    recycable = [x.strip() for x in recycable]

with open('landfill.txt', 'r+') as f:
    landfill = f.readlines()
    landfill = [x.lower() for x in landfill]
    landfill = [x.strip() for x in landfill]

# Strawberry
url = """https://encrypted-tbn1.gstatic.com/images?q=tbn:ANd9GcTFPXRtq-D8ouFl3gZV88uT_0RzkI2xoL18jRhA-
    ng7H3zNVpky"""

# Pen
url2 = 'http://www.killercrossexamination.com/wp-content/uploads/2016/12/Blue-Bic-biro-pen.jpg'

# Water Botlle
url3 = "http://www.berkeleyside.com/wp-content/uploads/2011/03/plastic-water-bottle-281x360.jpg"

# Broccoli
url4 = "http://www.newkidscenter.com/images/10415723/broccoli.jpg"

# Cigarrete
url5 = "http://cdn.newsapi.com.au/image/v1/671c348120ffe64315ae2a67b627c98e?width=1024"

# Spoon
url6 = "https://upload.wikimedia.org/wikipedia/commons/3/32/Dessert_Spoon.jpg"

def sortImage(url):
    """
    Takes an image URL and returns its corresponding trash category.

    Args:
        url (str): an image url

    Returns:
        str: Image item's sorting category
    """
    tagImage = rapid.call('MicrosoftComputerVision', 'tagImage', {
                          'subscriptionKey':
                          '7bf3cea7fe0646ff83fddf20147a1fa5',
                          'image': url
                          })

    describeImage = rapid.call('MicrosoftComputerVision', 'describeImage', {
                               'subscriptionKey':
                               '7bf3cea7fe0646ff83fddf20147a1fa5',
                               'image': url,
                               'maxCandidates': ''
                               })

    tag = json.loads(tagImage)
    describe = json.loads(describeImage)

    pprint(tag['tags'])
    pprint(describe['description']['captions'][0]['text'])
    for i in range(len(tag['tags'])):
        if tag['tags'][i]['name'] in compost:
            print("ITEM: " + (tag['tags'][i]['name']).upper())
            print("DESCRIPTION: " + describe['description']['captions'][0]
                  ['text']
                  ).upper()
            return ("THIS ITEM GOES TO THE COMPOST BIN.")
            break
        elif tag['tags'][i]['name'] in recycable:
            print("ITEM: " + (tag['tags'][i]['name']).upper())
            print("DESCRIPTION: " + describe['description']['captions'][0]
                  ['text']
                  ).upper()
            return ("THIS ITEM GOES TO THE RECYCLING BIN.")
            break
        elif tag['tags'][i]['name'] in landfill:
            print("ITEM: " + (tag['tags'][i]['name']).upper())
            print("DESCRIPTION: " + describe['description']['captions'][0]
                  ['text']
                  ).upper()
            return ("THIS ITEM GOES TO THE LANDFILL BIN.")
            break
        if i == len(tag['tags']):
            print("THIS ITEM GOES TO THE LANDFILL BIN.")

print(sortImage(url))
<commit_msg>Improve accuracy<commit_after>from rapidconnect import RapidConnect
from os import path
from pprint import pprint
import json

app_dir = path.dirname(__file__)
image_dir = path.join(app_dir, 'images')

API_KEY = ''
# rapidAPI Initialize
rapid = RapidConnect('WasteZero', API_KEY)

# Opening text files and spliting them into words
with open('compost.txt', 'r+') as f:
    compost = f.readlines()
    compost = [x.lower() for x in compost]
    compost = [x.strip() for x in compost]

with open('recyclables.txt', 'r+') as f:
    recycable = f.readlines()
    recycable = [x.lower() for x in recycable]
    recycable = [x.strip() for x in recycable]

with open('landfill.txt', 'r+') as f:
    landfill = f.readlines()
    landfill = [x.lower() for x in landfill]
    landfill = [x.strip() for x in landfill]

# Strawberry
url = """https://encrypted-tbn1.gstatic.com/images?q=tbn:ANd9GcTFPXRtq-D8ouFl3gZV88uT_0RzkI2xoL18jRhA-
    ng7H3zNVpky"""

# Pen
url2 = 'http://www.killercrossexamination.com/wp-content/uploads/2016/12/Blue-Bic-biro-pen.jpg'

# Water Botlle
url3 = "http://www.berkeleyside.com/wp-content/uploads/2011/03/plastic-water-bottle-281x360.jpg"

# Broccoli
url4 = "http://www.newkidscenter.com/images/10415723/broccoli.jpg"

# Cigarrete
url5 = "http://cdn.newsapi.com.au/image/v1/671c348120ffe64315ae2a67b627c98e?width=1024"

# Spoon
url6 = "https://upload.wikimedia.org/wikipedia/commons/3/32/Dessert_Spoon.jpg"

# Orange
url7 = "https://d3nevzfk7ii3be.cloudfront.net/igi/rCjJcEgTAnWxY4Ck.large"

def sortImage(url):
    """
    Takes an image URL and returns its corresponding trash category.

    Args:
        url (str): an image url

    Returns:
        str: Image item's sorting category
    """
    tagImage = rapid.call('MicrosoftComputerVision', 'tagImage', {
                          'subscriptionKey':
                          API_KEY,
                          'image': url
                          })

    describeImage = rapid.call('MicrosoftComputerVision', 'describeImage', {
                               'subscriptionKey':
                               API_KEY,
                               'image': url,
                               'maxCandidates': ''
                               })

    tag = json.loads(tagImage)
    describe = json.loads(describeImage)

    pprint(tag['tags'])
    pprint(describe['description']['captions'][0]['text'])
    for i in range(len(tag['tags'])):
        if tag['tags'][i]['name'] in compost:
            print("ITEM: " + (tag['tags'][i]['name']).upper())
            print("DESCRIPTION: " + describe['description']['captions'][0]
                  ['text']
                  ).upper()
            return ("THIS ITEM GOES TO THE COMPOST BIN.")
            break
        elif tag['tags'][i]['name'] in recycable:
            print("ITEM: " + (tag['tags'][i]['name']).upper())
            print("DESCRIPTION: " + describe['description']['captions'][0]
                  ['text']
                  ).upper()
            return ("THIS ITEM GOES TO THE RECYCLING BIN.")
            break
        elif tag['tags'][i]['name'] in landfill:
            print("ITEM: " + (tag['tags'][i]['name']).upper())
            print("DESCRIPTION: " + describe['description']['captions'][0]
                  ['text']
                  ).upper()
            return ("THIS ITEM GOES TO THE LANDFILL BIN.")
            break
        if i == len(tag['tags']):
            print("THIS ITEM GOES TO THE LANDFILL BIN.")

print(sortImage("https://d3nevzfk7ii3be.cloudfront.net/igi/rCjJcEgTAnWxY4Ck.large"))
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Run a cross-validation experiment using RSMTool, RSMEval, and RSMSummarize.

The complete cross-validation workflow is as follows:

- Divide the training data file into folds either by randomly shuffling
  and then using a k-fold split or by using a pre-specified folds file
  that provides a fold number for each example ID in the data

- For each fold (in parallel), use rsmtool to run a train/test experiment
  and save all output in a separate directory for that fold under
  `<outdir>/folds/<num>`.

- Combine the predictions for each fold's test set and run rsmeval on that
  file. Save all output under `<outdir>/evaluation`.

- Run rsmsummarize on all of the fold directories and save output under
  `<outdir>/fold-summary`.

- Run rsmtool on the full training set to generate a final model along
  with a report including model/feature descriptives, all saved under
  `<outdir>/final-model`.

:author: Nitin Madnani (nmadnani@ets.org)
:organization: ETS
"""

import logging
from os import listdir, makedirs
import sys
from os.path import abspath, exists, join
from pathlib import Path

from joblib.parallel import Parallel, delayed
from tqdm import tqdm

from .configuration_parser import Configuration, configure
from .rsmeval import run_evaluation
from .rsmtool import run_experiment
from .rsmsummarize import run_summary
from .utils.commandline import ConfigurationGenerator, setup_rsmcmd_parser
from .utils.constants import VALID_PARSER_SUBCOMMANDS
from .utils.cross_validation import combine_fold_prediction_files, create_xval_files, process_fold
from .utils.logging import LogFormatter, tqdm_joblib, get_file_logger
from .writer import DataWriter

def run_cross_validation(config_file_or_obj_or_dict,
                         output_dir):
    """
    Run cross-validation experiment.

    Parameters
    ----------
    config_file_or_obj_or_dict : str or pathlib.Path or dict or Configuration
        Path to the experiment configuration file either a a string
        or as a ``pathlib.Path`` object. Users can also pass a
        ``Configuration`` object that is in memory or a Python dictionary
        with keys corresponding to fields in the configuration file. Given a
        configuration file, any relative paths in the configuration file
        will be interpreted relative to the location of the file. Given a
        ``Configuration`` object, relative paths will be interpreted
        relative to the ``configdir`` attribute, that _must_ be set. Given
        a dictionary, the reference path is set to the current directory.
    output_dir : str
        Path to the experiment output directory.
    silence_tqdm : bool, optional
        Whether to silence the progress bar that is shown when running
        rsmtool for each fold. This option should only be used when
        running the unit tests.
        Defaults to ``False``.

    Raises
    ------
    IOError
        If ``output_dir`` already contains the output of a previous experiment.

    """
    logger = logging.getLogger(__name__)

    # Get absolute paths to output directories
    foldsdir = abspath(join(output_dir, 'folds'))
    summarydir = abspath(join(output_dir, 'fold-summary'))
    evaldir = abspath(join(output_dir, 'evaluation'))
    modeldir = abspath(join(output_dir, 'final-model'))

    # Make directories, if necessary
    makedirs(foldsdir, exist_ok=True)
    makedirs(summarydir, exist_ok=True)
    makedirs(evaldir, exist_ok=True)
    makedirs(modeldir, exist_ok=True)

    # Raise an error if the specified output directory
    # already contains a non-empty `folds` directory;
    # we do not allow overwriting the output directory
    # for cross-validation experiments because of the
    # multitude of moving parts
    non_empty_foldsdir = exists(foldsdir) and listdir(foldsdir)
    if non_empty_foldsdir:
        raise IOError("'{}' already contains a non-empty 'folds' "
                      "directory.".format(output_dir))

    configuration = configure('rsmxval', config_file_or_obj_or_dict)

    logger.info("Saving configuration file.")
    with open(Path(output_dir) / "rsmxval.json", "w") as outfh:
        outfh.write(str(configuration))

    # create any cross-validation related files that are needed and
    # get the final number of folds that are to be used
    num_folds = create_xval_files(configuration, output_dir, logger=logger)

    # run RSMTool in parallel on each fold using joblib
    logger.info("Running RSMTool on each fold in parallel")
    with tqdm_joblib(tqdm(desc="Progress", total=num_folds, disable=silence_tqdm)):
        Parallel(n_jobs=num_folds)(delayed(process_fold)(fold_num, foldsdir)
                                   for fold_num in range(1, num_folds + 1))

    # generate an rsmsummarize configuration file
    logger.info("Creating fold summary")
    given_file_format = configuration.get("file_format")
    fold_summary_configdict = {
        "summary_id": f"{configuration['experiment_id']}_fold_summary",
        "experiment_dirs": [f"{foldsdir}/{fold_num:02}"
                            for fold_num in range(1, num_folds + 1)],
        "description": f"{configuration['description']} (Fold Summary)",
        "file_format": given_file_format,
        "use_thumbnails": f"{configuration['use_thumbnails']}"
    }
    fold_summary_configuration = Configuration(fold_summary_configdict,
                                               configdir=summarydir,
                                               context="rsmsummarize")

    # run rsmsummarize on all of the fold directories
    summary_logger = get_file_logger("fold-summary",
                                     Path(summarydir) / "rsmsummarize.log")
    run_summary(fold_summary_configuration, summarydir, False, logger=summary_logger)

    # combine all of the fold prediction files for evaluation
    df_predictions = combine_fold_prediction_files(foldsdir, given_file_format)
    predictions_file_prefix = str(Path(evaldir) / "xval_predictions")
    DataWriter.write_frame_to_file(df_predictions,
                                   predictions_file_prefix,
                                   file_format=given_file_format,
                                   index=False)

    # create a new rsmeval configuration dictionary;
    # note that we do not want to set "id_column" and "human_score_column"
    # here since those columns already have default names given that the
    # predictions are generated from RSMTool
    evaluation_configdict = {
        "experiment_id": f"{configuration['experiment_id']}_evaluation",
        "description": f"{configuration['description']} (Evaluating Fold Predictions)",
        "predictions_file": f"{predictions_file_prefix}.{given_file_format}",
        "system_score_column": "scale" if configuration["use_scaled_predictions"] else "raw",
        "scale_with": "asis" if configuration["use_scaled_predictions"] else "raw",
        "trim_min": int(f"{configuration['trim_min']}"),
        "trim_max": int(f"{configuration['trim_max']}"),
        "file_format": given_file_format
    }

    # copy over the relevant configuration fields from the main configuration
    for field_name in ["second_human_score_column",
                       "candidate_column",
                       "flag_column",
                       "exclude_zero_scores",
                       "min_items_per_candidate",
                       "rater_error_variance",
                       "subgroups",
                       "trim_tolerance",
                       "use_thumbnails"]:
        evaluation_configdict[field_name] = configuration[field_name]

    # create an rsmeval configuration object with this dictionary
    evaluation_configuration = Configuration(evaluation_configdict,
                                             configdir=evaldir,
                                             context="rsmeval")

    # run rsmeval on the combined predictions file
    logger.info("Evaluating combined fold predictions")
    eval_logger = get_file_logger("evaluation", Path(evaldir) / "rsmeval.log")
    run_evaluation(evaluation_configuration, evaldir, False, logger=eval_logger)

    # run rsmtool on the full dataset and generate only model/feature
    # descriptives report; we will use the dummy test set that we
    # created by calling `create_xval_files()` for this purpose
    logger.info("Training model on full data")
    model_logger = get_file_logger("final_model", Path(modeldir) / "rsmtool.log")
    final_rsmtool_configdict = configuration.to_dict().copy()
    final_rsmtool_configdict["test_file"] = str(Path(modeldir) / f"dummy_test.{given_file_format}")
    final_rsmtool_configdict["test_label_column"] = final_rsmtool_configdict["train_label_column"]
    final_rsmtool_configdict["general_sections"] = [
        "feature_descriptives",
        "preprocessed_features",
        "model",
        "intermediate_file_paths",
        "sysinfo"
    ]
    final_rsmtool_configuration = Configuration(final_rsmtool_configdict,
                                                configdir=configuration.configdir,
                                                context="rsmtool")
    run_experiment(final_rsmtool_configuration, modeldir, False, logger=model_logger)

def main():  # noqa: D103

    # set up the basic logging configuration
    formatter = LogFormatter()

    # we need two handlers, one that prints to stdout
    # for the "run" command and one that prints to stderr
    # from the "generate" command; the latter is important
    # because do not want the warning to show up in the
    # generated configuration file
    stdout_handler = logging.StreamHandler(sys.stdout)
    stdout_handler.setFormatter(formatter)

    stderr_handler = logging.StreamHandler(sys.stderr)
    stderr_handler.setFormatter(formatter)

    logging.root.setLevel(logging.INFO)
    logger = logging.getLogger(__name__)

    # set up an argument parser via our helper function
    parser = setup_rsmcmd_parser('rsmxval',
                                 uses_output_directory=True,
                                 allows_overwriting=False,
                                 uses_subgroups=True)

    # if we have no arguments at all then just show the help message
    if len(sys.argv) < 2:
        sys.argv.append("-h")

    # if the first argument is not one of the valid sub-commands
    # or one of the valid optional arguments, then assume that they
    # are arguments for the "run" sub-command. This allows the
    # old style command-line invocations to work without modification.
    if sys.argv[1] not in VALID_PARSER_SUBCOMMANDS + ['-h', '--help',
                                                      '-V', '--version']:
        args_to_pass = ['run'] + sys.argv[1:]
    else:
        args_to_pass = sys.argv[1:]
    args = parser.parse_args(args=args_to_pass)

    # call the appropriate function based on which sub-command was run
    if args.subcommand == 'run':

        # when running, log to stdout
        logging.root.addHandler(stdout_handler)

        # run the experiment
        logger.info('Output directory: {}'.format(args.output_dir))
        run_cross_validation(abspath(args.config_file),
                             abspath(args.output_dir))

    else:

        # when generating, log to stderr
        logging.root.addHandler(stderr_handler)

        # auto-generate an example configuration and print it to STDOUT
        generator = ConfigurationGenerator('rsmtool',
                                           as_string=True,
                                           suppress_warnings=args.quiet,
                                           use_subgroups=args.subgroups)
        configuration = generator.interact() if args.interactive else generator.generate()
        print(configuration)

if __name__ == '__main__':
    main()
<commit_msg>Ensure subgroups & consistency are properly handled<commit_after>#!/usr/bin/env python

"""
Run a cross-validation experiment using RSMTool, RSMEval, and RSMSummarize.

The complete cross-validation workflow is as follows:

- Divide the training data file into folds either by randomly shuffling
  and then using a k-fold split or by using a pre-specified folds file
  that provides a fold number for each example ID in the data

- For each fold (in parallel), use rsmtool to run a train/test experiment
  and save all output in a separate directory for that fold under
  `<outdir>/folds/<num>`.

- Combine the predictions for each fold's test set and run rsmeval on that
  file. Save all output under `<outdir>/evaluation`.

- Run rsmsummarize on all of the fold directories and save output under
  `<outdir>/fold-summary`.

- Run rsmtool on the full training set to generate a final model along
  with a report including model/feature descriptives, all saved under
  `<outdir>/final-model`.

:author: Nitin Madnani (nmadnani@ets.org)
:organization: ETS
"""

import logging
from os import listdir, makedirs
import sys
from os.path import abspath, exists, join
from pathlib import Path

from joblib.parallel import Parallel, delayed
from tqdm import tqdm

from .configuration_parser import Configuration, configure
from .rsmeval import run_evaluation
from .rsmtool import run_experiment
from .rsmsummarize import run_summary
from .utils.commandline import ConfigurationGenerator, setup_rsmcmd_parser
from .utils.constants import VALID_PARSER_SUBCOMMANDS
from .utils.cross_validation import combine_fold_prediction_files, create_xval_files, process_fold
from .utils.logging import LogFormatter, tqdm_joblib, get_file_logger
from .writer import DataWriter

# a constant defining all of the sections we can use when
# generating the final model report in the last stage
FINAL_MODEL_SECTION_LIST = ['feature_descriptives',
                            'features_by_group',
                            'preprocessed_features',
                            'dff_by_group',
                            'model',
                            'intermediate_file_paths',
                            'sysinfo']

def run_cross_validation(config_file_or_obj_or_dict, output_dir, silence_tqdm=False):
    """
    Run cross-validation experiment.

    Parameters
    ----------
    config_file_or_obj_or_dict : str or pathlib.Path or dict or Configuration
        Path to the experiment configuration file either a a string
        or as a ``pathlib.Path`` object. Users can also pass a
        ``Configuration`` object that is in memory or a Python dictionary
        with keys corresponding to fields in the configuration file. Given a
        configuration file, any relative paths in the configuration file
        will be interpreted relative to the location of the file. Given a
        ``Configuration`` object, relative paths will be interpreted
        relative to the ``configdir`` attribute, that _must_ be set. Given
        a dictionary, the reference path is set to the current directory.
    output_dir : str
        Path to the experiment output directory.
    silence_tqdm : bool, optional
        Whether to silence the progress bar that is shown when running
        rsmtool for each fold. This option should only be used when
        running the unit tests.
        Defaults to ``False``.

    Raises
    ------
    IOError
        If ``output_dir`` already contains the output of a previous experiment.

    """
    logger = logging.getLogger(__name__)

    # Get absolute paths to output directories
    foldsdir = abspath(join(output_dir, 'folds'))
    summarydir = abspath(join(output_dir, 'fold-summary'))
    evaldir = abspath(join(output_dir, 'evaluation'))
    modeldir = abspath(join(output_dir, 'final-model'))

    # Make directories, if necessary
    makedirs(foldsdir, exist_ok=True)
    makedirs(summarydir, exist_ok=True)
    makedirs(evaldir, exist_ok=True)
    makedirs(modeldir, exist_ok=True)

    # Raise an error if the specified output directory
    # already contains a non-empty `folds` directory;
    # we do not allow overwriting the output directory
    # for cross-validation experiments because of the
    # multitude of moving parts
    non_empty_foldsdir = exists(foldsdir) and listdir(foldsdir)
    if non_empty_foldsdir:
        raise IOError("'{}' already contains a non-empty 'folds' "
                      "directory.".format(output_dir))

    configuration = configure('rsmxval', config_file_or_obj_or_dict)

    logger.info("Saving configuration file.")
    with open(Path(output_dir) / "rsmxval.json", "w") as outfh:
        outfh.write(str(configuration))

    # create any cross-validation related files that are needed and
    # get a data frame containing the training set and the final
    # number of folds that are to be used in the experiment
    df_train, num_folds = create_xval_files(configuration, output_dir, logger=logger)

    # run RSMTool in parallel on each fold using joblib
    logger.info("Running RSMTool on each fold in parallel")
    with tqdm_joblib(tqdm(desc="Progress", total=num_folds, disable=silence_tqdm)):
        Parallel(n_jobs=num_folds)(delayed(process_fold)(fold_num, foldsdir)
                                   for fold_num in range(1, num_folds + 1))

    # generate an rsmsummarize configuration file
    logger.info("Creating fold summary")
    given_file_format = configuration.get("file_format")
    fold_summary_configdict = {
        "summary_id": f"{configuration['experiment_id']}_fold_summary",
        "experiment_dirs": [f"{foldsdir}/{fold_num:02}"
                            for fold_num in range(1, num_folds + 1)],
        "description": f"{configuration['description']} (Fold Summary)",
        "file_format": given_file_format,
        "use_thumbnails": f"{configuration['use_thumbnails']}"
    }
    fold_summary_configuration = Configuration(fold_summary_configdict,
                                               configdir=summarydir,
                                               context="rsmsummarize")

    # run rsmsummarize on all of the fold directories
    summary_logger = get_file_logger("fold-summary",
                                     Path(summarydir) / "rsmsummarize.log")
    run_summary(fold_summary_configuration, summarydir, False, logger=summary_logger)

    # combine all of the fold prediction files for evaluation
    df_predictions = combine_fold_prediction_files(foldsdir, given_file_format)

    # if there were subgroups and a second human score column specified, then
    # we need to add those to the combined predictions file as well
    id_column = configuration["id_column"]
    columns_to_use = [id_column]
    if subgroups := configuration["subgroups"]:
        columns_to_use.extend(subgroups)
    if second_human_score_column := configuration["second_human_score_column"]:
        columns_to_use.append(second_human_score_column)
    if len(columns_to_use) > 1:
        df_to_add = df_train[columns_to_use]
        df_predictions = df_predictions.merge(df_to_add,
                                              left_on="spkitemid",
                                              right_on=id_column)
        # drop any extra ID column if we have added one
        if id_column != "spkitemid":
            df_predictions.drop(id_column, axis=1, inplace=True)

    # write out the predictions file to disk
    predictions_file_prefix = str(Path(evaldir) / "xval_predictions")
    DataWriter.write_frame_to_file(df_predictions,
                                   predictions_file_prefix,
                                   file_format=given_file_format,
                                   index=False)

    # create a new rsmeval configuration dictionary;
    # note that we do not want to set "id_column" and "human_score_column"
    # here since those columns already have default names given that the
    # predictions are generated from RSMTool
    evaluation_configdict = {
        "experiment_id": f"{configuration['experiment_id']}_evaluation",
        "description": f"{configuration['description']} (Evaluating Fold Predictions)",
        "predictions_file": f"{predictions_file_prefix}.{given_file_format}",
        "system_score_column": "scale" if configuration["use_scaled_predictions"] else "raw",
        "scale_with": "asis" if configuration["use_scaled_predictions"] else "raw",
        "trim_min": int(f"{configuration['trim_min']}"),
        "trim_max": int(f"{configuration['trim_max']}"),
        "file_format": given_file_format
    }

    # copy over the relevant configuration fields from the main configuration
    for field_name in ["second_human_score_column",
                       "candidate_column",
                       "flag_column",
                       "exclude_zero_scores",
                       "min_items_per_candidate",
                       "rater_error_variance",
                       "subgroups",
                       "trim_tolerance",
                       "use_thumbnails"]:
        evaluation_configdict[field_name] = configuration[field_name]

    # create an rsmeval configuration object with this dictionary
    evaluation_configuration = Configuration(evaluation_configdict,
                                             configdir=evaldir,
                                             context="rsmeval")

    # run rsmeval on the combined predictions file
    logger.info("Evaluating combined fold predictions")
    eval_logger = get_file_logger("evaluation", Path(evaldir) / "rsmeval.log")
    run_evaluation(evaluation_configuration, evaldir, False, logger=eval_logger)

    # run rsmtool on the full dataset and generate only model/feature
    # descriptives report; we will use the dummy test set that we
    # created by calling `create_xval_files()` for this purpose
    logger.info("Training model on full data")
    model_logger = get_file_logger("final_model", Path(modeldir) / "rsmtool.log")
    final_rsmtool_configdict = configuration.to_dict().copy()
    final_rsmtool_configdict["experiment_id"] = f"{configuration['experiment_id']}_model"
    final_rsmtool_configdict["test_file"] = str(Path(modeldir) / f"dummy_test.{given_file_format}")
    final_rsmtool_configdict["test_label_column"] = final_rsmtool_configdict["train_label_column"]

    # remove by-group sections if we don't have the info
    sections_to_use = []
    for section in FINAL_MODEL_SECTION_LIST:
        if section.endswith("_by_group") and not subgroups:
            continue
        sections_to_use.append(section)
    final_rsmtool_configdict["general_sections"] = sections_to_use
    final_rsmtool_configuration = Configuration(final_rsmtool_configdict,
                                                configdir=configuration.configdir,
                                                context="rsmtool")
    run_experiment(final_rsmtool_configuration, modeldir, False, logger=model_logger)

def main():  # noqa: D103

    # set up the basic logging configuration
    formatter = LogFormatter()

    # we need two handlers, one that prints to stdout
    # for the "run" command and one that prints to stderr
    # from the "generate" command; the latter is important
    # because do not want the warning to show up in the
    # generated configuration file
    stdout_handler = logging.StreamHandler(sys.stdout)
    stdout_handler.setFormatter(formatter)

    stderr_handler = logging.StreamHandler(sys.stderr)
    stderr_handler.setFormatter(formatter)

    logging.root.setLevel(logging.INFO)
    logger = logging.getLogger(__name__)

    # set up an argument parser via our helper function
    parser = setup_rsmcmd_parser('rsmxval',
                                 uses_output_directory=True,
                                 allows_overwriting=False,
                                 uses_subgroups=True)

    # if we have no arguments at all then just show the help message
    if len(sys.argv) < 2:
        sys.argv.append("-h")

    # if the first argument is not one of the valid sub-commands
    # or one of the valid optional arguments, then assume that they
    # are arguments for the "run" sub-command. This allows the
    # old style command-line invocations to work without modification.
    if sys.argv[1] not in VALID_PARSER_SUBCOMMANDS + ['-h', '--help',
                                                      '-V', '--version']:
        args_to_pass = ['run'] + sys.argv[1:]
    else:
        args_to_pass = sys.argv[1:]
    args = parser.parse_args(args=args_to_pass)

    # call the appropriate function based on which sub-command was run
    if args.subcommand == 'run':

        # when running, log to stdout
        logging.root.addHandler(stdout_handler)

        # run the experiment
        logger.info('Output directory: {}'.format(args.output_dir))
        run_cross_validation(abspath(args.config_file),
                             abspath(args.output_dir))

    else:

        # when generating, log to stderr
        logging.root.addHandler(stderr_handler)

        # auto-generate an example configuration and print it to STDOUT
        generator = ConfigurationGenerator('rsmtool',
                                           as_string=True,
                                           suppress_warnings=args.quiet,
                                           use_subgroups=args.subgroups)
        configuration = generator.interact() if args.interactive else generator.generate()
        print(configuration)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import logging
import os
from pathlib import Path
from time import sleep
from typing import Optional, Union

from tensorflow.keras.callbacks import Callback

from huggingface_hub import Repository

from . import IntervalStrategy, PreTrainedTokenizerBase
from .file_utils import get_full_repo_name
from .modelcard import TrainingSummary

logger = logging.getLogger(__name__)

class PushToHubCallback(Callback):
    def __init__(
        self,
        output_dir: Union[str, Path],
        save_strategy: Union[str, IntervalStrategy] = "epoch",
        save_steps: Optional[int] = None,
        tokenizer: Optional[PreTrainedTokenizerBase] = None,
        hub_model_id: Optional[str] = None,
        hub_token: Optional[str] = None,
        checkpoint: bool = False,
        **model_card_args
    ):
        """
        output_dir (`str`):
            The output directory where the model predictions and checkpoints will be written and synced with the
            repository on the Hub.
        save_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"epoch"`):
            The checkpoint save strategy to adopt during training. Possible values are:

                - `"no"`: No save is done during training.
                - `"epoch"`: Save is done at the end of each epoch.
                - `"steps"`: Save is done every `save_steps`
        save_steps (`int`, *optional*):
            The number of steps between saves when using the "steps" save_strategy.
        tokenizer (`PreTrainedTokenizerBase`, *optional*):
            The tokenizer used by the model. If supplied, will be uploaded to the repo alongside the weights.
        hub_model_id (`str`, *optional*):
            The name of the repository to keep in sync with the local *output_dir*. It can be a simple model ID in
            which case the model will be pushed in your namespace. Otherwise it should be the whole repository name,
            for instance `"user_name/model"`, which allows you to push to an organization you are a member of with
            `"organization_name/model"`.

            Will default to to the name of `output_dir`.
        hub_token (`str`, *optional*):
            The token to use to push the model to the Hub. Will default to the token in the cache folder obtained with
            `huggingface-cli login`.
        checkpoint (`bool`, *optional*, defaults to `False`):
            Whether to save full training checkpoints (including epoch and optimizer state) to allow training to be
            resumed. Only usable when *save_strategy* is *epoch*.
        """
        super().__init__()
        if checkpoint and save_strategy != "epoch":
            raise ValueError("Cannot save checkpoints when save_strategy is not 'epoch'!")
        if isinstance(save_strategy, str):
            save_strategy = IntervalStrategy(save_strategy.lower())
        self.save_strategy = save_strategy
        if self.save_strategy == IntervalStrategy.STEPS and (not isinstance(save_steps, int) or save_steps <= 0):
            raise ValueError("Please supply a positive integer argument for save_steps when save_strategy == 'steps'!")
        self.save_steps = save_steps
        output_dir = Path(output_dir)
        if hub_model_id is None:
            hub_model_id = output_dir.absolute().name
        if "/" not in hub_model_id:
            hub_model_id = get_full_repo_name(hub_model_id, token=hub_token)

        self.output_dir = output_dir
        self.hub_model_id = hub_model_id
        self.repo = Repository(
            str(self.output_dir),
            clone_from=self.hub_model_id,
            use_auth_token=hub_token if hub_token else True,
        )
        self.tokenizer = tokenizer
        self.last_job = None
        self.checkpoint = checkpoint
        self.training_history = None
        self.model_card_args = model_card_args

    def on_train_begin(self, logs=None):
        # Although we can access model.history, we have no guarantees that the History callback will fire before this
        # one, so we keep track of it here too
        self.training_history = []

    def on_train_batch_end(self, batch, logs=None):
        if self.save_strategy == IntervalStrategy.STEPS and batch + 1 % self.save_steps == 0:
            if self.last_job is not None and not self.last_job.is_done:
                return  # The last upload is still running, don't start another
            self.model.save_pretrained(self.output_dir)
            if self.tokenizer is not None:
                self.tokenizer.save_pretrained(self.output_dir)
            _, self.last_job = self.repo.push_to_hub(
                commit_message=f"Training in progress steps {batch}", blocking=False
            )

    def on_epoch_end(self, epoch, logs=None):
        if "epoch" not in logs:
            logs["epoch"] = epoch
        self.training_history.append(logs)
        if self.save_strategy == IntervalStrategy.EPOCH:
            if self.last_job is not None and not self.last_job.is_done:
                return  # The last upload is still running, don't start another
            self.model.save_pretrained(self.output_dir)
            if self.tokenizer is not None:
                self.tokenizer.save_pretrained(self.output_dir)
            if self.checkpoint:
                checkpoint_dir = os.path.join(self.output_dir, "checkpoint")
                self.model._save_checkpoint(checkpoint_dir, epoch)
            train_summary = TrainingSummary.from_keras(
                model=self.model,
                model_name=self.hub_model_id,
                keras_history=self.training_history,
                **self.model_card_args,
            )
            model_card = train_summary.to_model_card()
            with (self.output_dir / "README.md").open("w") as f:
                f.write(model_card)
            _, self.last_job = self.repo.push_to_hub(
                commit_message=f"Training in progress epoch {epoch}", blocking=False
            )

    def on_train_end(self, logs=None):
        if self.last_job is not None and not self.last_job.is_done:
            logger.info("Waiting for existing upload to finish...")
            while not self.last_job.is_done:
                sleep(1)
        self.model.save_pretrained(self.output_dir)
        if self.tokenizer is not None:
            self.tokenizer.save_pretrained(self.output_dir)
        train_summary = TrainingSummary.from_keras(
            model=self.model, model_name=self.hub_model_id, keras_history=self.training_history, **self.model_card_args
        )
        model_card = train_summary.to_model_card()
        with (self.output_dir / "README.md").open("w") as f:
            f.write(model_card)
        self.repo.push_to_hub(commit_message="End of training", blocking=True)
<commit_msg>Keras metric callback (#14867)<commit_after>import logging
import os
from pathlib import Path
from time import sleep
from typing import Callable, List, Optional, Union

import numpy as np
import tensorflow as tf
from packaging.version import parse
from tensorflow.keras.callbacks import Callback

from huggingface_hub import Repository

from . import IntervalStrategy, PreTrainedTokenizerBase
from .file_utils import get_full_repo_name
from .modelcard import TrainingSummary

logger = logging.getLogger(__name__)

class KerasMetricCallback(Callback):
    """
    Callback to compute metrics at the end of every epoch. Unlike normal Keras metrics, these do not need to be
    compilable by TF. It is particularly useful for common NLP metrics like BLEU and ROUGE that require string
    operations or generation loops that cannot be compiled. Predictions (or generations) will be computed on the
    `eval_dataset` before being passed to the `metric_fn` in `np.ndarray` format. The `metric_fn` should compute
    metrics and return a dict mapping metric names to metric values.

    We provide an example of a suitable metric_fn that computes ROUGE scores for a summarization model below.
    Note that this example skips some post-processing for readability and simplicity, and should probably
    not be used as-is!

    ```py
    from datasets import load_metric
    rouge_metric = load_metric("rouge")

    def rouge_fn(predictions, labels):
        decoded_predictions = tokenizer.batch_decode(predictions, skip_special_tokens=True))
        decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
        result = rouge_metric.compute(predictions=decoded_predictions, references=decoded_labels)
        return {key: value.mid.fmeasure * 100 for key, value in result.items()}
    ```

    The above function will return a dict containing values which will be logged like any other Keras metric:

    ```
    {'rouge1': 37.4199, 'rouge2': 13.9768, 'rougeL': 34.361, 'rougeLsum': 35.0781
    ```

    Args:
        metric_fn (`Callable`):
            Metric function provided by the user. It will be called with two arguments - `predictions` and `labels`.
            These contain the model's outputs and matching labels from the dataset. It should return a dict mapping
            metric names to numerical values.
        eval_dataset (`tf.data.Dataset` or `dict` or `tuple` or `np.ndarray` or `tf.Tensor`):
            Validation data to be used to generate predictions for the `metric_fn`.
        metric_fn_kwargs (`dict`, *optional*):
            Additional keyword arguments to be passed to the metric_fn.
        tokenizer ([`PretrainedTokenizerBase`], *optional*):
            Tokenizer used to validate column names to be passed to the generate() function.
        output_cols (`List[str], *optional*):
            A list of columns to be retained from the model output as the predictions. Defaults to all.
        label_cols ('`List[str]`, *optional*'):
            A list of columns to be retained from the input dataset as the labels. Will be autodetected if this is not
            supplied.
        batch_size (`int`, *optional*):
            Batch size. Only used when the data is not a pre-batched `tf.data.Dataset`.
        predict_with_generate: (`bool`, *optional*, defaults to `False`):
            Whether we should use `model.generate()` to get outputs for the model.

    """

    def __init__(
        self,
        metric_fn: Callable,
        eval_dataset: Union[tf.data.Dataset, np.ndarray, tf.Tensor, tuple, dict],
        tokenizer: Optional[PreTrainedTokenizerBase] = None,
        metric_fn_kwargs: Optional[dict] = None,
        output_cols: Optional[List[str]] = None,
        label_cols: Optional[List[str]] = None,
        batch_size: Optional[int] = None,
        predict_with_generate: Optional[bool] = False,
    ):
        super().__init__()
        self.metric_fn = metric_fn
        self.batch_size = batch_size
        if not isinstance(eval_dataset, tf.data.Dataset):
            if batch_size is None:
                raise ValueError(
                    "When passing data to KerasMetricCallback that is not a pre-batched tf.data.Dataset "
                    "the batch_size argument must be set."
                )
            # Wrap a tf.data.Dataset around it
            eval_dataset = tf.data.Dataset.from_tensor_slices(eval_dataset).batch(batch_size, drop_remainder=False)
        self.eval_dataset = eval_dataset
        self.predict_with_generate = predict_with_generate
        self.output_cols = output_cols
        self.metric_fn_kwargs = metric_fn_kwargs or dict()
        if tokenizer is not None:
            self.model_input_names = tokenizer.model_input_names
        else:
            self.model_input_names = ["input_ids"]

        # This next block attempts to parse out which elements of the dataset should be appended to the labels list
        # that is passed to the metric_fn
        if isinstance(eval_dataset.element_spec, tuple) and len(eval_dataset.element_spec) == 2:
            input_spec, label_spec = eval_dataset.element_spec
        else:
            input_spec = eval_dataset.element_spec
            label_spec = None
        if label_cols is not None:
            for label in label_cols:
                if label not in input_spec:
                    raise ValueError(f"Label {label} is in label_cols but could not be found in the dataset inputs!")
            self.label_cols = label_cols
            self.use_keras_label = False
        elif label_spec is not None:
            # If the dataset inputs are split into a 2-tuple of inputs and labels,
            # assume the second element is the labels
            self.label_cols = None
            self.use_keras_label = True
        elif "labels" in input_spec:
            self.label_cols = ["labels"]
            self.use_keras_label = False
            logging.warning("No label_cols specified for KerasMetricCallback, assuming you want the 'labels' key.")
        else:
            raise ValueError("Could not autodetect label_cols for KerasMetricCallback, please specify them!")
        if parse(tf.__version__).minor < parse("2.7"):
            logging.warning("TF versions less than 2.7 may encounter issues with KerasMetricCallback!")

    @staticmethod
    def _concatenate_batches(batches):
        # Flattens Numpy array batches into a list of single samples, where each sample is still np.ndarray
        return [sample for batch in batches for sample in batch]

    def _postprocess_predictions_or_labels(self, inputs):
        if isinstance(inputs[0], dict):
            outputs = dict()
            for key in inputs[0].keys():
                outputs[key] = self._concatenate_batches(batch[key] for batch in inputs)
        elif isinstance(inputs[0], list) or isinstance(inputs[0], tuple):
            outputs = []
            for input_list in zip(*inputs):
                outputs.append(self._concatenate_batches(input_list))
        elif isinstance(inputs[0], np.ndarray):
            outputs = self._concatenate_batches(inputs)
        else:
            raise TypeError(f"Couldn't handle batch of type {type(inputs[0])}!")
        return outputs

    def on_epoch_end(self, epoch, logs=None):
        prediction_list = []
        label_list = []

        # The whole predict/generate loop is handled inside this method
        for batch in self.eval_dataset:
            if isinstance(batch, tuple):
                batch, labels = batch
            else:
                labels = None
            if self.predict_with_generate:
                if isinstance(batch, dict):
                    # generate() gets stressed out by any unexpected keys
                    batch = {key: array for key, array in batch.items() if key in self.model_input_names}
                predictions = self.model.generate(batch)
            else:
                predictions = self.model.predict(batch)
            predictions = dict(predictions)
            if self.output_cols is not None:
                predictions = {key: predictions[key] for key in self.output_cols}
            prediction_list.append(predictions)
            if not self.use_keras_label:
                labels = {key: batch[key].numpy() for key in self.label_cols}
            elif isinstance(labels, dict):
                labels = {key: array.numpy() for key, array in labels.items()}
            elif isinstance(labels, list) or isinstance(labels, tuple):
                labels = [array.numpy() for array in labels]
            elif isinstance(labels, tf.Tensor):
                labels = labels.numpy()
            else:
                raise TypeError(f"Confused by labels of type {type(labels)}")
            label_list.append(labels)

        prediction_list = self._postprocess_predictions_or_labels(prediction_list)
        label_list = self._postprocess_predictions_or_labels(label_list)

        metric_output = self.metric_fn(prediction_list, label_list, **self.metric_fn_kwargs)
        if not isinstance(metric_output, dict):
            raise TypeError(
                f"metric_fn should return a dict mapping metric names to values but instead returned {metric_output}"
            )
        # This is the critical bit - Keras passes a dict containing the loss and standard metric values for this epoch
        # in the logs argument. Ordinarily, this is so the callback can read them, but in this case we write a bunch of
        # new keys in there, which will then get read by the History callback and treated like any other metric value.
        # I promise that I have it in writing from Chollet that this is okay.
        logs.update(metric_output)

class PushToHubCallback(Callback):
    def __init__(
        self,
        output_dir: Union[str, Path],
        save_strategy: Union[str, IntervalStrategy] = "epoch",
        save_steps: Optional[int] = None,
        tokenizer: Optional[PreTrainedTokenizerBase] = None,
        hub_model_id: Optional[str] = None,
        hub_token: Optional[str] = None,
        checkpoint: bool = False,
        **model_card_args
    ):
        """
        output_dir (`str`):
            The output directory where the model predictions and checkpoints will be written and synced with the
            repository on the Hub.
        save_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"epoch"`):
            The checkpoint save strategy to adopt during training. Possible values are:

                - `"no"`: No save is done during training.
                - `"epoch"`: Save is done at the end of each epoch.
                - `"steps"`: Save is done every `save_steps`
        save_steps (`int`, *optional*):
            The number of steps between saves when using the "steps" save_strategy.
        tokenizer (`PreTrainedTokenizerBase`, *optional*):
            The tokenizer used by the model. If supplied, will be uploaded to the repo alongside the weights.
        hub_model_id (`str`, *optional*):
            The name of the repository to keep in sync with the local *output_dir*. It can be a simple model ID in
            which case the model will be pushed in your namespace. Otherwise it should be the whole repository name,
            for instance `"user_name/model"`, which allows you to push to an organization you are a member of with
            `"organization_name/model"`.

            Will default to to the name of `output_dir`.
        hub_token (`str`, *optional*):
            The token to use to push the model to the Hub. Will default to the token in the cache folder obtained with
            `huggingface-cli login`.
        checkpoint (`bool`, *optional*, defaults to `False`):
            Whether to save full training checkpoints (including epoch and optimizer state) to allow training to be
            resumed. Only usable when *save_strategy* is *epoch*.
        """
        super().__init__()
        if checkpoint and save_strategy != "epoch":
            raise ValueError("Cannot save checkpoints when save_strategy is not 'epoch'!")
        if isinstance(save_strategy, str):
            save_strategy = IntervalStrategy(save_strategy.lower())
        self.save_strategy = save_strategy
        if self.save_strategy == IntervalStrategy.STEPS and (not isinstance(save_steps, int) or save_steps <= 0):
            raise ValueError("Please supply a positive integer argument for save_steps when save_strategy == 'steps'!")
        self.save_steps = save_steps
        output_dir = Path(output_dir)
        if hub_model_id is None:
            hub_model_id = output_dir.absolute().name
        if "/" not in hub_model_id:
            hub_model_id = get_full_repo_name(hub_model_id, token=hub_token)

        self.output_dir = output_dir
        self.hub_model_id = hub_model_id
        self.repo = Repository(
            str(self.output_dir),
            clone_from=self.hub_model_id,
            use_auth_token=hub_token if hub_token else True,
        )
        self.tokenizer = tokenizer
        self.last_job = None
        self.checkpoint = checkpoint
        self.training_history = None
        self.model_card_args = model_card_args

    def on_train_begin(self, logs=None):
        # Although we can access model.history, we have no guarantees that the History callback will fire before this
        # one, so we keep track of it here too
        self.training_history = []

    def on_train_batch_end(self, batch, logs=None):
        if self.save_strategy == IntervalStrategy.STEPS and batch + 1 % self.save_steps == 0:
            if self.last_job is not None and not self.last_job.is_done:
                return  # The last upload is still running, don't start another
            self.model.save_pretrained(self.output_dir)
            if self.tokenizer is not None:
                self.tokenizer.save_pretrained(self.output_dir)
            _, self.last_job = self.repo.push_to_hub(
                commit_message=f"Training in progress steps {batch}", blocking=False
            )

    def on_epoch_end(self, epoch, logs=None):
        if "epoch" not in logs:
            logs["epoch"] = epoch
        self.training_history.append(logs)
        if self.save_strategy == IntervalStrategy.EPOCH:
            if self.last_job is not None and not self.last_job.is_done:
                return  # The last upload is still running, don't start another
            self.model.save_pretrained(self.output_dir)
            if self.tokenizer is not None:
                self.tokenizer.save_pretrained(self.output_dir)
            if self.checkpoint:
                checkpoint_dir = os.path.join(self.output_dir, "checkpoint")
                self.model._save_checkpoint(checkpoint_dir, epoch)
            train_summary = TrainingSummary.from_keras(
                model=self.model,
                model_name=self.hub_model_id,
                keras_history=self.training_history,
                **self.model_card_args,
            )
            model_card = train_summary.to_model_card()
            with (self.output_dir / "README.md").open("w") as f:
                f.write(model_card)
            _, self.last_job = self.repo.push_to_hub(
                commit_message=f"Training in progress epoch {epoch}", blocking=False
            )

    def on_train_end(self, logs=None):
        if self.last_job is not None and not self.last_job.is_done:
            logger.info("Waiting for existing upload to finish...")
            while not self.last_job.is_done:
                sleep(1)
        self.model.save_pretrained(self.output_dir)
        if self.tokenizer is not None:
            self.tokenizer.save_pretrained(self.output_dir)
        train_summary = TrainingSummary.from_keras(
            model=self.model, model_name=self.hub_model_id, keras_history=self.training_history, **self.model_card_args
        )
        model_card = train_summary.to_model_card()
        with (self.output_dir / "README.md").open("w") as f:
            f.write(model_card)
        self.repo.push_to_hub(commit_message="End of training", blocking=True)
<|endoftext|>


<commit_before>from __future__ import print_function
import abc
import collections
import enum

class UGen(object):
    r'''A UGen.
    '''

    ### CLASS VARIABLES ###

    class Rate(enum.IntEnum):
        AUDIO_RATE = 2
        CONTROL_RATE = 1
        SCALAR_RATE = 0
        TRIGGER_RATE = -1

    class SignalRange(enum.IntEnum):
        UNIPOLAR = 0
        BIPOLAR = 1

    __metaclass__ = abc.ABCMeta

    __slots__ = (
        '_antecedents',
        '_calculation_rate',
        '_descendants',
        '_inputs',
        '_output_proxies',
        '_special_index',
        '_synthdef',
        '_width_first_antecedents',
        )

    _argument_specifications = ()

    _unexpanded_argument_names = None

    ### INITIALIZER ###

    @abc.abstractmethod
    def __init__(
        self,
        calculation_rate=None,
        special_index=0,
        **kwargs
        ):
        from supriya import audiolib
        assert isinstance(calculation_rate, audiolib.UGen.Rate), calculation_rate
        self._calculation_rate = calculation_rate
        self._inputs = []
        self._special_index = special_index
        for i in range(len(self._argument_specifications)):
            argument_specification = self._argument_specifications[i]
            argument_name = argument_specification.name
            argument_value = kwargs.get(argument_name, None)
            argument_value = None
            if argument_name in kwargs:
                argument_value = kwargs[argument_name]
                del(kwargs[argument_name])
            prototype = (
                type(None),
                float,
                int,
                UGen,
                audiolib.OutputProxy,
                )
            assert isinstance(argument_value, prototype), argument_value
            argument_specification.configure(self, argument_value)
        if kwargs:
            raise ValueError(kwargs)
        self._antecedents = []
        self._descendants = []
        self._output_proxies = tuple(
            audiolib.OutputProxy(self, i)
            for i in range(len(self))
            )
        self._synthdef = None
        self._width_first_antecedents = []

    ### SPECIAL METHODS ###

    def __add__(self, expr):
        from supriya import audiolib
        calculation_rate = self._compute_binary_rate(self, expr)
        special_index = audiolib.BinaryOpUGen.BinaryOperator.PLUS.value
        return audiolib.BinaryOpUGen._new(
            calculation_rate,
            special_index,
            left=self,
            right=expr,
            )

    def __div__(self, expr):
        from supriya import audiolib
        calculation_rate = self._compute_binary_rate(self, expr)
        special_index = audiolib.BinaryOpUGen.BinaryOperator.DIVIDE.value
        return audiolib.BinaryOpUGen._new(
            calculation_rate,
            special_index,
            left=self,
            right=expr,
            )

    def __getattr__(self, attr):
        try:
            object.__getattr__(self, attr)
        except AttributeError:
            for i, argument_specification in enumerate(
                self._argument_specifications):
                if argument_specification.name == attr:
                    return self.inputs[i]
        raise AttributeError

    def __getitem__(self, i):
        return self._output_proxies[i]

    def __len__(self):
        return 1

    def __mod__(self, expr):
        from supriya import audiolib
        calculation_rate = self._compute_binary_rate(self, expr)
        special_index = audiolib.BinaryOpUGen.BinaryOperator.MOD.value
        return audiolib.BinaryOpUGen._new(
            calculation_rate,
            special_index,
            left=self,
            right=expr,
            )

    def __mul__(self, expr):
        from supriya import audiolib
        calculation_rate = self._compute_binary_rate(self, expr)
        special_index = audiolib.BinaryOpUGen.BinaryOperator.TIMES.value
        return audiolib.BinaryOpUGen._new(
            calculation_rate,
            special_index,
            left=self,
            right=expr,
            )

    def __neg__(self):
        from supriya import audiolib
        calculation_rate = self.calculation_rate
        special_index = audiolib.UnaryOpUGen.UnaryOperator.NEG.value
        return audiolib.UnaryOpUGen._new(
            calculation_rate,
            special_index,
            source=self,
            )

    def __radd__(self, expr):
        return self.__add__(expr)

    def __rdiv__(self, expr):
        return self.__div__(expr)

    def __repr__(self):
        calculation_abbreviations = {
            self.Rate.AUDIO_RATE: 'ar',
            self.Rate.CONTROL_RATE: 'kr',
            self.Rate.SCALAR_RATE: 'ir',
            self.Rate.TRIGGER_RATE: 'tr',
            }
        string = '{}.{}()'.format(
            type(self).__name__,
            calculation_abbreviations[self.calculation_rate]
            )
        return string

    def __rmul__(self, expr):
        return self.__mul__(expr)

    def __rsub__(self, expr):
        return self.__sub__(expr)

    def __sub__(self, expr):
        from supriya import audiolib
        calculation_rate = self._compute_binary_rate(self, expr)
        special_index = audiolib.BinaryOpUGen.BinaryOperator.MINUS.value
        return audiolib.BinaryOpUGen._new(
            calculation_rate,
            special_index,
            left=self,
            right=expr,
            )

    ### PRIVATE METHODS ###

    def _add_constant_input(self, value):
        self._inputs.append(float(value))

    def _add_ugen_input(self, ugen, output_index=None):
        from supriya import audiolib
        if isinstance(ugen, audiolib.OutputProxy):
            output_proxy = ugen
        else:
            output_proxy = audiolib.OutputProxy(
                output_index=output_index,
                source=ugen,
                )
        self._inputs.append(output_proxy)

    def _collect_constants(self):
        from supriya import audiolib
        for input_ in self._inputs:
            if not isinstance(input_, audiolib.OutputProxy):
                self.synthdef._add_constant(float(input_))

    @staticmethod
    def _compute_binary_rate(ugen_a, ugen_b):
        if ugen_a.calculation_rate == UGen.Rate.AUDIO_RATE:
            return UGen.Rate.AUDIO_RATE
        if hasattr(ugen_b, 'calculation_rate') and \
            ugen_b.calculation_rate == UGen.RAte.AUDIO_RATE:
            return UGen.Rate.AUDIO_RATE
        if ugen_a.calculation_rate == UGen.Rate.CONTROL_RATE:
            return UGen.Rate.CONTROL_RATE
        if hasattr(ugen_b, 'calculation_rate') and \
            ugen_b.calculation_rate == UGen.Rate.CONTROL_RATE:
            return UGen.Rate.CONTROL_RATE
        return UGen.Rate.SCALAR_RATE

    def _get_output_number(self):
        return 0

    def _get_outputs(self):
        return [self.calculation_rate]

    def _get_source(self):
        return self

    def _initialize_topological_sort(self):
        from supriya import audiolib
        for input_ in self.inputs:
            if isinstance(input_, audiolib.OutputProxy):
                ugen = input_.source
                if ugen not in self.antecedents:
                    self.antecedents.append(ugen)
                if self not in ugen.descendants:
                    ugen.descendants.append(self)
        for ugen in self.width_first_antecedents:
            if ugen not in self.antecedents:
                self.antecedents.append(ugen)
            if self not in ugen.descendants:
                ugen.descendants.append(self)

    def _make_available(self):
        if not self.antecedents:
            if self not in self.synthdef._available_ugens:
                self.synthdef._available_ugens.append(self)

    @classmethod
    def _new(cls, calculation_rate, special_index, **kwargs):
        import sys
        from supriya import audiolib
        if sys.version_info[0] == 2:
            import funcsigs
            get_signature = funcsigs.signature
        else:
            import inspect
            get_signature = inspect.signature
        assert isinstance(calculation_rate, UGen.Rate)
        argument_dicts = UGen.expand_arguments(
            kwargs, unexpanded_argument_names=cls._unexpanded_argument_names)
        ugens = []
        signature = get_signature(cls.__init__)
        has_custom_special_index = 'special_index' in signature.parameters
        for argument_dict in argument_dicts:
            if has_custom_special_index:
                ugen = cls(
                    calculation_rate=calculation_rate,
                    special_index=special_index,
                    **argument_dict
                    )
            else:
                ugen = cls(
                    calculation_rate=calculation_rate,
                    **argument_dict
                    )
            ugens.append(ugen)
        if len(ugens) == 1:
            return ugens[0]
        return audiolib.UGenArray(ugens)

    def _optimize_graph(self):
        pass

    def _remove_antecedent(self, ugen):
        self.antecedents.remove(ugen)
        self._make_available()

    def _schedule(self, out_stack):
        for ugen in reversed(self.descendants):
            ugen._remove_antecedent(self)
        out_stack.append(self)

    ### PUBLIC METHODS ###

    @classmethod
    def ar(cls, **kwargs):
        ugen = cls._new(
            calculation_rate=UGen.Rate.AUDIO_RATE,
            special_index=0,
            **kwargs
            )
        return ugen

    def compile(self, synthdef):
        def compile_input_spec(i, synthdef):
            from supriya import audiolib
            result = []
            if isinstance(i, float):
                result.append(SynthDef._encode_unsigned_int_32bit(0xffffffff))
                constant_index = synthdef._get_constant_index(i)
                result.append(SynthDef._encode_unsigned_int_32bit(
                    constant_index))
            elif isinstance(i, audiolib.OutputProxy):
                ugen = i.source
                output_index = i.output_index
                ugen_index = synthdef._get_ugen_index(ugen)
                result.append(SynthDef._encode_unsigned_int_32bit(ugen_index))
                result.append(SynthDef._encode_unsigned_int_32bit(output_index))
            else:
                raise Exception('Unhandled input spec: {}'.format(i))
            return bytearray().join(result)
        from supriya.library.audiolib import SynthDef
        outputs = self._get_outputs()
        result = []
        result.append(SynthDef._encode_string(type(self).__name__))
        result.append(SynthDef._encode_unsigned_int_8bit(self.calculation_rate))
        result.append(SynthDef._encode_unsigned_int_32bit(len(self.inputs)))
        result.append(SynthDef._encode_unsigned_int_32bit(len(outputs)))
        result.append(SynthDef._encode_unsigned_int_16bit(int(self.special_index)))
        for i in self.inputs:
            result.append(compile_input_spec(i, synthdef))
        for o in outputs:
            result.append(SynthDef._encode_unsigned_int_8bit(o))
        result = bytearray().join(result)
        return result

    @staticmethod
    def expand_arguments(arguments, unexpanded_argument_names=None):
        r'''Expands ugens into multichannel arrays.

        ::

            >>> import supriya
            >>> arguments = {'foo': 0, 'bar': (1, 2), 'baz': (3, 4, 5)}
            >>> result = supriya.audiolib.UGen.expand_arguments(arguments)
            >>> for x in result:
            ...     x
            ...
            {'bar': 1, 'foo': 0, 'baz': 3}
            {'bar': 2, 'foo': 0, 'baz': 4}
            {'bar': 1, 'foo': 0, 'baz': 5}

        ::

            >>> arguments = {'bus': (8, 9), 'source': (1, 2, 3)}
            >>> result = supriya.audiolib.UGen.expand_arguments(
            ...     arguments,
            ...     unexpanded_argument_names=('source',),
            ...     )
            >>> for x in result:
            ...     x
            ...
            {'bus': 8, 'source': (1, 2, 3)}
            {'bus': 9, 'source': (1, 2, 3)}

        '''
        cached_unexpanded_arguments = {}
        if unexpanded_argument_names is not None:
            for argument_name in unexpanded_argument_names:
                if argument_name not in arguments:
                    continue
                cached_unexpanded_arguments[argument_name] = \
                    arguments[argument_name]
                del(arguments[argument_name])
        maximum_length = 1
        result = []
        for name, value in arguments.items():
            if isinstance(value, collections.Sequence):
                maximum_length = max(maximum_length, len(value))
        for i in range(maximum_length):
            result.append({})
            for name, value in arguments.items():
                if isinstance(value, collections.Sequence):
                    value = value[i % len(value)]
                    result[i][name] = value
                else:
                    result[i][name] = value
        for expanded_arguments in result:
            expanded_arguments.update(cached_unexpanded_arguments)
        return result

    @classmethod
    def kr(cls, **kwargs):
        ugen = cls._new(
            calculation_rate=UGen.Rate.CONTROL_RATE,
            special_index=0,
            **kwargs
            )
        return ugen

    ### PUBLIC PROPERTIES ###

    @property
    def antecedents(self):
        return self._antecedents

    @property
    def calculation_rate(self):
        return self._calculation_rate

    @property
    def descendants(self):
        return self._descendants

    @property
    def inputs(self):
        return tuple(self._inputs)

    @property
    def signal_range(self):
        return self.SignalRange.BIPOLAR

    @property
    def special_index(self):
        return self._special_index

    @property
    def synthdef(self):
        return self._synthdef

    @synthdef.setter
    def synthdef(self, synthdef):
        from supriya import audiolib
        assert isinstance(synthdef, audiolib.SynthDef)
        self._synthdef = synthdef

    @property
    def width_first_antecedents(self):
        return self._width_first_antecedents
<commit_msg>Streamlined UGen.<commit_after>from __future__ import print_function
import abc
import collections
import enum

class UGen(object):
    r'''A UGen.
    '''

    ### CLASS VARIABLES ###

    class Rate(enum.IntEnum):
        AUDIO_RATE = 2
        CONTROL_RATE = 1
        SCALAR_RATE = 0
        TRIGGER_RATE = -1

    class SignalRange(enum.IntEnum):
        UNIPOLAR = 0
        BIPOLAR = 1

    __metaclass__ = abc.ABCMeta

    __slots__ = (
        '_antecedents',
        '_calculation_rate',
        '_descendants',
        '_inputs',
        '_output_proxies',
        '_special_index',
        '_synthdef',
        '_width_first_antecedents',
        )

    _argument_specifications = ()

    _unexpanded_argument_names = None

    ### INITIALIZER ###

    @abc.abstractmethod
    def __init__(
        self,
        calculation_rate=None,
        special_index=0,
        **kwargs
        ):
        from supriya import audiolib
        assert isinstance(calculation_rate, audiolib.UGen.Rate), calculation_rate
        self._calculation_rate = calculation_rate
        self._inputs = []
        self._special_index = special_index
        for i in range(len(self._argument_specifications)):
            argument_specification = self._argument_specifications[i]
            argument_name = argument_specification.name
            argument_value = kwargs.get(argument_name, None)
            argument_value = None
            if argument_name in kwargs:
                argument_value = kwargs[argument_name]
                del(kwargs[argument_name])
            prototype = (
                type(None),
                float,
                int,
                UGen,
                audiolib.OutputProxy,
                )
            assert isinstance(argument_value, prototype), argument_value
            argument_specification.configure(self, argument_value)
        if kwargs:
            raise ValueError(kwargs)
        self._antecedents = []
        self._descendants = []
        self._output_proxies = tuple(
            audiolib.OutputProxy(self, i)
            for i in range(len(self))
            )
        self._synthdef = None
        self._width_first_antecedents = []

    ### SPECIAL METHODS ###

    def __add__(self, expr):
        return self._compute_binary_op(expr, 'ADD')

    def __div__(self, expr):
        return self._compute_binary_op(expr, 'FDIV')

    def __getattr__(self, attr):
        try:
            object.__getattr__(self, attr)
        except AttributeError:
            for i, argument_specification in enumerate(
                self._argument_specifications):
                if argument_specification.name == attr:
                    return self.inputs[i]
        raise AttributeError

    def __getitem__(self, i):
        return self._output_proxies[i]

    def __len__(self):
        return 1

    def __mod__(self, expr):
        return self._compute_binary_op(expr, 'MOD')

    def __mul__(self, expr):
        return self._compute_binary_op(expr, 'MUL')

    def __neg__(self):
        return self._compute_unary_op('NEG')

    def __radd__(self, expr):
        return self.__add__(expr)

    def __rdiv__(self, expr):
        return self.__div__(expr)

    def __repr__(self):
        calculation_abbreviations = {
            self.Rate.AUDIO_RATE: 'ar',
            self.Rate.CONTROL_RATE: 'kr',
            self.Rate.SCALAR_RATE: 'ir',
            self.Rate.TRIGGER_RATE: 'tr',
            }
        string = '{}.{}()'.format(
            type(self).__name__,
            calculation_abbreviations[self.calculation_rate]
            )
        return string

    def __rmul__(self, expr):
        return self.__mul__(expr)

    def __rsub__(self, expr):
        return self.__sub__(expr)

    def __sub__(self, expr):
        return self._compute_binary_op(expr, 'SUB')

    ### PRIVATE METHODS ###

    def _add_constant_input(self, value):
        self._inputs.append(float(value))

    def _add_ugen_input(self, ugen, output_index=None):
        from supriya import audiolib
        if isinstance(ugen, audiolib.OutputProxy):
            output_proxy = ugen
        else:
            output_proxy = audiolib.OutputProxy(
                output_index=output_index,
                source=ugen,
                )
        self._inputs.append(output_proxy)

    def _collect_constants(self):
        from supriya import audiolib
        for input_ in self._inputs:
            if not isinstance(input_, audiolib.OutputProxy):
                self.synthdef._add_constant(float(input_))

    def _compute_binary_op(self, expr, op_name):
        from supriya import audiolib
        calculation_rate = self._compute_binary_rate(self, expr)
        operator = audiolib.BinaryOpUGen.BinaryOperator[op_name]
        special_index = operator.value
        return audiolib.BinaryOpUGen._new(
            calculation_rate=calculation_rate,
            left=self,
            right=expr,
            special_index=special_index,
            )

    @staticmethod
    def _compute_binary_rate(ugen_a, ugen_b):
        if ugen_a.calculation_rate == UGen.Rate.AUDIO_RATE:
            return UGen.Rate.AUDIO_RATE
        if hasattr(ugen_b, 'calculation_rate') and \
            ugen_b.calculation_rate == UGen.RAte.AUDIO_RATE:
            return UGen.Rate.AUDIO_RATE
        if ugen_a.calculation_rate == UGen.Rate.CONTROL_RATE:
            return UGen.Rate.CONTROL_RATE
        if hasattr(ugen_b, 'calculation_rate') and \
            ugen_b.calculation_rate == UGen.Rate.CONTROL_RATE:
            return UGen.Rate.CONTROL_RATE
        return UGen.Rate.SCALAR_RATE

    def _compute_unary_op(self, op_name):
        from supriya import audiolib
        operator = audiolib.UnaryOpUGen.UnaryOperator[op_name]
        special_index = operator.value
        return audiolib.UnaryOpUGen._new(
            calculation_rate=self.calculation_rate,
            source=self,
            special_index=special_index,
            )

    def _get_output_number(self):
        return 0

    def _get_outputs(self):
        return [self.calculation_rate]

    def _get_source(self):
        return self

    def _initialize_topological_sort(self):
        from supriya import audiolib
        for input_ in self.inputs:
            if isinstance(input_, audiolib.OutputProxy):
                ugen = input_.source
                if ugen not in self.antecedents:
                    self.antecedents.append(ugen)
                if self not in ugen.descendants:
                    ugen.descendants.append(self)
        for ugen in self.width_first_antecedents:
            if ugen not in self.antecedents:
                self.antecedents.append(ugen)
            if self not in ugen.descendants:
                ugen.descendants.append(self)

    def _make_available(self):
        if not self.antecedents:
            if self not in self.synthdef._available_ugens:
                self.synthdef._available_ugens.append(self)

    @classmethod
    def _new(cls, calculation_rate, special_index, **kwargs):
        import sys
        from supriya import audiolib
        if sys.version_info[0] == 2:
            import funcsigs
            get_signature = funcsigs.signature
        else:
            import inspect
            get_signature = inspect.signature
        assert isinstance(calculation_rate, UGen.Rate)
        argument_dicts = UGen.expand_arguments(
            kwargs, unexpanded_argument_names=cls._unexpanded_argument_names)
        ugens = []
        signature = get_signature(cls.__init__)
        has_custom_special_index = 'special_index' in signature.parameters
        for argument_dict in argument_dicts:
            if has_custom_special_index:
                ugen = cls(
                    calculation_rate=calculation_rate,
                    special_index=special_index,
                    **argument_dict
                    )
            else:
                ugen = cls(
                    calculation_rate=calculation_rate,
                    **argument_dict
                    )
            ugens.append(ugen)
        if len(ugens) == 1:
            return ugens[0]
        return audiolib.UGenArray(ugens)

    def _optimize_graph(self):
        pass

    def _remove_antecedent(self, ugen):
        self.antecedents.remove(ugen)
        self._make_available()

    def _schedule(self, out_stack):
        for ugen in reversed(self.descendants):
            ugen._remove_antecedent(self)
        out_stack.append(self)

    ### PUBLIC METHODS ###

    @classmethod
    def ar(cls, **kwargs):
        ugen = cls._new(
            calculation_rate=UGen.Rate.AUDIO_RATE,
            special_index=0,
            **kwargs
            )
        return ugen

    def compile(self, synthdef):
        def compile_input_spec(i, synthdef):
            from supriya import audiolib
            result = []
            if isinstance(i, float):
                result.append(SynthDef._encode_unsigned_int_32bit(0xffffffff))
                constant_index = synthdef._get_constant_index(i)
                result.append(SynthDef._encode_unsigned_int_32bit(
                    constant_index))
            elif isinstance(i, audiolib.OutputProxy):
                ugen = i.source
                output_index = i.output_index
                ugen_index = synthdef._get_ugen_index(ugen)
                result.append(SynthDef._encode_unsigned_int_32bit(ugen_index))
                result.append(SynthDef._encode_unsigned_int_32bit(output_index))
            else:
                raise Exception('Unhandled input spec: {}'.format(i))
            return bytearray().join(result)
        from supriya.library.audiolib import SynthDef
        outputs = self._get_outputs()
        result = []
        result.append(SynthDef._encode_string(type(self).__name__))
        result.append(SynthDef._encode_unsigned_int_8bit(self.calculation_rate))
        result.append(SynthDef._encode_unsigned_int_32bit(len(self.inputs)))
        result.append(SynthDef._encode_unsigned_int_32bit(len(outputs)))
        result.append(SynthDef._encode_unsigned_int_16bit(int(self.special_index)))
        for i in self.inputs:
            result.append(compile_input_spec(i, synthdef))
        for o in outputs:
            result.append(SynthDef._encode_unsigned_int_8bit(o))
        result = bytearray().join(result)
        return result

    @staticmethod
    def expand_arguments(arguments, unexpanded_argument_names=None):
        r'''Expands ugens into multichannel arrays.

        ::

            >>> import supriya
            >>> arguments = {'foo': 0, 'bar': (1, 2), 'baz': (3, 4, 5)}
            >>> result = supriya.audiolib.UGen.expand_arguments(arguments)
            >>> for x in result:
            ...     x
            ...
            {'bar': 1, 'foo': 0, 'baz': 3}
            {'bar': 2, 'foo': 0, 'baz': 4}
            {'bar': 1, 'foo': 0, 'baz': 5}

        ::

            >>> arguments = {'bus': (8, 9), 'source': (1, 2, 3)}
            >>> result = supriya.audiolib.UGen.expand_arguments(
            ...     arguments,
            ...     unexpanded_argument_names=('source',),
            ...     )
            >>> for x in result:
            ...     x
            ...
            {'bus': 8, 'source': (1, 2, 3)}
            {'bus': 9, 'source': (1, 2, 3)}

        '''
        cached_unexpanded_arguments = {}
        if unexpanded_argument_names is not None:
            for argument_name in unexpanded_argument_names:
                if argument_name not in arguments:
                    continue
                cached_unexpanded_arguments[argument_name] = \
                    arguments[argument_name]
                del(arguments[argument_name])
        maximum_length = 1
        result = []
        for name, value in arguments.items():
            if isinstance(value, collections.Sequence):
                maximum_length = max(maximum_length, len(value))
        for i in range(maximum_length):
            result.append({})
            for name, value in arguments.items():
                if isinstance(value, collections.Sequence):
                    value = value[i % len(value)]
                    result[i][name] = value
                else:
                    result[i][name] = value
        for expanded_arguments in result:
            expanded_arguments.update(cached_unexpanded_arguments)
        return result

    @classmethod
    def kr(cls, **kwargs):
        ugen = cls._new(
            calculation_rate=UGen.Rate.CONTROL_RATE,
            special_index=0,
            **kwargs
            )
        return ugen

    ### PUBLIC PROPERTIES ###

    @property
    def antecedents(self):
        return self._antecedents

    @property
    def calculation_rate(self):
        return self._calculation_rate

    @property
    def descendants(self):
        return self._descendants

    @property
    def inputs(self):
        return tuple(self._inputs)

    @property
    def signal_range(self):
        return self.SignalRange.BIPOLAR

    @property
    def special_index(self):
        return self._special_index

    @property
    def synthdef(self):
        return self._synthdef

    @synthdef.setter
    def synthdef(self, synthdef):
        from supriya import audiolib
        assert isinstance(synthdef, audiolib.SynthDef)
        self._synthdef = synthdef

    @property
    def width_first_antecedents(self):
        return self._width_first_antecedents
<|endoftext|>