<commit_before>#!/usr/bin/env bash

from __future__ import absolute_import, print_function, unicode_literals

import argparse
import json
import numpy as np
import os
import pandas as pd
import pickle
import sys
import tensorflow as tf

from itertools import compress
from keras import backend as keras_backend
from thesis.classification.semisupervised import ActiveLearningWrapper
from thesis.dataset import SenseCorpusDatasets, UnlabeledCorpusDataset
from thesis.dataset.utils import NotEnoughSensesError
from thesis.utils import try_number
from thesis.constants import CLASSIFIERS
from tqdm import tqdm

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('labeled_dataset_path')
    parser.add_argument('base_results_path')
    parser.add_argument('--unlabeled_dataset_path', default=None)
    parser.add_argument('--full_senses_path', default=None)
    parser.add_argument('--sentences_path', default=None)
    parser.add_argument('--simulation_indices_path', default=None)
    parser.add_argument('--word_vector_model_path', default=None)
    parser.add_argument('--labeled_dataset_extra', default=None)
    parser.add_argument('--unlabeled_dataset_extra', default=None)
    parser.add_argument('--classifier', type=str, default='svm')
    parser.add_argument('--classifier_config_file', type=str, default=None)
    parser.add_argument('--classifier_config', type=lambda config: tuple(config.split('=')),
                        default=list(), nargs='+')
    parser.add_argument('--layers', type=int, nargs='+', default=list())
    parser.add_argument('--unlabeled_data_limit', type=int, default=1000)
    parser.add_argument('--candidates_limit', type=int, default=0)
    parser.add_argument('--min_count', type=int, default=2)
    parser.add_argument('--validation_ratio', type=float, default=0.2)
    parser.add_argument('--candidates_selection', default='min')
    parser.add_argument('--error_sigma', type=float, default=0.1)
    parser.add_argument('--random_seed', type=int, default=1234)
    parser.add_argument('--folds', type=int, default=0)
    parser.add_argument('--annotation_lemmas', nargs='+', default=set())
    parser.add_argument('--corpus_name', default='NA')
    parser.add_argument('--representation', default='NA')
    parser.add_argument('--vector_domain', default='NA')

    args = parser.parse_args()

    if (args.unlabeled_dataset_path is None) == (args.simulation_indices_path is None):
        print('Either give an unlabeled dataset path or a simulation indices path', file=sys.stderr)
        sys.exit(1)

    if args.classifier_config_file is not None:
        with open(args.classifier_config_file, 'r') as fconfig:
            config = json.load(fconfig)
    else:
        config = {}

    args.classifier_config = [args.classifier_config] \
        if isinstance(args.classifier_config, tuple) else args.classifier_config

    if args.annotation_lemmas:
        args.annotation_lemmas = set(args.annotation_lemmas) if not isinstance(args.annotation_lemmas, set) else\
            args.annotation_lemmas

    for key, value in args.classifier_config:
        config[key] = try_number(value)

    if args.classifier == 'svm':
        config['kernel'] = 'linear'
        config['probability'] = True

    if args.layers:
        args.layers = [args.layers] if isinstance(args.layers, int) else args.layers
        config['layers'] = args.layers

    labeled_datasets_path = os.path.join(args.labeled_dataset_path, '%s_dataset.npz')
    labeled_features_path = os.path.join(args.labeled_dataset_path, '%s_features.p')
    labeled_datasets_extra_path = os.path.join(args.labeled_dataset_extra, '%s_dataset.npz') \
        if args.labeled_dataset_extra is not None else None

    print('Loading labeled dataset', file=sys.stderr)
    labeled_datasets = SenseCorpusDatasets(train_dataset_path=labeled_datasets_path % 'train',
                                           train_features_dict_path=labeled_features_path % 'train'
                                           if args.word_vector_model_path is None else None,
                                           test_dataset_path=labeled_datasets_path % 'test',
                                           test_features_dict_path=labeled_features_path % 'test'
                                           if args.word_vector_model_path is None else None,
                                           word_vector_model_path=args.word_vector_model_path,
                                           train_dataset_extra=labeled_datasets_extra_path % 'train'
                                           if labeled_datasets_extra_path is not None else None,
                                           test_dataset_extra=labeled_datasets_extra_path % 'test'
                                           if labeled_datasets_extra_path is not None else None)

    if args.unlabeled_dataset_path:
        unlabeled_dataset_path = os.path.join(args.unlabeled_dataset_path, 'dataset.npz')
        unlabeled_features_path = os.path.join(args.unlabeled_dataset_path, 'features.p')
        unlabeled_dataset_extra_path = os.path.join(args.unlabeled_dataset_extra, 'dataset.npz') \
            if args.unlabeled_dataset_extra is not None else None

        print('Loading unlabeled dataset', file=sys.stderr)
        unlabeled_dataset = UnlabeledCorpusDataset(dataset_path=unlabeled_dataset_path,
                                                   features_dict_path=unlabeled_features_path
                                                   if args.word_vector_model_path is None else None,
                                                   word_vector_model=labeled_datasets.train_dataset.word_vector_model,
                                                   dataset_extra=unlabeled_dataset_extra_path)
        initial_indices = None
        unlabeled_indices = None

    else:
        simulation_indices = np.load(args.simulation_indices_path)
        initial_indices = simulation_indices['initial_indices']
        unlabeled_indices = simulation_indices['unlabeled_indices']
        unlabeled_dataset = None
        full_senses_dict = None

    if args.full_senses_path is not None:
        with open(args.full_senses_path, 'rb') as f:
            full_senses_dict = pickle.load(f)

    prediction_results = []
    certainty_progression = []
    features_progression = []
    cross_validation_results = []
    senses = []
    results = (prediction_results, certainty_progression, features_progression, cross_validation_results)
    bootstrapped_instances = []
    bootstrapped_targets = []
    text_sentences = {}

    if args.sentences_path is not None:
        with open(args.sentences_path, 'r') as sfin:
            for line in sfin:
                iid, sent = line.strip().split('\t', 1)
                text_sentences[iid] = sent

    print('Running experiments per lemma', file=sys.stderr)
    for lemma, data, target, features in \
            tqdm(labeled_datasets.train_dataset.traverse_dataset_by_lemma(return_features=True),
                 total=labeled_datasets.train_dataset.num_lemmas):
        if unlabeled_dataset and not unlabeled_dataset.has_lemma(lemma):
            continue
        if args.annotation_lemmas and lemma not in args.annotation_lemmas:
            continue
        try:
            tf.reset_default_graph()
            with tf.Session() as sess:
                keras_backend.set_session(sess)

                if unlabeled_dataset is not None:
                    unlabeled_data = unlabeled_dataset.data(lemma, limit=args.unlabeled_data_limit)
                    unlabeled_target = np.array([])
                    unlabeled_features = unlabeled_dataset.features_dictionaries(lemma, limit=args.unlabeled_data_limit)
                    instances_id = unlabeled_dataset.instances_id(lemma, limit=args.unlabeled_data_limit)
                    lemma_unlabeled_sentences = [text_sentences[':'.join(iid)] for iid in instances_id]
                else:
                    li = np.in1d(labeled_datasets.train_dataset.lemmas_index(lemma), initial_indices)
                    ui = np.in1d(labeled_datasets.train_dataset.lemmas_index(lemma), unlabeled_indices)
                    unlabeled_data = data[ui]
                    unlabeled_target = target[ui]
                    unlabeled_features = list(compress(features, ui))
                    data = data[li]
                    target = target[li]
                    features = list(compress(features, li))
                    lemma_unlabeled_sentences = None

                test_data = labeled_datasets.test_dataset.data(lemma)
                test_target = labeled_datasets.test_dataset.target(lemma)

                _, zero_based_indices = np.unique(np.concatenate([target, unlabeled_target, test_target]),
                                                  return_inverse=True)
                train_classes = {label: idx for idx, label in
                                 enumerate(labeled_datasets.train_dataset.train_classes(lemma))}

                train_size = target.shape[0]
                unlabeled_size = unlabeled_target.shape[0]
                test_size = test_target.shape[0]

                target = zero_based_indices[:train_size]
                unlabeled_target = zero_based_indices[train_size:train_size+unlabeled_size]
                test_target = zero_based_indices[train_size+unlabeled_size:]

                semisupervised = ActiveLearningWrapper(
                    labeled_train_data=data, labeled_train_target=target, labeled_test_data=test_data,
                    labeled_test_target=test_target, unlabeled_data=unlabeled_data, unlabeled_target=unlabeled_target,
                    lemma=lemma, labeled_features=features, min_count=args.min_count, full_senses_dict=full_senses_dict,
                    validation_ratio=args.validation_ratio, candidates_selection=args.candidates_selection,
                    candidates_limit=args.candidates_limit, unlabeled_features=unlabeled_features,
                    error_sigma=args.error_sigma, folds=args.folds, random_seed=args.random_seed,
                    acceptance_threshold=0, unlabeled_sentences=lemma_unlabeled_sentences, train_classes=train_classes
                )

                if semisupervised.run(CLASSIFIERS[args.classifier], config) > 0:
                    for rst_agg, rst in zip(results, semisupervised.get_results()):
                        if rst is not None:
                            rst.insert(0, 'error_sigma', semisupervised.error_sigma)
                            rst.insert(0, 'folds', args.folds if args.folds > 0 else 'NA')
                            rst.insert(0, 'num_classes', semisupervised.classes.shape[0])
                            rst.insert(0, 'lemma', lemma)
                            rst.insert(0, 'candidates_limit', args.candidates_limit)
                            rst.insert(0, 'candidates_selection', args.candidates_selection)
                            rst.insert(0, 'layers', '_'.join(str(l) for l in args.layers) if args.layers else 'NA')
                            rst.insert(0, 'classifier', args.classifier)
                            rst.insert(0, 'vector_domain', args.vector_domain or 'NA')
                            rst.insert(0, 'representation', args.representation or 'NA')
                            rst.insert(0, 'corpus', args.corpus_name)
                            rst_agg.append(rst)

                    if args.full_senses_path is not None:
                        senses.append(semisupervised.get_senses())

                    # Save the bootstrapped data (if there is unlabeled data to save)
                    if unlabeled_dataset is not None:
                        bi, bt = semisupervised.bootstrapped()
                        bootstrapped_targets.extend(bt)

                        ul_instances = unlabeled_dataset.instances_id(lemma, limit=args.unlabeled_data_limit)
                        bootstrapped_instances.extend(':'.join(ul_instances[idx]) for idx in bi)
                else:
                    tqdm.write('Lemma %s - No iterations' % lemma, file=sys.stderr)
        except NotEnoughSensesError:
            tqdm.write('Lemma %s - Not enough senses with at least %d occurrences'
                       % (lemma, args.min_count), file=sys.stderr)
            continue

    print('Saving results', file=sys.stderr)

    if unlabeled_dataset is not None:
        pd.DataFrame({'instance': bootstrapped_instances, 'predicted_target': bootstrapped_targets}) \
            .to_csv('%s_unlabeled_dataset_predictions.csv' % args.base_results_path, index=False)
    pd.concat(prediction_results, ignore_index=True) \
        .to_csv('%s_prediction_results.csv' % args.base_results_path, index=False, float_format='%.2f')
    pd.concat(certainty_progression, ignore_index=True) \
        .to_csv('%s_certainty_progression.csv' % args.base_results_path, index=False, float_format='%.2e')
    pd.concat(features_progression, ignore_index=True) \
        .to_csv('%s_features_progression.csv' % args.base_results_path, index=False, float_format='%d')

    if senses:
        pd.concat(senses, ignore_index=True).\
            to_csv('%s_senses_description.csv' % args.base_results_path, index=False, float='%d')

    if cross_validation_results:
        pd.concat(cross_validation_results, ignore_index=True) \
            .to_csv('%s_cross_validation_results.csv' % args.base_results_path, index=False, float_format='%d')
<commit_msg>Added max iterations to metadata<commit_after>#!/usr/bin/env bash

from __future__ import absolute_import, print_function, unicode_literals

import argparse
import json
import numpy as np
import os
import pandas as pd
import pickle
import sys
import tensorflow as tf

from itertools import compress
from keras import backend as keras_backend
from thesis.classification.semisupervised import ActiveLearningWrapper
from thesis.dataset import SenseCorpusDatasets, UnlabeledCorpusDataset
from thesis.dataset.utils import NotEnoughSensesError
from thesis.utils import try_number
from thesis.constants import CLASSIFIERS
from tqdm import tqdm

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('labeled_dataset_path')
    parser.add_argument('base_results_path')
    parser.add_argument('--unlabeled_dataset_path', default=None)
    parser.add_argument('--full_senses_path', default=None)
    parser.add_argument('--sentences_path', default=None)
    parser.add_argument('--simulation_indices_path', default=None)
    parser.add_argument('--word_vector_model_path', default=None)
    parser.add_argument('--labeled_dataset_extra', default=None)
    parser.add_argument('--unlabeled_dataset_extra', default=None)
    parser.add_argument('--classifier', type=str, default='svm')
    parser.add_argument('--classifier_config_file', type=str, default=None)
    parser.add_argument('--classifier_config', type=lambda config: tuple(config.split('=')),
                        default=list(), nargs='+')
    parser.add_argument('--layers', type=int, nargs='+', default=list())
    parser.add_argument('--unlabeled_data_limit', type=int, default=1000)
    parser.add_argument('--candidates_limit', type=int, default=0)
    parser.add_argument('--min_count', type=int, default=2)
    parser.add_argument('--validation_ratio', type=float, default=0.2)
    parser.add_argument('--candidates_selection', default='min')
    parser.add_argument('--error_sigma', type=float, default=0.1)
    parser.add_argument('--random_seed', type=int, default=1234)
    parser.add_argument('--folds', type=int, default=0)
    parser.add_argument('--annotation_lemmas', nargs='+', default=set())
    parser.add_argument('--corpus_name', default='NA')
    parser.add_argument('--representation', default='NA')
    parser.add_argument('--vector_domain', default='NA')

    args = parser.parse_args()

    if (args.unlabeled_dataset_path is None) == (args.simulation_indices_path is None):
        print('Either give an unlabeled dataset path or a simulation indices path', file=sys.stderr)
        sys.exit(1)

    if args.classifier_config_file is not None:
        with open(args.classifier_config_file, 'r') as fconfig:
            config = json.load(fconfig)
    else:
        config = {}

    args.classifier_config = [args.classifier_config] \
        if isinstance(args.classifier_config, tuple) else args.classifier_config

    if args.annotation_lemmas:
        args.annotation_lemmas = set(args.annotation_lemmas) if not isinstance(args.annotation_lemmas, set) else\
            args.annotation_lemmas

    for key, value in args.classifier_config:
        config[key] = try_number(value)

    if args.classifier == 'svm':
        config['kernel'] = 'linear'
        config['probability'] = True

    if args.layers:
        args.layers = [args.layers] if isinstance(args.layers, int) else args.layers
        config['layers'] = args.layers

    labeled_datasets_path = os.path.join(args.labeled_dataset_path, '%s_dataset.npz')
    labeled_features_path = os.path.join(args.labeled_dataset_path, '%s_features.p')
    labeled_datasets_extra_path = os.path.join(args.labeled_dataset_extra, '%s_dataset.npz') \
        if args.labeled_dataset_extra is not None else None

    print('Loading labeled dataset', file=sys.stderr)
    labeled_datasets = SenseCorpusDatasets(train_dataset_path=labeled_datasets_path % 'train',
                                           train_features_dict_path=labeled_features_path % 'train'
                                           if args.word_vector_model_path is None else None,
                                           test_dataset_path=labeled_datasets_path % 'test',
                                           test_features_dict_path=labeled_features_path % 'test'
                                           if args.word_vector_model_path is None else None,
                                           word_vector_model_path=args.word_vector_model_path,
                                           train_dataset_extra=labeled_datasets_extra_path % 'train'
                                           if labeled_datasets_extra_path is not None else None,
                                           test_dataset_extra=labeled_datasets_extra_path % 'test'
                                           if labeled_datasets_extra_path is not None else None)

    if args.unlabeled_dataset_path:
        unlabeled_dataset_path = os.path.join(args.unlabeled_dataset_path, 'dataset.npz')
        unlabeled_features_path = os.path.join(args.unlabeled_dataset_path, 'features.p')
        unlabeled_dataset_extra_path = os.path.join(args.unlabeled_dataset_extra, 'dataset.npz') \
            if args.unlabeled_dataset_extra is not None else None

        print('Loading unlabeled dataset', file=sys.stderr)
        unlabeled_dataset = UnlabeledCorpusDataset(dataset_path=unlabeled_dataset_path,
                                                   features_dict_path=unlabeled_features_path
                                                   if args.word_vector_model_path is None else None,
                                                   word_vector_model=labeled_datasets.train_dataset.word_vector_model,
                                                   dataset_extra=unlabeled_dataset_extra_path)
        initial_indices = None
        unlabeled_indices = None

    else:
        simulation_indices = np.load(args.simulation_indices_path)
        initial_indices = simulation_indices['initial_indices']
        unlabeled_indices = simulation_indices['unlabeled_indices']
        unlabeled_dataset = None
        full_senses_dict = None

    if args.full_senses_path is not None:
        with open(args.full_senses_path, 'rb') as f:
            full_senses_dict = pickle.load(f)

    prediction_results = []
    certainty_progression = []
    features_progression = []
    cross_validation_results = []
    senses = []
    results = (prediction_results, certainty_progression, features_progression, cross_validation_results)
    bootstrapped_instances = []
    bootstrapped_targets = []
    text_sentences = {}

    if args.sentences_path is not None:
        with open(args.sentences_path, 'r') as sfin:
            for line in sfin:
                iid, sent = line.strip().split('\t', 1)
                text_sentences[iid] = sent

    print('Running experiments per lemma', file=sys.stderr)
    for lemma, data, target, features in \
            tqdm(labeled_datasets.train_dataset.traverse_dataset_by_lemma(return_features=True),
                 total=labeled_datasets.train_dataset.num_lemmas):
        if unlabeled_dataset and not unlabeled_dataset.has_lemma(lemma):
            continue
        if args.annotation_lemmas and lemma not in args.annotation_lemmas:
            continue
        try:
            tf.reset_default_graph()
            with tf.Session() as sess:
                keras_backend.set_session(sess)

                if unlabeled_dataset is not None:
                    unlabeled_data = unlabeled_dataset.data(lemma, limit=args.unlabeled_data_limit)
                    unlabeled_target = np.array([])
                    unlabeled_features = unlabeled_dataset.features_dictionaries(lemma, limit=args.unlabeled_data_limit)
                    instances_id = unlabeled_dataset.instances_id(lemma, limit=args.unlabeled_data_limit)
                    lemma_unlabeled_sentences = [text_sentences[':'.join(iid)] for iid in instances_id]
                else:
                    li = np.in1d(labeled_datasets.train_dataset.lemmas_index(lemma), initial_indices)
                    ui = np.in1d(labeled_datasets.train_dataset.lemmas_index(lemma), unlabeled_indices)
                    unlabeled_data = data[ui]
                    unlabeled_target = target[ui]
                    unlabeled_features = list(compress(features, ui))
                    data = data[li]
                    target = target[li]
                    features = list(compress(features, li))
                    lemma_unlabeled_sentences = None

                test_data = labeled_datasets.test_dataset.data(lemma)
                test_target = labeled_datasets.test_dataset.target(lemma)

                _, zero_based_indices = np.unique(np.concatenate([target, unlabeled_target, test_target]),
                                                  return_inverse=True)
                train_classes = {label: idx for idx, label in
                                 enumerate(labeled_datasets.train_dataset.train_classes(lemma))}

                train_size = target.shape[0]
                unlabeled_size = unlabeled_target.shape[0]
                test_size = test_target.shape[0]

                target = zero_based_indices[:train_size]
                unlabeled_target = zero_based_indices[train_size:train_size+unlabeled_size]
                test_target = zero_based_indices[train_size+unlabeled_size:]

                semisupervised = ActiveLearningWrapper(
                    labeled_train_data=data, labeled_train_target=target, labeled_test_data=test_data,
                    labeled_test_target=test_target, unlabeled_data=unlabeled_data, unlabeled_target=unlabeled_target,
                    lemma=lemma, labeled_features=features, min_count=args.min_count, full_senses_dict=full_senses_dict,
                    validation_ratio=args.validation_ratio, candidates_selection=args.candidates_selection,
                    candidates_limit=args.candidates_limit, unlabeled_features=unlabeled_features,
                    error_sigma=args.error_sigma, folds=args.folds, random_seed=args.random_seed,
                    acceptance_threshold=0, unlabeled_sentences=lemma_unlabeled_sentences, train_classes=train_classes
                )

                iterations = semisupervised.run(CLASSIFIERS[args.classifier], config)

                if iterations > 0:
                    for rst_agg, rst in zip(results, semisupervised.get_results()):
                        if rst is not None:
                            rst.insert(0, 'max_iterations', iterations)
                            rst.insert(0, 'error_sigma', semisupervised.error_sigma)
                            rst.insert(0, 'folds', args.folds if args.folds > 0 else 'NA')
                            rst.insert(0, 'num_classes', semisupervised.classes.shape[0])
                            rst.insert(0, 'lemma', lemma)
                            rst.insert(0, 'candidates_limit', args.candidates_limit)
                            rst.insert(0, 'candidates_selection', args.candidates_selection)
                            rst.insert(0, 'layers', '_'.join(str(l) for l in args.layers) if args.layers else 'NA')
                            rst.insert(0, 'classifier', args.classifier)
                            rst.insert(0, 'vector_domain', args.vector_domain or 'NA')
                            rst.insert(0, 'representation', args.representation or 'NA')
                            rst.insert(0, 'corpus', args.corpus_name)
                            rst_agg.append(rst)

                    if args.full_senses_path is not None:
                        senses.append(semisupervised.get_senses())

                    # Save the bootstrapped data (if there is unlabeled data to save)
                    if unlabeled_dataset is not None:
                        bi, bt = semisupervised.bootstrapped()
                        bootstrapped_targets.extend(bt)

                        ul_instances = unlabeled_dataset.instances_id(lemma, limit=args.unlabeled_data_limit)
                        bootstrapped_instances.extend(':'.join(ul_instances[idx]) for idx in bi)
                else:
                    tqdm.write('Lemma %s - No iterations' % lemma, file=sys.stderr)
        except NotEnoughSensesError:
            tqdm.write('Lemma %s - Not enough senses with at least %d occurrences'
                       % (lemma, args.min_count), file=sys.stderr)
            continue

    print('Saving results', file=sys.stderr)

    if unlabeled_dataset is not None:
        pd.DataFrame({'instance': bootstrapped_instances, 'predicted_target': bootstrapped_targets}) \
            .to_csv('%s_unlabeled_dataset_predictions.csv' % args.base_results_path, index=False)
    pd.concat(prediction_results, ignore_index=True) \
        .to_csv('%s_prediction_results.csv' % args.base_results_path, index=False, float_format='%.2f')
    pd.concat(certainty_progression, ignore_index=True) \
        .to_csv('%s_certainty_progression.csv' % args.base_results_path, index=False, float_format='%.2e')
    pd.concat(features_progression, ignore_index=True) \
        .to_csv('%s_features_progression.csv' % args.base_results_path, index=False, float_format='%d')

    if senses:
        pd.concat(senses, ignore_index=True).\
            to_csv('%s_senses_description.csv' % args.base_results_path, index=False, float='%d')

    if cross_validation_results:
        pd.concat(cross_validation_results, ignore_index=True) \
            .to_csv('%s_cross_validation_results.csv' % args.base_results_path, index=False, float_format='%d')
<|endoftext|>


<commit_before>import logging
import urllib
import csv
import http
import xml.etree.ElementTree as etree

from dipper.sources.Source import Source
from dipper.models.Model import Model
from dipper.models.Genotype import Genotype

LOG = logging.getLogger(__name__)
ENS_URL = 'uswest.ensembl.org'    # 'www.ensembl.org'

class Ensembl(Source):
    """
    This is the processing module for Ensembl.

    It only includes methods to acquire the equivalences between NCBIGene and
    ENSG ids using ENSEMBL's Biomart services.

    """

    files = {
        '9606': {'file': 'ensembl_9606.txt'},  # human
        '7955': {'file': 'ensembl_7955.txt'},  # fish
        '10090': {'file': 'ensembl_10090.txt'},  # mouse
        '28377': {'file': 'ensembl_28377.txt'},  #
        '3702': {'file': 'ensembl_3702.txt'},  #
        '9913': {'file': 'ensembl_9913.txt'},  #
        '6239': {'file': 'ensembl_6239.txt'},  #
        '9615': {'file': 'ensembl_9615.txt'},  #
        '9031': {'file': 'ensembl_9031.txt'},  #
        '44689': {'file': 'ensembl_44689.txt'},  #
        '7227': {'file': 'ensembl_7227.txt'},  #
        '9796': {'file': 'ensembl_9796.txt'},  #
        '9544': {'file': 'ensembl_9544.txt'},  #
        '13616': {'file': 'ensembl_13616.txt'},  #
        '9258': {'file': 'ensembl_9258.txt'},  #
        '9823': {'file': 'ensembl_9823.txt'},  #
        '10116': {'file': 'ensembl_10116.txt'},  #
        '4896': {'file': 'ensembl_4896.txt'},  #
        '31033': {'file': 'ensembl_31033.txt'},  #
        '8364': {'file': 'ensembl_8364.txt'},  #
        '4932': {'file': 'ensembl_4932.txt'},  #
    }
    columns = {
        'bmq_prot_gene': [
            'ensembl_gene_id',
            'external_gene_name',
            'description',
            'gene_biotype',
            'entrezgene',
            'ensembl_peptide_id',
            'uniprotswissprot',
        ],
        "ensembl_biomart": [
            "ensembl_gene_id",
            "external_gene_name",
            "description",
            "gene_biotype",
            "entrezgene",
            "ensembl_peptide_id",
            "uniprotswissprot",     # not in all queries
            "hgnc_id",              # not in all queries (human only)
        ]
    }

    def __init__(self, graph_type, are_bnodes_skolemized, tax_ids=None, gene_ids=None):
        super().__init__(
            graph_type,
            are_bnodes_skolemized,
            'ensembl',
            ingest_title='ENSEMBL',
            ingest_url='http://uswest.ensembl.org'
            # license_url=None,
            # data_rights=None,
            # file_handle=None
        )

        self.tax_ids = tax_ids
        self.gene_ids = gene_ids

        # Defaults
        if self.tax_ids is None:
            self.tax_ids = ['9606', '10090', '7955']

        self.tax_ids = [str(x) for x in self.tax_ids]  # for bare ints from commandline

        self.gene_ids = []
        if 'gene' not in self.all_test_ids:
            LOG.warning("not configured with gene test ids.")
        else:
            self.gene_ids = self.all_test_ids['gene']

        LOG.setLevel(logging.INFO)

        return

    def fetch(self, is_dl_forced=False):

        col = self.columns['ensembl_biomart']
        for txid in self.tax_ids:
            LOG.info("Fetching genes for %s", txid)
            loc_file = '/'.join((self.rawdir, 'ensembl_' + txid + '.txt'))
            # todo move into util?
            params = urllib.parse.urlencode(
                {'query': self._build_biomart_gene_query(txid, col)})
            conn = http.client.HTTPConnection(ENS_URL)
            conn.request("GET", '/biomart/martservice?' + params)
            resp = conn.getresponse()
            with open(loc_file, 'wb') as bin_writer:
                bin_writer.write(resp.read())

        return

    def parse(self, limit=None):
        if limit is not None:
            LOG.info("Only parsing first %d rows", limit)

        if self.test_only:
            self.test_mode = True

        LOG.info("Parsing files...")

        for txid in self.tax_ids:
            self._process_genes(txid, limit)

        LOG.info("Done parsing files.")

        return

    def fetch_protein_list(self, taxon_id):
        """
        Fetch a list of proteins for a species in biomart
        :param taxid:
        :return: list
        """
        protein_list = list()
        # col = self.columns['ensembl_biomart']
        col = ['ensembl_peptide_id', ]

        params = urllib.parse.urlencode(
            {'query': self._build_biomart_gene_query(taxon_id, col)})
        conn = http.client.HTTPConnection(ENS_URL)
        conn.request("GET", '/biomart/martservice?' + params)
        response = conn.getresponse()

        for line in response:
            line = line.decode('utf-8').rstrip()
            row = line.split('\t')
            if len(row) != len(col):
                LOG.warning("Data error for p-list query on %d", taxon_id)
                continue
            protein_list.append(row[col.index('ensembl_peptide_id')])
        conn.close()
        return protein_list

    def fetch_protein_gene_map(self, taxon_id):
        """
        Fetch a list of proteins for a species in biomart
        :param taxid:
        :return: dict
        """
        protein_dict = dict()
        # col = self.columns['ensembl_biomart']
        col = ['ensembl_peptide_id', 'ensembl_gene_id']
        raw_query = self._build_biomart_gene_query(taxon_id, col)

        params = urllib.parse.urlencode({'query': raw_query})

        conn = http.client.HTTPConnection(ENS_URL)
        conn.request("GET", '/biomart/martservice?' + params)
        response = conn.getresponse()

        for line in response:
            line = line.decode('utf-8').rstrip()
            row = line.split('\t')

            if len(row) != len(col):
                LOG.warning("Data error for p2g query on %s", taxon_id)
                LOG.warning("Expected columns for \n%s", col)
                LOG.warning("Got data \n%s", row)
                continue

            protein_dict[
                row[col.index('ensembl_peptide_id')]] = [
                    row[col.index('ensembl_gene_id')]
                ]

        conn.close()
        LOG.info(
            "length protien list for taxon: %s is %i", taxon_id, len(protein_dict))
        return protein_dict

    def fetch_uniprot_gene_map(self, taxon_id):
        """
        Fetch a dict of uniprot-gene  for a species in biomart
        :param taxid:
        :return: dict
        """
        protein_dict = dict()
        # col = self.columns['biomart_query']
        col = ['uniprotswissprot', 'ensembl_gene_id']
        params = urllib.parse.urlencode(
            {'query': self._build_biomart_gene_query(taxon_id, col)})
        conn = http.client.HTTPConnection(ENS_URL)
        conn.request("GET", '/biomart/martservice?' + params)
        response = conn.getresponse()
        for line in response:
            line = line.decode('utf-8').rstrip()
            row = line.split('\t')
            if len(row) != len(col):
                continue
            protein_dict[
                row[col.index('uniprotswissprot')]] = row[col.index('ensembl_gene_id')]
        conn.close()
        return protein_dict

    def _build_biomart_gene_query(self, taxid, cols_to_fetch):
        """
        Building url to fetch equivalent identifiers via Biomart Restful API.
        Documentation at
        http://uswest.ensembl.org/info/data/biomart/biomart_restful.html
        :param taxid:
        :param array of ensembl biomart attributes to include
        :return:

        """

        taxid = str(taxid)
        # basic stuff for ensembl ids.

        if taxid != '9606':  # drop hgnc column
            cols_to_fetch = [x for x in cols_to_fetch if x != 'hgnc_id']

        # LOG.info('Build BMQ with taxon %s and mapping %s', taxid, self.localtt)

        query_attributes = {
            "virtualSchemaName": "default", "formatter": "TSV", "header": "0",
            "uniqueRows": "1", "count": "0", "datasetConfigVersion": "0.6"}

        qry = etree.Element("Query", query_attributes)

        if taxid in self.localtt:
            object_attributes = {"name": self.localtt[taxid], "interface": "default"}
            dataset = etree.SubElement(qry, "Dataset", object_attributes)
            for col in cols_to_fetch:
                etree.SubElement(dataset, "Attribute", {"name": col})
            # is indent right?
            query = '<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE Query>' \
                + etree.tostring(qry, encoding="unicode")
        else:
            LOG.warning("not finding taxon  %s in the local translation table", taxid)
            query = None

        return query

    def _process_genes(self, taxid, limit=None):
        if self.test_mode:
            graph = self.testgraph
        else:
            graph = self.graph

        model = Model(graph)
        geno = Genotype(graph)

        raw = '/'.join((self.rawdir, self.files[taxid]['file']))
        line_counter = 0
        LOG.info("Processing Ensembl genes for tax %s", taxid)
        with open(raw, 'r', encoding="utf8") as csvfile:
            filereader = csv.reader(csvfile, delimiter='\t')
            for row in filereader:
                if len(row) < 4:
                    LOG.warning("Too few columns in: " + row)
                    raise ValueError("Data error for file %s", raw)
                (ensembl_gene_id, external_gene_name, description, gene_biotype,
                 entrezgene, ensembl_peptide_id, uniprotswissprot) = row[0:7]

                # in the case of human genes, we also get the hgnc id,
                # and is the last col
                if taxid == '9606':
                    hgnc_id = row[7]
                else:
                    hgnc_id = None

                if self.test_mode and entrezgene != '' and \
                        int(entrezgene) not in self.gene_ids:
                    continue

                line_counter += 1
                gene_id = 'ENSEMBL:' + ensembl_gene_id
                peptide_curie = 'ENSEMBL:{}'.format(ensembl_peptide_id)
                uniprot_curie = 'UniProtKB:{}'.format(uniprotswissprot)
                entrez_curie = 'NCBIGene:{}'.format(entrezgene)

                if description == '':
                    description = None
                gene_biotype = gene_biotype.strip()
                gene_type_id = self.resolve(gene_biotype, False)
                if gene_type_id == gene_biotype.strip():   # did not resolve
                    gene_type_id = self.globaltt['polypeptide']

                model.addClassToGraph(
                    gene_id, external_gene_name, gene_type_id, description)
                model.addIndividualToGraph(peptide_curie, None, gene_type_id)
                model.addIndividualToGraph(uniprot_curie, None, gene_type_id)

                if entrezgene != '':
                    if taxid == '9606':
                        # Use HGNC for eq in human data
                        model.addXref(gene_id, entrez_curie)
                    else:
                        model.addEquivalentClass(gene_id, entrez_curie)
                if hgnc_id is not None and hgnc_id != '':
                    model.addEquivalentClass(gene_id, hgnc_id)
                geno.addTaxon('NCBITaxon:'+taxid, gene_id)
                if ensembl_peptide_id != '':
                    geno.addGeneProduct(gene_id, peptide_curie)
                    if uniprotswissprot != '':
                        geno.addGeneProduct(gene_id, uniprot_curie)
                        model.addXref(peptide_curie, uniprot_curie)

                if not self.test_mode and limit is not None and line_counter > limit:
                    break

        return

    def getTestSuite(self):
        import unittest
        from tests.test_ensembl import EnsemblTestCase
        return unittest.TestLoader().loadTestsFromTestCase(EnsemblTestCase)
<commit_msg>significant std refactor, addressed wonky columns error<commit_after>import logging
import urllib
import csv
import http
import xml.etree.ElementTree as etree

from dipper.sources.Source import Source
from dipper.models.Model import Model
from dipper.models.Genotype import Genotype

LOG = logging.getLogger(__name__)
ENS_URL = 'uswest.ensembl.org'    # 'www.ensembl.org'

class Ensembl(Source):
    """
    This is the processing module for Ensembl.

    It only includes methods to acquire the equivalences between NCBIGene and
    ENSG ids using ENSEMBL's Biomart services.

    """

    files = {
        '9606': {'file': 'ensembl_9606.txt'},  # human
        '7955': {'file': 'ensembl_7955.txt'},  # fish
        '10090': {'file': 'ensembl_10090.txt'},  # mouse
        '28377': {'file': 'ensembl_28377.txt'},  #
        '3702': {'file': 'ensembl_3702.txt'},  #
        '9913': {'file': 'ensembl_9913.txt'},  #
        '6239': {'file': 'ensembl_6239.txt'},  #
        '9615': {'file': 'ensembl_9615.txt'},  #
        '9031': {'file': 'ensembl_9031.txt'},  #
        '44689': {'file': 'ensembl_44689.txt'},  #
        '7227': {'file': 'ensembl_7227.txt'},  #
        '9796': {'file': 'ensembl_9796.txt'},  #
        '9544': {'file': 'ensembl_9544.txt'},  #
        '13616': {'file': 'ensembl_13616.txt'},  #
        '9258': {'file': 'ensembl_9258.txt'},  #
        '9823': {'file': 'ensembl_9823.txt'},  #
        '10116': {'file': 'ensembl_10116.txt'},  #
        '4896': {'file': 'ensembl_4896.txt'},  #
        '31033': {'file': 'ensembl_31033.txt'},  #
        '8364': {'file': 'ensembl_8364.txt'},  #
        '4932': {'file': 'ensembl_4932.txt'},  #
    }
    columns = {
        "bmq_attributes": [  # to be sent
            "ensembl_gene_id",
            "external_gene_name",
            "description",
            "gene_biotype",
            "entrezgene_id",
            "ensembl_peptide_id",
            "uniprotswissprot",
            "hgnc_id",   # human only
        ],
        'bmq_headers': [  # to be recived
            'Gene stable ID',
            'Gene name',
            'Gene description',
            'Gene type',
            'NCBI gene ID',
            'Protein stable ID',
            'UniProtKB/Swiss-Prot ID',
            'HGNC ID',  # human only
        ]
    }

    def __init__(self, graph_type, are_bnodes_skolemized, tax_ids=None, gene_ids=None):
        super().__init__(
            graph_type,
            are_bnodes_skolemized,
            'ensembl',
            ingest_title='ENSEMBL',
            ingest_url='http://uswest.ensembl.org'
            # license_url=None,
            # data_rights=None,
            # file_handle=None
        )

        self.tax_ids = tax_ids
        self.gene_ids = gene_ids

        # Defaults
        if self.tax_ids is None:
            self.tax_ids = ['9606', '10090', '7955']

        self.tax_ids = [str(x) for x in self.tax_ids]  # for bare ints from commandline

        LOG.info('Have Taxon ID(s)  %s', self.tax_ids)

        self.gene_ids = []
        if 'gene' not in self.all_test_ids:
            LOG.warning("not configured with gene test ids.")
        else:
            self.gene_ids = self.all_test_ids['gene']

        LOG.setLevel(logging.INFO)

    def fetch(self, is_dl_forced=True):  # it is a database query... so no timestamps

        for txid in self.tax_ids:
            LOG.info("Fetching genes for %s", txid)
            loc_file = '/'.join((self.rawdir, 'ensembl_' + txid + '.txt'))

            # todo move into util?
            params = urllib.parse.urlencode(
                {'query': self._build_biomart_gene_query(txid)})
            conn = http.client.HTTPConnection(ENS_URL)
            conn.request("GET", '/biomart/martservice?' + params)
            resp = conn.getresponse()
            with open(loc_file, 'wb') as bin_writer:
                bin_writer.write(resp.read())

    def parse(self, limit=None):
        if limit is not None:
            LOG.info("Only parsing first %d rows", limit)

        if self.test_only:
            self.test_mode = True

        LOG.info("Parsing files...")

        for txid in self.tax_ids:
            self._process_genes(txid, limit)

        LOG.info("Done parsing files.")

    # this seems so un specific as to be a bit pointless
    # and will be deleted if there are no objections
    # def fetch_protein_list(self, taxon_id):
    #    """
    #    Fetch a list of proteins for a species in biomart
    #    :param taxid:
    #    :return: list
    #    """
    #    protein_list = list()
    #    att = ['ensembl_peptide_id', ]
    #
    #    params = urllib.parse.urlencode(
    #        {'query': self._build_biomart_gene_query(taxon_id, att)})
    #    conn = http.client.HTTPConnection(ENS_URL)
    #    conn.request("GET", '/biomart/martservice?' + params)
    #    response = conn.getresponse()
    #    header = next(response).rstrip().split('\t')
    #    for line in response:
    #        line = line.decode('utf-8').rstrip()
    #        row = line.split('\t')
    #        if len(row) != len(att):
    #            LOG.warning("Data error for p-list query on %d", taxon_id)
    #            continue
    #        protein_list.append(row[att.index('ensembl_peptide_id')])
    #    conn.close()
    #    return protein_list

    def fetch_protein_gene_map(self, taxon_id):
        """
        Fetch a list of proteins for a species in biomart
        :param taxid:
        :return: dict
        """
        # called in StringDB.py
        protein_dict = dict()
        col = ['ensembl_gene_id', 'ensembl_peptide_id']
        col_exp = [
            self.columns['bmq_headers'][col.index('ensembl_gene_id')],
            self.columns['bmq_headers'][col.index('ensembl_peptide_id')],
        ]
        raw_query = self._build_biomart_gene_query(taxon_id, col)

        params = urllib.parse.urlencode({'query': raw_query})

        conn = http.client.HTTPConnection(ENS_URL)
        conn.request("GET", '/biomart/martservice?' + params)
        response = conn.getresponse()
        row = next(response).rstrip().split('\t')
        if not self.check_fileheader(col_exp, row):
            pass  #

        for line in response:
            line = line.decode('utf-8').rstrip()
            row = line.split('\t')
            protein_dict[
                row[col.index('ensembl_peptide_id')]] = [
                    row[col.index('ensembl_gene_id')]
                ]

        conn.close()
        LOG.info(
            "length gene-protien dict for taxon: %s is %i", taxon_id, len(protein_dict))
        return protein_dict

    def fetch_uniprot_gene_map(self, taxon_id):
        """
        Fetch a dict of uniprot-gene  for a species in biomart
        :param taxid:
        :return: dict
        """
        # unused
        protein_dict = dict()
        col = ['uniprotswissprot', 'ensembl_gene_id']
        params = urllib.parse.urlencode(
            {'query': self._build_biomart_gene_query(taxon_id, col)})
        conn = http.client.HTTPConnection(ENS_URL)
        conn.request("GET", '/biomart/martservice?' + params)
        response = conn.getresponse()
        for line in response:
            line = line.decode('utf-8').rstrip()
            row = line.split('\t')
            if len(row) != len(col):
                continue
            protein_dict[
                row[col.index('uniprotswissprot')]] = row[col.index('ensembl_gene_id')]
        conn.close()
        return protein_dict

    def _build_biomart_gene_query(
            self, taxid, cols_to_fetch=list(columns['bmq_attributes'])):
        """
        Building url to fetch equivalent identifiers via Biomart Restful API.
        Documentation at
        http://uswest.ensembl.org/info/data/biomart/biomart_restful.html
        :param taxid:
        :param array of ensembl biomart attributes to include
        :return:

        """

        if taxid != '9606' and 'hgnc_id' in cols_to_fetch:
            cols_to_fetch.remove('hgnc_id')

        query_attributes = {
            "virtualSchemaName": "default", "formatter": "TSV", "header": "1",
            "uniqueRows": "1", "count": "0", "datasetConfigVersion": "0.6"}

        qry = etree.Element("Query", query_attributes)

        if taxid in self.localtt:
            object_attributes = {"name": self.localtt[taxid], "interface": "default"}
            dataset = etree.SubElement(qry, "Dataset", object_attributes)
            for col in cols_to_fetch:
                etree.SubElement(dataset, "Attribute", {"name": col})
            # is indent right?
            query = '<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE Query>' \
                + etree.tostring(qry, encoding="unicode")
        else:
            LOG.warning("not finding taxon  %s in the local translation table", taxid)
            query = None

        return query

    def _process_genes(self, taxid, limit=None):
        if self.test_mode:
            graph = self.testgraph
        else:
            graph = self.graph

        model = Model(graph)
        geno = Genotype(graph)

        raw = '/'.join((self.rawdir, self.files[taxid]['file']))
        col = list(self.columns['bmq_attributes'])
        if taxid != '9606' and 'hgnc_id' in col:
            col.remove('hgnc_id')
        col_exp = [
            self.columns['bmq_headers'][self.columns['bmq_attributes'].index(x)]
            for x in col]

        LOG.info("Processing Ensembl genes for NCBITaxon:%s", taxid)
        with open(raw, 'r', encoding="utf8") as csvfile:
            reader = csv.reader(csvfile, delimiter='\t')
            row = next(reader)
            if not self.check_fileheader(col_exp, row):
                pass
            for row in reader:
                ensembl_gene_id = row[col.index('ensembl_gene_id')]
                external_gene_name = row[col.index('external_gene_name')]
                description = row[col.index('description')].strip()
                gene_biotype = row[col.index('gene_biotype')].strip()
                entrezgene = row[col.index('entrezgene_id')].strip()
                ensembl_peptide_id = row[col.index('ensembl_peptide_id')].strip()
                uniprotswissprot = row[col.index('uniprotswissprot')].strip()
                hgnc_id = None
                # in the case of human genes, we also get the hgnc id,
                if taxid == '9606' and 'hgnc_id' in col:
                    hgnc_id = row[col.index('hgnc_id')].strip()

                if self.test_mode and entrezgene != '' and \
                        entrezgene not in self.gene_ids:
                    continue

                gene_id = 'ENSEMBL:' + ensembl_gene_id
                entrez_curie = 'NCBIGene:{}'.format(entrezgene)

                if description == '':
                    description = None

                gene_type_id = self.resolve(
                    gene_biotype, mandatory=False,
                    default=self.globaltt['polypeptide'])

                model.addClassToGraph(
                    gene_id, external_gene_name, gene_type_id, description)

                if entrezgene != '':
                    if taxid == '9606':
                        # Use HGNC for eq in human data
                        model.addXref(gene_id, entrez_curie)
                    else:
                        model.addEquivalentClass(gene_id, entrez_curie)

                if hgnc_id is not None and hgnc_id != '':
                    model.addEquivalentClass(gene_id, 'HGNC:' + hgnc_id)
                geno.addTaxon('NCBITaxon:' + taxid, gene_id)
                if ensembl_peptide_id != '':
                    peptide_curie = 'ENSEMBL:{}'.format(ensembl_peptide_id)
                    model.addIndividualToGraph(peptide_curie, None, gene_type_id)
                    geno.addGeneProduct(gene_id, peptide_curie)
                    if uniprotswissprot != '':
                        uniprot_curie = 'UniProtKB:{}'.format(uniprotswissprot)
                        model.addIndividualToGraph(uniprot_curie, None, gene_type_id)
                        geno.addGeneProduct(gene_id, uniprot_curie)
                        model.addXref(peptide_curie, uniprot_curie)

                if not self.test_mode and limit is not None and reader.line_num > limit:
                    break

    def getTestSuite(self):
        import unittest
        from tests.test_ensembl import EnsemblTestCase
        return unittest.TestLoader().loadTestsFromTestCase(EnsemblTestCase)
<|endoftext|>


<commit_before>from appliance import *
from utils import *
import evaluation_metrics
import PecanStreetDatasetAdapter
from TracebaseDatasetAdapter import TracebaseDatasetAdapter
import utils
import appliance
<commit_msg>update init file with docstring<commit_after>"""
.. module:: disaggregator
   :platform: Unix
   :synopsis: A package for performing disaggregation on smart meter data.

.. moduleauthor:: Phil Ngo <ngo.phil@gmail.com>
.. moduleauthor:: Miguel Perez <miguel@invalid.com>
.. moduleauthor:: Stephen Suffian <steve@invalid.com>
.. moduleauthor:: Sabina Tomkins <sabina@invalid.com>
"""

# TODO remove these
from appliance import *
from utils import *

import evaluation_metrics
import PecanStreetDatasetAdapter
from TracebaseDatasetAdapter import TracebaseDatasetAdapter
import utils
import appliance
<|endoftext|>


<commit_before>import logging
l=logging.getLogger(__name__)

from django.conf import settings
from django.core.management.base import BaseCommand, CommandError

from experiments.reports import (EngagementReportGenerator,
                                 ConversionReportGenerator)

class Command(BaseCommand):
    help = ('update_experiment_reports : Generate all the daily reports for'
            ' for the SplitTesting experiments')

    def __init__(self):
        super(self.__class__, self).__init__()

    def handle(self, *args, **options):
        if len(args):
            raise CommandError("This command does not take any arguments")

        engagement_calculator = _load_function(settings.LEAN_ENGAGEMENT_CALCULATOR)

        EngagementReportGenerator(engagement_score_calculator=engagement_calculator).generate_all_daily_reports()
        ConversionReportGenerator().generate_all_daily_reports()

def _load_function(fully_qualified_name):
    i = fully_qualified_name.rfind('.')
    module, attr = fully_qualified_name[:i], fully_qualified_name[i+1:]
    try:
        mod = __import__(module, globals(), locals(), [attr])
    except ImportError, e:
        raise ImproperlyConfigured, 'Error importing template source loader %s: "%s"' % (module, e)
    try:
        func = getattr(mod, attr)
    except AttributeError:
        raise ImproperlyConfigured, 'Module "%s" does not define a "%s" callable template source loader' % (module, attr)
    return func
<commit_msg>Fix typos in exception messages and use less specific exception in management commands.<commit_after>import logging
l=logging.getLogger(__name__)

from django.conf import settings
from django.core.management.base import BaseCommand, CommandError

from experiments.reports import (EngagementReportGenerator,
                                 ConversionReportGenerator)

class Command(BaseCommand):
    help = ('update_experiment_reports : Generate all the daily reports for'
            ' for the SplitTesting experiments')

    def __init__(self):
        super(self.__class__, self).__init__()

    def handle(self, *args, **options):
        if len(args):
            raise CommandError("This command does not take any arguments")

        engagement_calculator = _load_function(settings.LEAN_ENGAGEMENT_CALCULATOR)

        EngagementReportGenerator(engagement_score_calculator=engagement_calculator).generate_all_daily_reports()
        ConversionReportGenerator().generate_all_daily_reports()

def _load_function(fully_qualified_name):
    i = fully_qualified_name.rfind('.')
    module, attr = fully_qualified_name[:i], fully_qualified_name[i+1:]
    try:
        mod = __import__(module, globals(), locals(), [attr])
    except ImportError, e:
        raise Exception, 'Error importing engagement function %s: "%s"' % (module, e)
    try:
        func = getattr(mod, attr)
    except AttributeError:
        raise Exception, 'Module "%s does not define a "%s" attribute' % (module, attr)
    return func
<|endoftext|>


<commit_before># Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Various classes representing distributed values for PS."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import copy
import weakref

from tensorflow.python.distribute import distribute_lib
from tensorflow.python.distribute import distribute_utils
from tensorflow.python.distribute import distribution_strategy_context as ds_context
from tensorflow.python.distribute import values
from tensorflow.python.distribute import values_util
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import variables as variables_lib
from tensorflow.python.training.tracking import base as trackable
from tensorflow.python.types import core

# Variable used in PSStrategy TF 1, TF2 and CentralStorageStrategy.
class AggregatingVariable(variables_lib.Variable, core.Tensor):
  """A wrapper around a variable that aggregates updates across replicas."""

  def __init__(self, strategy, v, aggregation):
    self._distribute_strategy = strategy
    self._v = v
    # NOTE: We don't use "_distributed_container" here because we don't want
    # to trigger that code path in regroup().
    v._aggregating_container = weakref.ref(self)  # pylint: disable=protected-access
    self._aggregation = aggregation

  def __deepcopy__(self, memo):
    """Perform a deepcopy of the `AggregatingVariable`.

    Unlike the deepcopy of a regular tf.Variable, this keeps the original
    strategy and devices of the `AggregatingVariable`.  To avoid confusion
    with the behavior of deepcopy on a regular `Variable` (which does
    copy into new devices), we only allow a deepcopy of a `AggregatingVariable`
    within its originating strategy scope.

    Args:
      memo: The memoization object for `deepcopy`.

    Returns:
      A deep copy of the current `AggregatingVariable`.

    Raises:
      RuntimeError: If trying to deepcopy into a different strategy.
    """
    with ds_context.enter_or_assert_strategy(self._distribute_strategy):
      v = copy.deepcopy(self._v, memo)

    copied_variable = type(self)(
        strategy=self._distribute_strategy,
        v=v,
        aggregation=self._aggregation)

    memo[id(self)] = copied_variable

    return copied_variable

  def get(self):
    return self._v

  @property
  def distribute_strategy(self):
    return self._distribute_strategy

  def __getattr__(self, name):
    return getattr(self._v, name)

  def _assign_func(self, *args, **kwargs):
    with ds_context.enter_or_assert_strategy(self._distribute_strategy):
      f = kwargs.pop("f")
      if ds_context.in_cross_replica_context():
        if distribute_lib.get_update_replica_id() is not None:
          # We are calling an assign function in an update context.
          return f(self._v, *args, **kwargs)

        # We are calling an assign function in cross replica context, wrap it in
        # an update call.
        return self._distribute_strategy.extended.update(
            self, f, args=args, kwargs=kwargs)
      else:
        replica_context = ds_context.get_replica_context()
        assert replica_context
        # We are calling an assign function in replica context.
        # We reduce the value we want to assign/add/sub. More details about how
        # we handle the different use cases can be found in the _reduce method.
        # We call the function with the reduced value.
        if self._aggregation == vs.VariableAggregation.NONE:
          raise ValueError(
              values_util.aggregation_error_msg.format(
                  variable_type="AggregatingVariable"))

        def merge_fn(strategy,
                     value,
                     use_locking=False,
                     name=None,
                     read_value=True):
          v = values_util.apply_aggregation(strategy, value, self._aggregation,
                                            self)
          if name and isinstance(name, values.PerReplica):
            name = name.values[0]
          return strategy.extended.update(
              self,
              f,
              args=(v,),
              kwargs={
                  "use_locking": use_locking,
                  "name": name,
                  "read_value": read_value
              })
        return replica_context.merge_call(merge_fn, args=args, kwargs=kwargs)

  def assign_sub(self, *args, **kwargs):
    assign_sub_fn = lambda var, *a, **kw: var.assign_sub(*a, **kw)
    return self._assign_func(f=assign_sub_fn, *args, **kwargs)

  def assign_add(self, *args, **kwargs):
    assign_add_fn = lambda var, *a, **kw: var.assign_add(*a, **kw)
    return self._assign_func(f=assign_add_fn, *args, **kwargs)

  def assign(self, *args, **kwargs):
    assign_fn = lambda var, *a, **kw: var.assign(*a, **kw)
    return self._assign_func(f=assign_fn, *args, **kwargs)

  @property
  def initializer(self):
    return self._v.initializer

  def initialized_value(self):
    return self._v.initialized_value()

  @property
  def initial_value(self):
    return self._v.initial_value

  @property
  def op(self):
    return self._v.op

  def value(self):
    return self._v.value()

  def read_value(self):
    return self._v.read_value()

  def eval(self, session=None):
    return self._v.eval(session)

  @property
  def graph(self):
    return self._v.graph

  @property
  def device(self):
    return self._v.device

  @property
  def shape(self):
    return self._v.shape

  @property
  def aggregation(self):
    return self._aggregation

  @property
  def synchronization(self):
    return self._v.synchronization

  @property
  def name(self):
    return self._v.name

  @property
  def trainable(self):
    return self._v.trainable

  @property
  def dtype(self):
    return self._v.dtype

  # TODO(josh11b): Test saving & restoring.
  def _gather_saveables_for_checkpoint(self):
    if isinstance(self._v, CachingVariable):
      return self._v._gather_saveables_for_checkpoint()  # pylint:disable=protected-access
    return {trackable.VARIABLE_VALUE_KEY: self._v}

  def _map_resources(self, save_options):
    """For implementing `Trackable`."""
    # By delegating this method to the wrapped variable, SavedModel with
    # AggregatingVariable are identical to SavedModel with normal variables.
    obj_map, resource_map = self._v._map_resources(save_options)  # pylint:disable=protected-access
    obj_map[self] = obj_map[self._v]
    return obj_map, resource_map

  # pylint: disable=multiple-statements
  def __add__(self, o):
    return self._v + o

  def __radd__(self, o):
    return o + self._v

  def __sub__(self, o):
    return self._v - o

  def __rsub__(self, o):
    return o - self._v

  def __mul__(self, o):
    return self._v * o

  def __rmul__(self, o):
    return o * self._v

  def __truediv__(self, o):
    return self._v / o

  def __rtruediv__(self, o):
    return o / self._v

  def __floordiv__(self, o):
    return self._v // o

  def __rfloordiv__(self, o):
    return o // self._v

  def __mod__(self, o):
    return self._v % o

  def __rmod__(self, o):
    return o % self._v

  def __lt__(self, o):
    return self._v < o

  def __le__(self, o):
    return self._v <= o

  def __gt__(self, o):
    return self._v > o

  def __ge__(self, o):
    return self._v >= o

  def __and__(self, o):
    return self._v & o

  def __rand__(self, o):
    return o & self._v

  def __or__(self, o):
    return self._v | o

  def __ror__(self, o):
    return o | self._v

  def __xor__(self, o):
    return self._v ^ o

  def __rxor__(self, o):
    return o ^ self._v

  def __getitem__(self, o):
    return self._v[o]

  def __pow__(self, o, modulo=None):
    return pow(self._v, o, modulo)

  def __rpow__(self, o):
    return pow(o, self._v)

  def __invert__(self):
    return ~self._v

  def __neg__(self):
    return -self._v

  def __abs__(self):
    return abs(self._v)

  def __div__(self, o):
    try:
      return self._v.__div__(o)
    except AttributeError:
      # See https://docs.python.org/3/library/constants.html#NotImplemented
      return NotImplemented

  def __rdiv__(self, o):
    try:
      return self._v.__rdiv__(o)
    except AttributeError:
      # See https://docs.python.org/3/library/constants.html#NotImplemented
      return NotImplemented

  def __matmul__(self, o):
    try:
      return self._v.__matmul__(o)
    except AttributeError:
      # See https://docs.python.org/3/library/constants.html#NotImplemented
      return NotImplemented

  def __rmatmul__(self, o):
    try:
      return self._v.__rmatmul__(o)
    except AttributeError:
      # See https://docs.python.org/3/library/constants.html#NotImplemented
      return NotImplemented

  def __str__(self):
    return str(self._v)

  def __repr__(self):
    return repr(self._v)

  def _should_act_as_resource_variable(self):
    """Pass resource_variable_ops.is_resource_variable check."""
    pass

  def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False):
    return ops.convert_to_tensor(self.get(), dtype=dtype, name=name,
                                 as_ref=as_ref)

class CachingVariable(variables_lib.Variable, core.Tensor):
  """A wrapper around a variable that caches read value locally."""

  def __init__(self, v):
    self._v = v
    self._cache = None
    self._current_new_cache_scope_count = 0

  def get(self):
    return self._v

  def __getattr__(self, name):
    return getattr(self._v, name)

  def read_value(self):
    if distribute_utils.caching_scope_local.in_caching_scope():
      return self.cached_read_value()
    return self._v.read_value()

  def cached_read_value(self):
    if (distribute_utils.caching_scope_local.new_cache_scope_count >
        self._current_new_cache_scope_count):
      self._current_new_cache_scope_count += 1
      self._cache = None

    with ops.device("CPU:0"):
      if self._cache is not None:
        return self._cache
      else:
        self._cache = array_ops.identity(self._v)
        return self._cache

  def assign_sub(self, *args, **kwargs):
    return self._v.assign_sub(*args, **kwargs)

  def assign_add(self, *args, **kwargs):
    return self._v.assign_add(*args, **kwargs)

  def assign(self, *args, **kwargs):
    return self._v.assign(*args, **kwargs)

  @property
  def initializer(self):
    return self._v.initializer

  def initialized_value(self):
    return self._v.initialized_value()

  @property
  def initial_value(self):
    return self._v.initial_value

  @property
  def op(self):
    return self._v.op

  def value(self):
    if distribute_utils.caching_scope_local.in_caching_scope():
      return self.cached_read_value()
    return self._v.value()

  def eval(self, session=None):
    return self._v.eval(session)

  @property
  def graph(self):
    return self._v.graph

  @property
  def device(self):
    return self._v.device

  @property
  def shape(self):
    return self._v.shape

  @property
  def synchronization(self):
    return self._v.synchronization

  @property
  def name(self):
    return self._v.name

  @property
  def trainable(self):
    return self._v.trainable

  @property
  def dtype(self):
    return self._v.dtype

  @property
  def constraint(self):
    return self._v.constraint

  def __str__(self):
    return str(self._v)

  def __repr__(self):
    return repr(self._v)

  def _should_act_as_resource_variable(self):
    """Pass resource_variable_ops.is_resource_variable check."""
    pass

  def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False):
    if distribute_utils.caching_scope_local.in_caching_scope():
      return self.cached_read_value()
    return ops.convert_to_tensor(
        self.get(), dtype=dtype, name=name, as_ref=as_ref)

  @classmethod
  def _overload_overloadable_operators(cls):
    """Register overloads for all operators."""
    for operator in ops.Tensor.OVERLOADABLE_OPERATORS:
      # Overloading __eq__ or __ne__ does not work as expected.
      if operator == "__eq__" or operator == "__ne__":
        continue
      cls._tensor_overload_operator(operator)

  @classmethod
  def _tensor_overload_operator(cls, operator):
    """Delegate an operator overload to `ops.Tensor`."""
    tensor_operator = getattr(ops.Tensor, operator)

    def _operator(v, *args, **kwargs):
      return tensor_operator(v.value(), *args, **kwargs)  # pylint: disable=protected-access
    setattr(cls, operator, _operator)

  def _gather_saveables_for_checkpoint(self):
    return {trackable.VARIABLE_VALUE_KEY: self._v}

  def _map_resources(self, save_options):
    """For implementing `Trackable`."""
    # By delegating this method to the wrapped variable, SavedModel with
    # AggregatingVariable are identical to SavedModel with normal variables.
    obj_map, resource_map = self._v._map_resources(save_options)  # pylint:disable=protected-access
    obj_map[self] = obj_map[self._v]
    return obj_map, resource_map

# Register a conversion function which reads the value of the variable,
# allowing instances of the class to be used as tensors.
def _tensor_conversion_aggregate(var, dtype=None, name=None, as_ref=False):
  return var._dense_var_to_tensor(dtype, name, as_ref)  # pylint: disable=protected-access

ops.register_tensor_conversion_function(AggregatingVariable,
                                        _tensor_conversion_aggregate)

# Register a conversion function which reads the value of the variable,
# allowing instances of the class to be used as tensors.
def _tensor_conversion_caching(var, dtype=None, name=None, as_ref=False):
  return var._dense_var_to_tensor(dtype, name, as_ref)  # pylint: disable=protected-access

ops.register_tensor_conversion_function(CachingVariable,
                                        _tensor_conversion_caching)

CachingVariable._overload_overloadable_operators()  # pylint: disable=protected-access

<commit_msg>Add int and float overrides for caching variable to support logging the variables with %d, %f. Also add numpy and array overrides for the CachingVariable.<commit_after># Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Various classes representing distributed values for PS."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import copy
import weakref

import numpy as np

from tensorflow.python.distribute import distribute_lib
from tensorflow.python.distribute import distribute_utils
from tensorflow.python.distribute import distribution_strategy_context as ds_context
from tensorflow.python.distribute import values
from tensorflow.python.distribute import values_util
from tensorflow.python.eager import context
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import variables as variables_lib
from tensorflow.python.training.tracking import base as trackable
from tensorflow.python.types import core

# Variable used in PSStrategy TF 1, TF2 and CentralStorageStrategy.
class AggregatingVariable(variables_lib.Variable, core.Tensor):
  """A wrapper around a variable that aggregates updates across replicas."""

  def __init__(self, strategy, v, aggregation):
    self._distribute_strategy = strategy
    self._v = v
    # NOTE: We don't use "_distributed_container" here because we don't want
    # to trigger that code path in regroup().
    v._aggregating_container = weakref.ref(self)  # pylint: disable=protected-access
    self._aggregation = aggregation

  def __deepcopy__(self, memo):
    """Perform a deepcopy of the `AggregatingVariable`.

    Unlike the deepcopy of a regular tf.Variable, this keeps the original
    strategy and devices of the `AggregatingVariable`.  To avoid confusion
    with the behavior of deepcopy on a regular `Variable` (which does
    copy into new devices), we only allow a deepcopy of a `AggregatingVariable`
    within its originating strategy scope.

    Args:
      memo: The memoization object for `deepcopy`.

    Returns:
      A deep copy of the current `AggregatingVariable`.

    Raises:
      RuntimeError: If trying to deepcopy into a different strategy.
    """
    with ds_context.enter_or_assert_strategy(self._distribute_strategy):
      v = copy.deepcopy(self._v, memo)

    copied_variable = type(self)(
        strategy=self._distribute_strategy,
        v=v,
        aggregation=self._aggregation)

    memo[id(self)] = copied_variable

    return copied_variable

  def get(self):
    return self._v

  @property
  def distribute_strategy(self):
    return self._distribute_strategy

  def __getattr__(self, name):
    return getattr(self._v, name)

  def _assign_func(self, *args, **kwargs):
    with ds_context.enter_or_assert_strategy(self._distribute_strategy):
      f = kwargs.pop("f")
      if ds_context.in_cross_replica_context():
        if distribute_lib.get_update_replica_id() is not None:
          # We are calling an assign function in an update context.
          return f(self._v, *args, **kwargs)

        # We are calling an assign function in cross replica context, wrap it in
        # an update call.
        return self._distribute_strategy.extended.update(
            self, f, args=args, kwargs=kwargs)
      else:
        replica_context = ds_context.get_replica_context()
        assert replica_context
        # We are calling an assign function in replica context.
        # We reduce the value we want to assign/add/sub. More details about how
        # we handle the different use cases can be found in the _reduce method.
        # We call the function with the reduced value.
        if self._aggregation == vs.VariableAggregation.NONE:
          raise ValueError(
              values_util.aggregation_error_msg.format(
                  variable_type="AggregatingVariable"))

        def merge_fn(strategy,
                     value,
                     use_locking=False,
                     name=None,
                     read_value=True):
          v = values_util.apply_aggregation(strategy, value, self._aggregation,
                                            self)
          if name and isinstance(name, values.PerReplica):
            name = name.values[0]
          return strategy.extended.update(
              self,
              f,
              args=(v,),
              kwargs={
                  "use_locking": use_locking,
                  "name": name,
                  "read_value": read_value
              })
        return replica_context.merge_call(merge_fn, args=args, kwargs=kwargs)

  def assign_sub(self, *args, **kwargs):
    assign_sub_fn = lambda var, *a, **kw: var.assign_sub(*a, **kw)
    return self._assign_func(f=assign_sub_fn, *args, **kwargs)

  def assign_add(self, *args, **kwargs):
    assign_add_fn = lambda var, *a, **kw: var.assign_add(*a, **kw)
    return self._assign_func(f=assign_add_fn, *args, **kwargs)

  def assign(self, *args, **kwargs):
    assign_fn = lambda var, *a, **kw: var.assign(*a, **kw)
    return self._assign_func(f=assign_fn, *args, **kwargs)

  @property
  def initializer(self):
    return self._v.initializer

  def initialized_value(self):
    return self._v.initialized_value()

  @property
  def initial_value(self):
    return self._v.initial_value

  @property
  def op(self):
    return self._v.op

  def value(self):
    return self._v.value()

  def read_value(self):
    return self._v.read_value()

  def eval(self, session=None):
    return self._v.eval(session)

  @property
  def graph(self):
    return self._v.graph

  @property
  def device(self):
    return self._v.device

  @property
  def shape(self):
    return self._v.shape

  @property
  def aggregation(self):
    return self._aggregation

  @property
  def synchronization(self):
    return self._v.synchronization

  @property
  def name(self):
    return self._v.name

  @property
  def trainable(self):
    return self._v.trainable

  @property
  def dtype(self):
    return self._v.dtype

  # TODO(josh11b): Test saving & restoring.
  def _gather_saveables_for_checkpoint(self):
    if isinstance(self._v, CachingVariable):
      return self._v._gather_saveables_for_checkpoint()  # pylint:disable=protected-access
    return {trackable.VARIABLE_VALUE_KEY: self._v}

  def _map_resources(self, save_options):
    """For implementing `Trackable`."""
    # By delegating this method to the wrapped variable, SavedModel with
    # AggregatingVariable are identical to SavedModel with normal variables.
    obj_map, resource_map = self._v._map_resources(save_options)  # pylint:disable=protected-access
    obj_map[self] = obj_map[self._v]
    return obj_map, resource_map

  # pylint: disable=multiple-statements
  def __add__(self, o):
    return self._v + o

  def __radd__(self, o):
    return o + self._v

  def __sub__(self, o):
    return self._v - o

  def __rsub__(self, o):
    return o - self._v

  def __mul__(self, o):
    return self._v * o

  def __rmul__(self, o):
    return o * self._v

  def __truediv__(self, o):
    return self._v / o

  def __rtruediv__(self, o):
    return o / self._v

  def __floordiv__(self, o):
    return self._v // o

  def __rfloordiv__(self, o):
    return o // self._v

  def __mod__(self, o):
    return self._v % o

  def __rmod__(self, o):
    return o % self._v

  def __lt__(self, o):
    return self._v < o

  def __le__(self, o):
    return self._v <= o

  def __gt__(self, o):
    return self._v > o

  def __ge__(self, o):
    return self._v >= o

  def __and__(self, o):
    return self._v & o

  def __rand__(self, o):
    return o & self._v

  def __or__(self, o):
    return self._v | o

  def __ror__(self, o):
    return o | self._v

  def __xor__(self, o):
    return self._v ^ o

  def __rxor__(self, o):
    return o ^ self._v

  def __getitem__(self, o):
    return self._v[o]

  def __pow__(self, o, modulo=None):
    return pow(self._v, o, modulo)

  def __rpow__(self, o):
    return pow(o, self._v)

  def __invert__(self):
    return ~self._v

  def __neg__(self):
    return -self._v

  def __abs__(self):
    return abs(self._v)

  def __div__(self, o):
    try:
      return self._v.__div__(o)
    except AttributeError:
      # See https://docs.python.org/3/library/constants.html#NotImplemented
      return NotImplemented

  def __rdiv__(self, o):
    try:
      return self._v.__rdiv__(o)
    except AttributeError:
      # See https://docs.python.org/3/library/constants.html#NotImplemented
      return NotImplemented

  def __matmul__(self, o):
    try:
      return self._v.__matmul__(o)
    except AttributeError:
      # See https://docs.python.org/3/library/constants.html#NotImplemented
      return NotImplemented

  def __rmatmul__(self, o):
    try:
      return self._v.__rmatmul__(o)
    except AttributeError:
      # See https://docs.python.org/3/library/constants.html#NotImplemented
      return NotImplemented

  def __str__(self):
    return str(self._v)

  def __repr__(self):
    return repr(self._v)

  def _should_act_as_resource_variable(self):
    """Pass resource_variable_ops.is_resource_variable check."""
    pass

  def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False):
    return ops.convert_to_tensor(self.get(), dtype=dtype, name=name,
                                 as_ref=as_ref)

class CachingVariable(variables_lib.Variable, core.Tensor):
  """A wrapper around a variable that caches read value locally."""

  def __init__(self, v):
    self._v = v
    self._cache = None
    self._current_new_cache_scope_count = 0

  def get(self):
    return self._v

  def __getattr__(self, name):
    return getattr(self._v, name)

  def read_value(self):
    if distribute_utils.caching_scope_local.in_caching_scope():
      return self.cached_read_value()
    return self._v.read_value()

  def cached_read_value(self):
    if (distribute_utils.caching_scope_local.new_cache_scope_count >
        self._current_new_cache_scope_count):
      self._current_new_cache_scope_count += 1
      self._cache = None

    with ops.device("CPU:0"):
      if self._cache is not None:
        return self._cache
      else:
        self._cache = array_ops.identity(self._v)
        return self._cache

  def assign_sub(self, *args, **kwargs):
    return self._v.assign_sub(*args, **kwargs)

  def assign_add(self, *args, **kwargs):
    return self._v.assign_add(*args, **kwargs)

  def assign(self, *args, **kwargs):
    return self._v.assign(*args, **kwargs)

  @property
  def initializer(self):
    return self._v.initializer

  def initialized_value(self):
    return self._v.initialized_value()

  @property
  def initial_value(self):
    return self._v.initial_value

  @property
  def op(self):
    return self._v.op

  def value(self):
    if distribute_utils.caching_scope_local.in_caching_scope():
      return self.cached_read_value()
    return self._v.value()

  def eval(self, session=None):
    return self._v.eval(session)

  @property
  def graph(self):
    return self._v.graph

  @property
  def device(self):
    return self._v.device

  @property
  def shape(self):
    return self._v.shape

  @property
  def synchronization(self):
    return self._v.synchronization

  @property
  def name(self):
    return self._v.name

  @property
  def trainable(self):
    return self._v.trainable

  @property
  def dtype(self):
    return self._v.dtype

  @property
  def constraint(self):
    return self._v.constraint

  def __array__(self):
    return np.asarray(self.numpy())

  def __complex__(self):
    return complex(self.value().numpy())

  def __int__(self):
    return int(self.value().numpy())

  def __float__(self):
    return float(self.value().numpy())

  def numpy(self):
    if context.executing_eagerly():
      return self.read_value().numpy()
    else:
      raise NotImplementedError(
          "numpy() is only available when eager execution is enabled.")

  def __str__(self):
    return str(self._v)

  def __repr__(self):
    return repr(self._v)

  def _should_act_as_resource_variable(self):
    """Pass resource_variable_ops.is_resource_variable check."""
    pass

  def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False):
    if distribute_utils.caching_scope_local.in_caching_scope():
      return self.cached_read_value()
    return ops.convert_to_tensor(
        self.get(), dtype=dtype, name=name, as_ref=as_ref)

  @classmethod
  def _overload_overloadable_operators(cls):
    """Register overloads for all operators."""
    for operator in ops.Tensor.OVERLOADABLE_OPERATORS:
      # Overloading __eq__ or __ne__ does not work as expected.
      if operator == "__eq__" or operator == "__ne__":
        continue
      cls._tensor_overload_operator(operator)

  @classmethod
  def _tensor_overload_operator(cls, operator):
    """Delegate an operator overload to `ops.Tensor`."""
    tensor_operator = getattr(ops.Tensor, operator)

    def _operator(v, *args, **kwargs):
      return tensor_operator(v.value(), *args, **kwargs)  # pylint: disable=protected-access
    setattr(cls, operator, _operator)

  def _gather_saveables_for_checkpoint(self):
    return {trackable.VARIABLE_VALUE_KEY: self._v}

  def _map_resources(self, save_options):
    """For implementing `Trackable`."""
    # By delegating this method to the wrapped variable, SavedModel with
    # AggregatingVariable are identical to SavedModel with normal variables.
    obj_map, resource_map = self._v._map_resources(save_options)  # pylint:disable=protected-access
    obj_map[self] = obj_map[self._v]
    return obj_map, resource_map

# Register a conversion function which reads the value of the variable,
# allowing instances of the class to be used as tensors.
def _tensor_conversion_aggregate(var, dtype=None, name=None, as_ref=False):
  return var._dense_var_to_tensor(dtype, name, as_ref)  # pylint: disable=protected-access

ops.register_tensor_conversion_function(AggregatingVariable,
                                        _tensor_conversion_aggregate)

# Register a conversion function which reads the value of the variable,
# allowing instances of the class to be used as tensors.
def _tensor_conversion_caching(var, dtype=None, name=None, as_ref=False):
  return var._dense_var_to_tensor(dtype, name, as_ref)  # pylint: disable=protected-access

ops.register_tensor_conversion_function(CachingVariable,
                                        _tensor_conversion_caching)

CachingVariable._overload_overloadable_operators()  # pylint: disable=protected-access

<|endoftext|>