<commit_before>"""
These tests show basic procedures for training, validating, and extracting features from
models.

Note about MongoDB:
The tests require a MongoDB instance to be available on the port defined by "testport" in
the code below.   This db can either be local to where you run these tests (and therefore
on 'localhost' by default) or it can be running somewhere else and then by ssh-tunneled on
the relevant port to the host where you run these tests.  [That is, before testing, you'd run
         ssh -f -N -L  [testport]:localhost:[testport] [username]@mongohost.xx.xx
on the machine where you're running these tests.   [mongohost] is the where the mongodb
instance is running.
"""
from __future__ import division, print_function, absolute_import
import cPickle

import pymongo as pm
import gridfs
import numpy as np
import tensorflow as tf

from tfutils import base, model, utils, data

num_batches_per_epoch = 10000//256

testhost = 'localhost'       # Host on which the MongoDB instance to be used by tests needs to be running
testport = 27017             # port on which the MongoDB instance to be used by tests needs to be running
testdbname = 'tfutils-test'  # name of the mongodb database where results will be stored by tests
testcol = 'testcol'          # name of the mongodb collection where results will be stored by tests

def test_training():
    """This test illustrates how basic training is performed using the
       tfutils.base.train_from_params function.  This is the first in a sequence of
       interconnected tests. It creates a pretrained model that is used by
       the next few tests (test_validation and test_feature_extraction).

       As can be seen by looking at how the test checks for correctness, after the
       training is run, results of training, including (intermittently) the full variables
       needed to re-initialize the tensorflow model, are stored in a MongoDB.

       Also see docstring of the tfutils.base.train_from_params function for more detailed
       information about usage.
    """
    # delete old database if it exists
    conn = pm.MongoClient(host=testhost,
                          port=testport)
    conn.drop_database(testdbname)
    nm = testdbname + '_' + testcol + '_training0'
    [conn.drop_database(x) for x in conn.database_names() if x.startswith(nm) and '___RECENT' in x]
    nm = testdbname + '_' + testcol + '_training1'
    [conn.drop_database(x) for x in conn.database_names() if x.startswith(nm) and '___RECENT' in x]

    # set up the parameters
    params = {}
    params['model_params'] = {'func': model.mnist_tfutils}
    params['save_params'] = {'host': testhost,
                             'port': testport,
                             'dbname': testdbname,
                             'collname': testcol,
                             'exp_id': 'training0',
                             'save_valid_freq': 20,
                             'save_filters_freq': 200,
                             'cache_filters_freq': 100}
    params['train_params'] = {'data_params': {'func': data.MNIST,
                                              'batch_size': 100,
                                              'group': 'train'},
                              'queue_params': {'queue_type': 'fifo',
                                               'batch_size': 100,
                                               'n_threads': 4},
                              'num_steps': 500}
    params['learning_rate_params'] = {'learning_rate': 0.05,
                                      'decay_steps': num_batches_per_epoch,
                                      'decay_rate': 0.95,
                                      'staircase': True}
    params['validation_params'] = {'valid0': {'data_params': {'func': data.MNIST,
                                                              'batch_size': 100,
                                                              'group': 'test'},
                                              'queue_params': {'queue_type': 'fifo',
                                                               'batch_size': 100,
                                                               'n_threads': 4},
                                              'num_steps': 10,
                                              'agg_func': utils.mean_dict}}

    # actually run the training
    base.train_from_params(**params)
    # test if results are as expected
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0'}).count() == 26
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0',
                                                    'saved_filters': True}).distinct('step') == [0, 200, 400]

    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0', 'step': 0})[0]
    asserts_for_record(r, params, train=True)
    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0', 'step': 20})[0]
    asserts_for_record(r, params, train=True)

    # run another 500 steps of training on the same experiment id.
    params['train_params']['num_steps'] = 1000
    base.train_from_params(**params)
    # test if results are as expected
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0'}).count() == 51
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0',
                                                    'saved_filters': True}).distinct('step') == [0, 200, 400, 600, 800, 1000]
    assert conn['tfutils-test']['testcol.files'].distinct('exp_id') == ['training0']
    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0', 'step': 1000})[0]
    asserts_for_record(r, params, train=True)

    # run 500 more steps but save to a new experiment id.
    params['train_params']['num_steps'] = 1500
    params['load_params'] = {'exp_id': 'training0'}
    params['save_params']['exp_id'] = 'training1'
    base.train_from_params(**params)
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training1',
                                                    'saved_filters': True}).distinct('step') == [1200, 1400]

def test_validation():
    """
    This is a test illustrating how to compute performance on a trained model on a new dataset,
    using the tfutils.base.test_from_params function.  This test assumes that test_training function
    has run first (to provide a pre-trained model to validate).

    After the test is run, results from the validation are stored in the MongoDB.
    (The test shows how the record can be loaded for inspection.)

    See the docstring of tfutils.base.test_from_params for more detailed information on usage.
    """
    # specify the parameters for the validation
    params = {}
    params['model_params'] = {'func': model.mnist_tfutils}
    params['load_params'] = {'host': testhost,
                             'port': testport,
                             'dbname': testdbname,
                             'collname': testcol,
                             'exp_id': 'training0'}
    params['save_params'] = {'exp_id': 'validation0'}
    params['validation_params'] = {'valid0': {'data_params': {'func': data.MNIST,
                                                              'batch_size': 100,
                                                              'group': 'test'},
                                              'queue_params': {'queue_type': 'fifo',
                                                               'batch_size': 100,
                                                               'n_threads': 4},
                                              'num_steps': 10,
                                              'agg_func': utils.mean_dict}}

    # actually run the model
    base.test_from_params(**params)

    # check that the results are correct
    conn = pm.MongoClient(host=testhost,
                          port=testport)

    # ... specifically, there is now a record containing the validation0 performance results
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'validation0'}).count() == 1
    # ... here's how to load the record:
    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'validation0'})[0]
    asserts_for_record(r, params, train=False)

    # ... check that the recorrectly ties to the id information for the
    # pre-trained model it was supposed to validate
    assert r['validates']
    idval = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0'})[50]['_id']
    v = conn[testdbname][testcol+'.files'].find({'exp_id': 'validation0'})[0]['validates']
    assert idval == v

def get_extraction_target(inputs, outputs, to_extract, **loss_params):
    """
    Example validation target function to use to provide targets for extracting features.
    This function also adds a standard "loss" target which you may or not may not want

    The to_extract argument must be a dictionary of the form
          {name_for_saving: name_of_actual_tensor, ...}
    where the "name_for_saving" is a human-friendly name you want to save extracted
    features under, and name_of_actual_tensor is a name of the tensor in the tensorflow
    graph outputing the features desired to be extracted.  To figure out what the names
    of the tensors you want to extract are "to_extract" argument,  uncomment the
    commented-out lines, which will print a list of all available tensor names.
    """

    names = [[x.name for x in op.values()] for op in tf.get_default_graph().get_operations()]
    print("NAMES are: ", names)

    targets = {k: tf.get_default_graph().get_tensor_by_name(v) for k, v in to_extract.items()}
    targets['loss'] = utils.get_loss(inputs, outputs, **loss_params)
    return targets

def test_feature_extraction():
    """
    This is a test illustrating how to perform feature extraction using
    tfutils.base.test_from_params.
    The basic idea is to specify a validation target that is simply the actual output of
    the model at some layer. (See the "get_extraction_target" function above as well.)
    This test assumes that test_train has run first.

    After the test is run, the results of the feature extraction are saved in the Grid
    File System associated with the mongo database, with one file per batch of feature
    results.  See how the features are accessed by reading the test code below.
    """
    # set up parameters
    exp_id = 'validation1'
    params = {}
    params['model_params'] = {'func': model.mnist_tfutils}
    params['load_params'] = {'host': testhost,
                             'port': testport,
                             'dbname': testdbname,
                             'collname': testcol,
                             'exp_id': 'training0'}
    params['save_params'] = {'exp_id': exp_id,
                             'save_intermediate_freq': 1,
                             'save_to_gfs': ['features', 'more_features']}

    targdict = {'func': get_extraction_target,
                'to_extract': {'features': 'validation/valid1/hidden1/fc:0', 'more_features' : 'validation/valid1/hidden2/fc:0'}}
    targdict.update(base.DEFAULT_LOSS_PARAMS)
    params['validation_params'] = {'valid1': {'data_params': {'func': data.MNIST,
                                                              'batch_size': 100,
                                                              'group': 'test'},
                                              'queue_params': {'queue_type': 'fifo',
                                                               'batch_size': 100,
                                                               'n_threads': 4},
                                              'targets': targdict,
                                              'num_steps': 10,
                                              'online_agg_func': utils.reduce_mean_dict}}

    # actually run the feature extraction
    base.test_from_params(**params)

    # check that things are as expected.
    conn = pm.MongoClient(host=testhost,
                          port=testport)
    coll = conn[testdbname][testcol+'.files']
    assert coll.find({'exp_id': exp_id}).count() == 11

    # ... load the containing the final "aggregate" result after all features have been extracted
    q = {'exp_id': exp_id, 'validation_results.valid1.intermediate_steps': {'$exists': True}}
    assert coll.find(q).count() == 1
    r = coll.find(q)[0]
    # ... check that the record is well-formed
    asserts_for_record(r, params, train=False)

    # ... check that the correct "intermediate results" (the actual features extracted) records exist
    # and are correctly referenced.
    q1 = {'exp_id': exp_id, 'validation_results.valid1.intermediate_steps': {'$exists': False}}
    ids = coll.find(q1).distinct('_id')
    assert r['validation_results']['valid1']['intermediate_steps'] == ids

    # ... actually load feature batch 3
    idval = r['validation_results']['valid1']['intermediate_steps'][3]
    fn = coll.find({'item_for': idval})[0]['filename']
    fs = gridfs.GridFS(coll.database, testcol)
    fh = fs.get_last_version(fn)
    saved_data = cPickle.loads(fh.read())
    fh.close()
    first_results = saved_data['validation_results']['valid1']
    assert 'features' in first_results and 'more_features' in first_results
    features = saved_data['validation_results']['valid1']['features']
    more_features = saved_data['validation_results']['valid1']['more_features']
    assert features.shape == (100, 128)
    assert features.dtype == np.float32
    assert more_features.shape == (100, 32)
    assert more_features.dtype == np.float32

def asserts_for_record(r, params, train=False):
    if r.get('saved_filters'):
        assert r['_saver_write_version'] == 2
        assert r['_saver_num_data_files'] == 1
    assert type(r['duration']) == float

    should_contain = ['save_params', 'load_params', 'model_params', 'validation_params']
    assert set(should_contain).difference(r['params'].keys()) == set()

    vk = r['params']['validation_params'].keys()
    vk1 = r['validation_results'].keys()
    assert set(vk) == set(vk1)

    assert r['params']['model_params']['seed'] == 0
    assert r['params']['model_params']['func']['modname'] == 'tfutils.model'
    assert r['params']['model_params']['func']['objname'] == 'mnist_tfutils'
    assert set(['hidden1', 'hidden2', u'softmax_linear']).difference(r['params']['model_params']['cfg_final'].keys()) == set()

    _k = vk[0]
    should_contain = ['agg_func', 'data_params', 'num_steps', 'online_agg_func', 'queue_params', 'targets']
    assert set(should_contain).difference(r['params']['validation_params'][_k].keys()) == set()

    if train:
        assert r['params']['model_params']['train'] is True
        for k in ['num_steps', 'queue_params']:
            assert r['params']['train_params'][k] == params['train_params'][k]

        should_contain = ['loss_params', 'optimizer_params', 'train_params', 'learning_rate_params']
        assert set(should_contain).difference(r['params'].keys()) == set()
        assert r['params']['train_params']['thres_loss'] == 100
        assert r['params']['train_params']['data_params']['func']['modname'] == 'tfutils.data'
        assert r['params']['train_params']['data_params']['func']['objname'] == 'MNIST'

        assert r['params']['loss_params']['agg_func']['modname'] == 'tensorflow.python.ops.math_ops'
        assert r['params']['loss_params']['agg_func']['objname'] == 'reduce_mean'
        assert r['params']['loss_params']['loss_per_case_func']['modname'] == 'tensorflow.python.ops.nn_ops'
        assert r['params']['loss_params']['loss_per_case_func']['objname'] == 'sparse_softmax_cross_entropy_with_logits'
        assert r['params']['loss_params']['targets'] == ['labels']
    else:
        assert 'train' not in r['params']['model_params']
        assert 'train_params' not in r['params']
<commit_msg>Revert port number<commit_after>"""
These tests show basic procedures for training, validating, and extracting features from
models.

Note about MongoDB:
The tests require a MongoDB instance to be available on the port defined by "testport" in
the code below.   This db can either be local to where you run these tests (and therefore
on 'localhost' by default) or it can be running somewhere else and then by ssh-tunneled on
the relevant port to the host where you run these tests.  [That is, before testing, you'd run
         ssh -f -N -L  [testport]:localhost:[testport] [username]@mongohost.xx.xx
on the machine where you're running these tests.   [mongohost] is the where the mongodb
instance is running.
"""
from __future__ import division, print_function, absolute_import
import cPickle

import pymongo as pm
import gridfs
import numpy as np
import tensorflow as tf

from tfutils import base, model, utils, data

num_batches_per_epoch = 10000//256

testhost = 'localhost'       # Host on which the MongoDB instance to be used by tests needs to be running
testport = 31001             # port on which the MongoDB instance to be used by tests needs to be running
testdbname = 'tfutils-test'  # name of the mongodb database where results will be stored by tests
testcol = 'testcol'          # name of the mongodb collection where results will be stored by tests

def test_training():
    """This test illustrates how basic training is performed using the
       tfutils.base.train_from_params function.  This is the first in a sequence of
       interconnected tests. It creates a pretrained model that is used by
       the next few tests (test_validation and test_feature_extraction).

       As can be seen by looking at how the test checks for correctness, after the
       training is run, results of training, including (intermittently) the full variables
       needed to re-initialize the tensorflow model, are stored in a MongoDB.

       Also see docstring of the tfutils.base.train_from_params function for more detailed
       information about usage.
    """
    # delete old database if it exists
    conn = pm.MongoClient(host=testhost,
                          port=testport)
    conn.drop_database(testdbname)
    nm = testdbname + '_' + testcol + '_training0'
    [conn.drop_database(x) for x in conn.database_names() if x.startswith(nm) and '___RECENT' in x]
    nm = testdbname + '_' + testcol + '_training1'
    [conn.drop_database(x) for x in conn.database_names() if x.startswith(nm) and '___RECENT' in x]

    # set up the parameters
    params = {}
    params['model_params'] = {'func': model.mnist_tfutils}
    params['save_params'] = {'host': testhost,
                             'port': testport,
                             'dbname': testdbname,
                             'collname': testcol,
                             'exp_id': 'training0',
                             'save_valid_freq': 20,
                             'save_filters_freq': 200,
                             'cache_filters_freq': 100}
    params['train_params'] = {'data_params': {'func': data.MNIST,
                                              'batch_size': 100,
                                              'group': 'train'},
                              'queue_params': {'queue_type': 'fifo',
                                               'batch_size': 100,
                                               'n_threads': 4},
                              'num_steps': 500}
    params['learning_rate_params'] = {'learning_rate': 0.05,
                                      'decay_steps': num_batches_per_epoch,
                                      'decay_rate': 0.95,
                                      'staircase': True}
    params['validation_params'] = {'valid0': {'data_params': {'func': data.MNIST,
                                                              'batch_size': 100,
                                                              'group': 'test'},
                                              'queue_params': {'queue_type': 'fifo',
                                                               'batch_size': 100,
                                                               'n_threads': 4},
                                              'num_steps': 10,
                                              'agg_func': utils.mean_dict}}

    # actually run the training
    base.train_from_params(**params)
    # test if results are as expected
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0'}).count() == 26
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0',
                                                    'saved_filters': True}).distinct('step') == [0, 200, 400]

    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0', 'step': 0})[0]
    asserts_for_record(r, params, train=True)
    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0', 'step': 20})[0]
    asserts_for_record(r, params, train=True)

    # run another 500 steps of training on the same experiment id.
    params['train_params']['num_steps'] = 1000
    base.train_from_params(**params)
    # test if results are as expected
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0'}).count() == 51
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training0',
                                                    'saved_filters': True}).distinct('step') == [0, 200, 400, 600, 800, 1000]
    assert conn['tfutils-test']['testcol.files'].distinct('exp_id') == ['training0']
    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0', 'step': 1000})[0]
    asserts_for_record(r, params, train=True)

    # run 500 more steps but save to a new experiment id.
    params['train_params']['num_steps'] = 1500
    params['load_params'] = {'exp_id': 'training0'}
    params['save_params']['exp_id'] = 'training1'
    base.train_from_params(**params)
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'training1',
                                                    'saved_filters': True}).distinct('step') == [1200, 1400]

def test_validation():
    """
    This is a test illustrating how to compute performance on a trained model on a new dataset,
    using the tfutils.base.test_from_params function.  This test assumes that test_training function
    has run first (to provide a pre-trained model to validate).

    After the test is run, results from the validation are stored in the MongoDB.
    (The test shows how the record can be loaded for inspection.)

    See the docstring of tfutils.base.test_from_params for more detailed information on usage.
    """
    # specify the parameters for the validation
    params = {}
    params['model_params'] = {'func': model.mnist_tfutils}
    params['load_params'] = {'host': testhost,
                             'port': testport,
                             'dbname': testdbname,
                             'collname': testcol,
                             'exp_id': 'training0'}
    params['save_params'] = {'exp_id': 'validation0'}
    params['validation_params'] = {'valid0': {'data_params': {'func': data.MNIST,
                                                              'batch_size': 100,
                                                              'group': 'test'},
                                              'queue_params': {'queue_type': 'fifo',
                                                               'batch_size': 100,
                                                               'n_threads': 4},
                                              'num_steps': 10,
                                              'agg_func': utils.mean_dict}}

    # actually run the model
    base.test_from_params(**params)

    # check that the results are correct
    conn = pm.MongoClient(host=testhost,
                          port=testport)

    # ... specifically, there is now a record containing the validation0 performance results
    assert conn[testdbname][testcol+'.files'].find({'exp_id': 'validation0'}).count() == 1
    # ... here's how to load the record:
    r = conn[testdbname][testcol+'.files'].find({'exp_id': 'validation0'})[0]
    asserts_for_record(r, params, train=False)

    # ... check that the recorrectly ties to the id information for the
    # pre-trained model it was supposed to validate
    assert r['validates']
    idval = conn[testdbname][testcol+'.files'].find({'exp_id': 'training0'})[50]['_id']
    v = conn[testdbname][testcol+'.files'].find({'exp_id': 'validation0'})[0]['validates']
    assert idval == v

def get_extraction_target(inputs, outputs, to_extract, **loss_params):
    """
    Example validation target function to use to provide targets for extracting features.
    This function also adds a standard "loss" target which you may or not may not want

    The to_extract argument must be a dictionary of the form
          {name_for_saving: name_of_actual_tensor, ...}
    where the "name_for_saving" is a human-friendly name you want to save extracted
    features under, and name_of_actual_tensor is a name of the tensor in the tensorflow
    graph outputing the features desired to be extracted.  To figure out what the names
    of the tensors you want to extract are "to_extract" argument,  uncomment the
    commented-out lines, which will print a list of all available tensor names.
    """

    # names = [[x.name for x in op.values()] for op in tf.get_default_graph().get_operations()]
    # print("NAMES are: ", names)

    targets = {k: tf.get_default_graph().get_tensor_by_name(v) for k, v in to_extract.items()}
    targets['loss'] = utils.get_loss(inputs, outputs, **loss_params)
    return targets

def test_feature_extraction():
    """
    This is a test illustrating how to perform feature extraction using
    tfutils.base.test_from_params.
    The basic idea is to specify a validation target that is simply the actual output of
    the model at some layer. (See the "get_extraction_target" function above as well.)
    This test assumes that test_train has run first.

    After the test is run, the results of the feature extraction are saved in the Grid
    File System associated with the mongo database, with one file per batch of feature
    results.  See how the features are accessed by reading the test code below.
    """
    # set up parameters
    exp_id = 'validation1'
    params = {}
    params['model_params'] = {'func': model.mnist_tfutils}
    params['load_params'] = {'host': testhost,
                             'port': testport,
                             'dbname': testdbname,
                             'collname': testcol,
                             'exp_id': 'training0'}
    params['save_params'] = {'exp_id': exp_id,
                             'save_intermediate_freq': 1,
                             'save_to_gfs': ['features', 'more_features']}

    targdict = {'func': get_extraction_target,
                'to_extract': {'features': 'validation/valid1/hidden1/fc:0', 'more_features' : 'validation/valid1/hidden2/fc:0'}}
    targdict.update(base.DEFAULT_LOSS_PARAMS)
    params['validation_params'] = {'valid1': {'data_params': {'func': data.MNIST,
                                                              'batch_size': 100,
                                                              'group': 'test'},
                                              'queue_params': {'queue_type': 'fifo',
                                                               'batch_size': 100,
                                                               'n_threads': 4},
                                              'targets': targdict,
                                              'num_steps': 10,
                                              'online_agg_func': utils.reduce_mean_dict}}

    # actually run the feature extraction
    base.test_from_params(**params)

    # check that things are as expected.
    conn = pm.MongoClient(host=testhost,
                          port=testport)
    coll = conn[testdbname][testcol+'.files']
    assert coll.find({'exp_id': exp_id}).count() == 11

    # ... load the containing the final "aggregate" result after all features have been extracted
    q = {'exp_id': exp_id, 'validation_results.valid1.intermediate_steps': {'$exists': True}}
    assert coll.find(q).count() == 1
    r = coll.find(q)[0]
    # ... check that the record is well-formed
    asserts_for_record(r, params, train=False)

    # ... check that the correct "intermediate results" (the actual features extracted) records exist
    # and are correctly referenced.
    q1 = {'exp_id': exp_id, 'validation_results.valid1.intermediate_steps': {'$exists': False}}
    ids = coll.find(q1).distinct('_id')
    assert r['validation_results']['valid1']['intermediate_steps'] == ids

    # ... actually load feature batch 3
    idval = r['validation_results']['valid1']['intermediate_steps'][3]
    fn = coll.find({'item_for': idval})[0]['filename']
    fs = gridfs.GridFS(coll.database, testcol)
    fh = fs.get_last_version(fn)
    saved_data = cPickle.loads(fh.read())
    fh.close()
    first_results = saved_data['validation_results']['valid1']
    assert 'features' in first_results and 'more_features' in first_results
    features = saved_data['validation_results']['valid1']['features']
    more_features = saved_data['validation_results']['valid1']['more_features']
    assert features.shape == (100, 128)
    assert features.dtype == np.float32
    assert more_features.shape == (100, 32)
    assert more_features.dtype == np.float32

def asserts_for_record(r, params, train=False):
    if r.get('saved_filters'):
        assert r['_saver_write_version'] == 2
        assert r['_saver_num_data_files'] == 1
    assert type(r['duration']) == float

    should_contain = ['save_params', 'load_params', 'model_params', 'validation_params']
    assert set(should_contain).difference(r['params'].keys()) == set()

    vk = r['params']['validation_params'].keys()
    vk1 = r['validation_results'].keys()
    assert set(vk) == set(vk1)

    assert r['params']['model_params']['seed'] == 0
    assert r['params']['model_params']['func']['modname'] == 'tfutils.model'
    assert r['params']['model_params']['func']['objname'] == 'mnist_tfutils'
    assert set(['hidden1', 'hidden2', u'softmax_linear']).difference(r['params']['model_params']['cfg_final'].keys()) == set()

    _k = vk[0]
    should_contain = ['agg_func', 'data_params', 'num_steps', 'online_agg_func', 'queue_params', 'targets']
    assert set(should_contain).difference(r['params']['validation_params'][_k].keys()) == set()

    if train:
        assert r['params']['model_params']['train'] is True
        for k in ['num_steps', 'queue_params']:
            assert r['params']['train_params'][k] == params['train_params'][k]

        should_contain = ['loss_params', 'optimizer_params', 'train_params', 'learning_rate_params']
        assert set(should_contain).difference(r['params'].keys()) == set()
        assert r['params']['train_params']['thres_loss'] == 100
        assert r['params']['train_params']['data_params']['func']['modname'] == 'tfutils.data'
        assert r['params']['train_params']['data_params']['func']['objname'] == 'MNIST'

        assert r['params']['loss_params']['agg_func']['modname'] == 'tensorflow.python.ops.math_ops'
        assert r['params']['loss_params']['agg_func']['objname'] == 'reduce_mean'
        assert r['params']['loss_params']['loss_per_case_func']['modname'] == 'tensorflow.python.ops.nn_ops'
        assert r['params']['loss_params']['loss_per_case_func']['objname'] == 'sparse_softmax_cross_entropy_with_logits'
        assert r['params']['loss_params']['targets'] == ['labels']
    else:
        assert 'train' not in r['params']['model_params']
        assert 'train_params' not in r['params']
<|endoftext|>


<commit_before>import os
import glob
import types
import re
import magic
import tarfile
import shutil

import actions
import util

class InhibitorMinStage(actions.InhibitorStage):
    """
    Required:
        - InhibitorStage Requirements
    Optional:
        - package_list:   string/list of packages to add.
        - fs_add:     InhibitorSource, installed as last step over minroot.
        - files:      Files (aside from /bin/busybox) to add to minroot.
                      String or list of globable items.  Symlinks are handled.
        - TODO:  kernel?
    """
    def __init__(self, stage_conf, build_name, stage_name='minimal_stage', **keywds):
        self.package_list   = []
        self.files          = []
        self.symlinks       = {}
        self.full_minroot   = None
        self.full_minstage  = None
        self.tarpath        = None
        self.cpiopath       = None
        self.fs_add         = None
        self.copied_libs    = []
        self.checked_ldd    = []

        super(InhibitorMinStage, self).__init__(stage_conf, build_name, stage_name='min', **keywds)

        # We merge packages to here first, using ROOT=''
        self.minstage       = util.Path('/tmp/inhibitor/minstage')
        # Specific files from minstage are copied here to create
        # the minimal image.
        self.minroot        = util.Path('/tmp/inhibitor/minroot')

        self.portage_cf = self.minstage
        self.env.update({
            'PORTAGE_CONFIGROOT':   self.portage_cf,
        })
        self.root_env = {}
        self.root_env.update(self.env)
        self.root_env['ROOT'] = self.minstage

        self.lddre = re.compile("(/[^ ]*.so[^ ]*)")
        self.ms = magic.open(magic.MAGIC_NONE)
        self.ms.load()
        self.ms.setflags(magic.MAGIC_MIME)

        self.sh_scripts.append('kernel.sh')
        self.kerndir = util.Path('/tmp/inhibitor/kerncache')

    def post_conf(self, inhibitor_state):
        super(InhibitorMinStage, self).post_conf(inhibitor_state)
        self.full_minstage  = self.builddir.pjoin(self.minstage)
        self.full_minroot   = self.builddir.pjoin(self.minroot)
        self.tarpath        = self.istate.paths.stages.pjoin('%s/image.tar.bz2' % (self.build_name,))
        self.cpiopath       = self.istate.paths.stages.pjoin('%s/initramfs.gz' % (self.build_name,))

        if self.fs_add:
            self.fs_add.post_conf(inhibitor_state)
        kerndir = self.istate.paths.kernel.pjoin(self.build_name)
        self.ex_mounts.append(util.Mount(kerndir, self.kerndir, self.builddir))

    def parse_config(self):
        super(InhibitorMinStage, self).parse_config()

        if self.conf.has('package_list'):
            if type(self.conf.package_list) == types.StringType:
                self.package_list = self.conf.package_list.split(' ')
            elif type(self.conf.package_list) == types.ListType:
                self.package_list = self.conf.package_list[:]
            else:
                raise util.InhibitorError("Packages must be either a string or a list")

        if self.conf.has('fs_add'):
            self.fs_add = self.conf.fs_add
            self.fs_add.dest = self.minroot
            self.fs_add.keep = True

        self.files = []
        if self.conf.has('files'):
            if type(self.conf.files) == types.StringType:
                for line in self.conf.files.splitlines():
                    line = line.lstrip('\t ').rstrip('\t ')
                    if len(line) > 0:
                        self.files.extend(line.split('\t '))
            elif type(self.conf.files) == types.ListType:
                self.files.extend(self.conf.files)
            else:
                raise util.InhibitorError("Files must be either a string or a list")

        # Put at the beginning of the list so the user can still overwrite these.
        self.files.insert(0, '/bin/busybox')

        if self.conf.has('kernel'):
            if not self.conf.kernel.has('kconfig'):
                raise util.InhibitorError("No kconfig specified for kernel")
            else:
                self.conf.kernel.kconfig.keep = True
                self.conf.kernel.kconfig.dest = util.Path('/tmp/inhibitor/kconfig')
                self.sources.append(self.conf.kernel.kconfig)
            if not self.conf.kernel.has('kernel_pkg'):
                raise util.InhibitorError('No kernel_pkg specfied for kernel')

    def get_action_sequence(self):
        ret = self.setup_sequence[:]
        ret.append( util.Step(self.prep_dirs,           always=False) )
        ret.append( util.Step(self.merge_busybox,       always=False) )
        if self.conf.has('package_list'):
            ret.append( util.Step(self.merge_packages,  always=False) )
        ret.append( util.Step(self.copy_files,          always=False) )
        ret.append( util.Step(self.install_busybox,     always=False) )
        if self.conf.has('fs_add'):
            ret.append( util.Step(self.install_fs_add,  always=False) )
        if self.conf.has('kernel'):
            ret.append( util.Step(self.build_kernel,    always=False) )
        ret.append( util.Step(self.pack,                always=False) )
        ret.extend( self.cleanup_sequence )
        ret.append( util.Step(self.final_report,        always=True)  )
        return ret

    def prep_dirs(self):
        for d in ('dev', 'bin', 'sbin', 'proc', 'sys', 'etc', 'usr/sbin', 'usr/bin'):
            for i in (self.full_minroot, self.full_minstage):
                t = i.pjoin(d)
                if not os.path.exists(t):
                    os.makedirs(t)

        libpath = self.builddir.pjoin('lib')
        if os.path.islink( libpath ):
            os.symlink(os.readlink(libpath), self.full_minroot.pjoin('lib'))
            os.symlink(os.readlink(libpath), self.full_minstage.pjoin('lib'))
            os.makedirs( os.path.realpath(libpath).replace(self.builddir, self.full_minroot))
            os.makedirs( os.path.realpath(libpath).replace(self.builddir, self.full_minstage))

    def merge_busybox(self):
        env = {}
        use = ''
        if 'USE' in self.env:
            use = self.env['USE']

        if len(self.package_list) == 0:
            use += ' static'
        use += ' make-symlinks'
        env.update(self.root_env)
        env['USE'] = use

        util.chroot(
            path = self.builddir,
            function = util.cmd,
            fargs = {
                'cmdline':
                    '/tmp/inhibitor/sh/inhibitor-run.sh run_emerge --newuse --nodeps sys-apps/busybox',
                'env':env,
            },
            failuref = self._chroot_failure
        )

    def merge_packages(self):
        ir = '/tmp/inhibitor/sh/inhibitor-run.sh'
        util.chroot(
            path = self.builddir,
            function = util.cmd,
            fargs = {
                'cmdline':
                    '%s run_emerge --newuse --onlydeps %s' % (ir, ' '.join(self.package_list)),
                'env':self.env
            },
            failuref = self._chroot_failure
        )

        util.chroot(
            path = self.builddir,
            function = util.cmd,
            fargs = {
                'cmdline':
                    '%s run_emerge --newuse %s' % (ir, ' '.join(self.package_list)),
                'env':self.root_env,
            },
            failuref = self._chroot_failure
        )

    def _ldlibs(self, binp):
        binp = binp.replace(self.full_minstage, '')

        if binp in self.checked_ldd:
            return []
        self.checked_ldd.append(binp)

        rc, output = util.chroot(
                path = self.full_minstage,
                function = util.cmd_out,
                fargs = {
                    'cmdline':          'ldd %s' % (binp,),
                    'raise_exception':   False,
                    'shell':             '/bin/ash',
                },
                failuref = self._chroot_failure
            )
        if rc != 0:
            return []
        libs = []
        for line in output.splitlines():
            try:
                lib = self.lddre.search(line).group(1)
            except (IndexError, AttributeError):
                continue
            if not lib in self.copied_libs:
                self.copied_libs.append(lib)
                libs.append( self.full_minstage.pjoin(lib) )
        return libs

    def path_sync_callback(self, src, targ):
        try:
            mime_type = self.ms.file(src).split(';')[0]
        except AttributeError:
            return
        if not mime_type in ('application/x-executable', 'application/x-sharedlib'):
            return
        needed_libs = self._ldlibs(src)
        for needed_lib in needed_libs:
            util.path_sync(
                needed_lib,
                needed_lib.replace(self.full_minstage, self.full_minroot),
                root = self.full_minstage,
                file_copy_callback = self.path_sync_callback
            )

    def copy_files(self):
        fix_slash = re.compile('///*')

        # Required for ldd
        m = util.Mount('/dev', '/dev', self.full_minstage)
        util.mount(m, self.istate.mount_points)

        for min_path in self.files:
            if '*' in min_path:
                full_glob = self.full_minstage + '/' + min_path
                full_glob = fix_slash.sub('/', full_glob)
                files = glob.glob( full_glob )
            else:
                files = [ self.full_minstage.pjoin(min_path) ]

            for path in files:
                if not os.path.lexists(path):
                    util.warn('Path %s does not exist' % (min_path,))
                    continue
                util.path_sync(
                    path,
                    path.replace(self.full_minstage, self.full_minroot),
                    root = self.full_minstage,
                    file_copy_callback = self.path_sync_callback
                )
        util.umount(m, self.istate.mount_points)

    def install_busybox(self):
        ash_link = self.full_minroot.pjoin('/bin/ash')
        if os.path.exists(ash_link):
            os.unlink(ash_link)
        os.symlink('busybox', self.full_minroot.pjoin('/bin/ash'))

        # Busybox install reads the /proc/self/exe links
        m = util.Mount('/proc', '/proc', self.full_minroot)
        util.mount(m, self.istate.mount_points)

        util.chroot(
            path = self.full_minroot,
            function = util.cmd,
            fargs = {
                'cmdline': '/bin/busybox --install -s',
                'shell': '/bin/ash',
            },
            failuref = self._chroot_failure
        )
        util.umount(m, self.istate.mount_points)

    def install_fs_add(self):
        self.conf.fs_add.fetch()
        self.conf.fs_add.install( root=self.builddir )

    def pack(self):
        basedir = os.path.dirname(self.tarpath)
        if not os.path.lexists(basedir):
            os.makedirs(basedir)

        archive = tarfile.open(self.tarpath, 'w:bz2')
        archive.add(self.full_minroot,
            arcname = '/',
            recursive = True
        )
        archive.close()

        curdir = os.path.realpath(os.curdir)
        os.chdir(self.full_minroot)
        util.cmd('find ./ | cpio -H newc -o | gzip -c -9 > %s' % (self.cpiopath))
        os.chdir(curdir)

        if self.conf.has('kernel'):
            kernel_link = self.builddir.pjoin('/tmp/inhibitor/kernelbuild/boot/kernel')
            kernel_path = os.path.join( os.path.dirname(kernel_link), os.readlink(kernel_link))

            shutil.copy2(kernel_path, os.path.join(basedir, os.path.basename(kernel_path)) )
            link_path = os.path.join(basedir, 'kernel')
            if os.path.lexists(link_path):
                os.unlink(link_path)
            os.symlink(os.path.basename(kernel_path), link_path)

    def build_kernel(self):
        env = {}
        env.update(self.env)
        env['ROOT'] = '/tmp/inhibitor/kernelbuild'

        if not os.path.lexists( env['ROOT'] ):
            os.makedirs( env['ROOT'] )

        args = ['--build_name', self.build_name,
            '--kernel_pkg', self.conf.kernel.kernel_pkg]
        util.chroot(
            path = self.builddir,
            function = util.cmd,
            fargs = {
                'cmdline': '/tmp/inhibitor/sh/kernel.sh %s' % (' '.join(args),),
                'env': env
            },
            failuref = self._chroot_failure,
        )

    def final_report(self):
        util.info("Created %s" % (self.tarpath,))
        util.info("Created %s" % (self.cpiopath,))
        if self.conf.has('kernel'):
            util.info("Kernel copied into %s" % (os.path.dirname(self.tarpath),) )
<commit_msg>min_actions:  add modules and early userspace.<commit_after>import os
import glob
import types
import re
import magic
import tarfile
import shutil

import actions
import util
import source

class InhibitorMinStage(actions.InhibitorStage):
    """
    Required:
        - InhibitorStage Requirements
    Optional:
        - package_list:   string/list of packages to add.
        - fs_add:     InhibitorSource, installed as last step over minroot.
        - files:      Files (aside from /bin/busybox) to add to minroot.
                      String or list of globable items.  Symlinks are handled.
        - TODO:  kernel?
    """
    def __init__(self, stage_conf, build_name, stage_name='minimal_stage', **keywds):
        self.package_list   = []
        self.files          = []
        self.symlinks       = {}
        self.full_minroot   = None
        self.full_minstage  = None
        self.tarpath        = None
        self.cpiopath       = None
        self.fs_add         = None
        self.baselayout     = None
        self.copied_libs    = []
        self.checked_ldd    = []
        self.modules        = []
        self.moduledir      = None

        super(InhibitorMinStage, self).__init__(stage_conf, build_name, stage_name='min', **keywds)

        # We merge packages to here first, using ROOT=''
        self.minstage       = util.Path('/tmp/inhibitor/minstage')
        # Specific files from minstage are copied here to create
        # the minimal image.
        self.minroot        = util.Path('/tmp/inhibitor/minroot')

        self.portage_cf = self.minstage
        self.env.update({
            'PORTAGE_CONFIGROOT':   self.portage_cf,
        })
        self.root_env = {}
        self.root_env.update(self.env)
        self.root_env['ROOT'] = self.minstage

        self.lddre = re.compile("(/[^ ]*.so[^ ]*)")
        self.ms = magic.open(magic.MAGIC_NONE)
        self.ms.load()
        self.ms.setflags(magic.MAGIC_MIME)

        self.sh_scripts.append('kernel.sh')
        self.kerndir = util.Path('/tmp/inhibitor/kerncache')

    def post_conf(self, inhibitor_state):
        super(InhibitorMinStage, self).post_conf(inhibitor_state)
        self.full_minstage  = self.builddir.pjoin(self.minstage)
        self.full_minroot   = self.builddir.pjoin(self.minroot)
        self.moduledir      = inhibitor_state.paths.share.pjoin('sh/early-userspace/modules')
        self.tarpath        = self.istate.paths.stages.pjoin('%s/image.tar.bz2' % (self.build_name,))
        self.cpiopath       = self.istate.paths.stages.pjoin('%s/initramfs.gz' % (self.build_name,))

        if self.fs_add:
            self.fs_add.post_conf(inhibitor_state)
        kerndir = self.istate.paths.kernel.pjoin(self.build_name)
        self.ex_mounts.append(util.Mount(kerndir, self.kerndir, self.builddir))

        self.baselayout     = source.InhibitorSource(
            'file://%s/sh/early-userspace/root' % inhibitor_state.paths.share,
            keep = True,
            dest = self.minroot
        )
        self.baselayout.post_conf(inhibitor_state)

    def parse_config(self):
        super(InhibitorMinStage, self).parse_config()

        if self.conf.has('package_list'):
            if type(self.conf.package_list) == types.StringType:
                self.package_list = self.conf.package_list.split(' ')
            elif type(self.conf.package_list) == types.ListType:
                self.package_list = self.conf.package_list[:]
            else:
                raise util.InhibitorError("Packages must be either a string or a list")

        if self.conf.has('fs_add'):
            self.fs_add = self.conf.fs_add
            self.fs_add.dest = self.minroot
            self.fs_add.keep = True

        if self.conf.has('modules'):
            self.modules = self.conf.modules

        self.files = []
        if self.conf.has('files'):
            if type(self.conf.files) == types.StringType:
                for line in self.conf.files.splitlines():
                    line = line.lstrip('\t ').rstrip('\t ')
                    if len(line) > 0:
                        self.files.extend(line.split('\t '))
            elif type(self.conf.files) == types.ListType:
                self.files.extend(self.conf.files)
            else:
                raise util.InhibitorError("Files must be either a string or a list")

        # Put at the beginning of the list so the user can still overwrite these.
        self.files.insert(0, '/bin/busybox')

        if self.conf.has('kernel'):
            if not self.conf.kernel.has('kconfig'):
                raise util.InhibitorError("No kconfig specified for kernel")
            else:
                self.conf.kernel.kconfig.keep = True
                self.conf.kernel.kconfig.dest = util.Path('/tmp/inhibitor/kconfig')
                self.sources.append(self.conf.kernel.kconfig)
            if not self.conf.kernel.has('kernel_pkg'):
                raise util.InhibitorError('No kernel_pkg specfied for kernel')

    def get_action_sequence(self):
        ret = self.setup_sequence[:]
        ret.append( util.Step(self.prep_minroot,        always=False) )
        ret.append( util.Step(self.merge_busybox,       always=False) )
        if self.conf.has('package_list'):
            ret.append( util.Step(self.merge_packages,  always=False) )
        ret.append( util.Step(self.copy_files,          always=False) )
        ret.append( util.Step(self.install_busybox,     always=False) )
        ret.append( util.Step(self.install_modules,     always=False) )
        if self.conf.has('fs_add'):
            ret.append( util.Step(self.install_fs_add,  always=False) )
        if self.conf.has('kernel'):
            ret.append( util.Step(self.build_kernel,    always=False) )
        ret.append( util.Step(self.update_init,         always=False) )
        ret.append( util.Step(self.pack,                always=False) )
        ret.extend( self.cleanup_sequence )
        ret.append( util.Step(self.final_report,        always=True)  )
        return ret

    def prep_minroot(self):
        for d in ('dev', 'bin', 'sbin', 'proc', 'sys', 'etc', '/tmp',
                    'usr/sbin', 'usr/bin', 'etc/rc.d', 'etc/conf.d'):
            for i in (self.full_minroot, self.full_minstage):
                t = i.pjoin(d)
                if not os.path.exists(t):
                    os.makedirs(t)

        libpath = self.builddir.pjoin('lib')
        if os.path.islink( libpath ):
            os.symlink(os.readlink(libpath), self.full_minroot.pjoin('lib'))
            os.symlink(os.readlink(libpath), self.full_minstage.pjoin('lib'))
            os.makedirs( os.path.realpath(libpath).replace(self.builddir, self.full_minroot))
            os.makedirs( os.path.realpath(libpath).replace(self.builddir, self.full_minstage))

        self.baselayout.fetch()
        self.baselayout.install( root=self.builddir )

    def merge_busybox(self):
        env = {}
        use = ''
        if 'USE' in self.env:
            use = self.env['USE']

        if len(self.package_list) == 0:
            use += ' static'
        use += ' make-symlinks'
        env.update(self.root_env)
        env['USE'] = use

        util.chroot(
            path = self.builddir,
            function = util.cmd,
            fargs = {
                'cmdline':
                    '/tmp/inhibitor/sh/inhibitor-run.sh run_emerge --newuse --nodeps sys-apps/busybox',
                'env':env,
            },
            failuref = self._chroot_failure
        )

    def merge_packages(self):
        ir = '/tmp/inhibitor/sh/inhibitor-run.sh'
        util.chroot(
            path = self.builddir,
            function = util.cmd,
            fargs = {
                'cmdline':
                    '%s run_emerge --newuse --onlydeps %s' % (ir, ' '.join(self.package_list)),
                'env':self.env
            },
            failuref = self._chroot_failure
        )

        util.chroot(
            path = self.builddir,
            function = util.cmd,
            fargs = {
                'cmdline':
                    '%s run_emerge --newuse %s' % (ir, ' '.join(self.package_list)),
                'env':self.root_env,
            },
            failuref = self._chroot_failure
        )

    def _ldlibs(self, binp):
        binp = binp.replace(self.full_minstage, '')

        if binp in self.checked_ldd:
            return []
        self.checked_ldd.append(binp)

        rc, output = util.chroot(
                path = self.full_minstage,
                function = util.cmd_out,
                fargs = {
                    'cmdline':          'ldd %s' % (binp,),
                    'raise_exception':   False,
                    'shell':             '/bin/ash',
                },
                failuref = self._chroot_failure
            )
        if rc != 0:
            return []
        libs = []
        for line in output.splitlines():
            try:
                lib = self.lddre.search(line).group(1)
            except (IndexError, AttributeError):
                continue
            if not lib in self.copied_libs:
                self.copied_libs.append(lib)
                libs.append( self.full_minstage.pjoin(lib) )
        return libs

    def path_sync_callback(self, src, targ):
        try:
            mime_type = self.ms.file(src).split(';')[0]
        except AttributeError:
            return
        if not mime_type in ('application/x-executable', 'application/x-sharedlib'):
            return
        needed_libs = self._ldlibs(src)
        for needed_lib in needed_libs:
            util.path_sync(
                needed_lib,
                needed_lib.replace(self.full_minstage, self.full_minroot),
                root = self.full_minstage,
                file_copy_callback = self.path_sync_callback
            )

    def copy_files(self):
        fix_slash = re.compile('///*')

        # Required for ldd
        m = util.Mount('/dev', '/dev', self.full_minstage)
        util.mount(m, self.istate.mount_points)

        for min_path in self.files:
            if '*' in min_path:
                full_glob = self.full_minstage + '/' + min_path
                full_glob = fix_slash.sub('/', full_glob)
                files = glob.glob( full_glob )
            else:
                files = [ self.full_minstage.pjoin(min_path) ]

            for path in files:
                if not os.path.lexists(path):
                    util.warn('Path %s does not exist' % (min_path,))
                    continue
                util.path_sync(
                    path,
                    path.replace(self.full_minstage, self.full_minroot),
                    root = self.full_minstage,
                    file_copy_callback = self.path_sync_callback
                )
        util.umount(m, self.istate.mount_points)

    def install_busybox(self):
        ash_link = self.full_minroot.pjoin('/bin/ash')
        if os.path.exists(ash_link):
            os.unlink(ash_link)
        os.symlink('busybox', self.full_minroot.pjoin('/bin/ash'))

        # Busybox install reads the /proc/self/exe links
        m = util.Mount('/proc', '/proc', self.full_minroot)
        util.mount(m, self.istate.mount_points)

        util.chroot(
            path = self.full_minroot,
            function = util.cmd,
            fargs = {
                'cmdline': '/bin/busybox --install -s',
                'shell': '/bin/ash',
            },
            failuref = self._chroot_failure
        )
        util.umount(m, self.istate.mount_points)

    def install_modules(self):
        for m in self.modules:
            init = self.moduledir.pjoin('%s.init' % m)
            conf = self.moduledir.pjoin('%s.conf' % m)
            if os.path.exists(init):
                shutil.copy2(init, self.full_minroot.pjoin('etc/init.d/%s' % m))
            if os.path.exists(conf):
                shutil.copy2(conf, self.full_minroot.pjoin('etc/conf.d/%s' % m))

    def install_fs_add(self):
        self.conf.fs_add.fetch()
        self.conf.fs_add.install( root=self.builddir )

    def update_init(self):
        for initd in glob.iglob('%s/*' % self.full_minroot.pjoin('etc/init.d')):
            int_path = initd.replace(self.full_minroot, '')
            util.dbg('Adding %s to init' % int_path)
            util.chroot(
                path = self.full_minroot,
                function = util.cmd,
                fargs = {
                    'cmdline':  '%s enable' % int_path,
                    'shell':    '/bin/ash'
                },
                failuref = self._chroot_failure,
            )

    def pack(self):
        basedir = os.path.dirname(self.tarpath)
        if not os.path.lexists(basedir):
            os.makedirs(basedir)

        archive = tarfile.open(self.tarpath, 'w:bz2')
        archive.add(self.full_minroot,
            arcname = '/',
            recursive = True
        )
        archive.close()

        curdir = os.path.realpath(os.curdir)
        os.chdir(self.full_minroot)
        util.cmd('find ./ | cpio -H newc -o | gzip -c -9 > %s' % (self.cpiopath))
        os.chdir(curdir)

        if self.conf.has('kernel'):
            kernel_link = self.builddir.pjoin('/tmp/inhibitor/kernelbuild/boot/kernel')
            kernel_path = os.path.join( os.path.dirname(kernel_link), os.readlink(kernel_link))

            shutil.copy2(kernel_path, os.path.join(basedir, os.path.basename(kernel_path)) )
            link_path = os.path.join(basedir, 'kernel')
            if os.path.lexists(link_path):
                os.unlink(link_path)
            os.symlink(os.path.basename(kernel_path), link_path)

    def build_kernel(self):
        env = {}
        env.update(self.env)
        env['ROOT'] = '/tmp/inhibitor/kernelbuild'

        if not os.path.lexists( env['ROOT'] ):
            os.makedirs( env['ROOT'] )

        args = ['--build_name', self.build_name,
            '--kernel_pkg', self.conf.kernel.kernel_pkg]
        util.chroot(
            path = self.builddir,
            function = util.cmd,
            fargs = {
                'cmdline': '/tmp/inhibitor/sh/kernel.sh %s' % (' '.join(args),),
                'env': env
            },
            failuref = self._chroot_failure,
        )

    def final_report(self):
        util.info("Created %s" % (self.tarpath,))
        util.info("Created %s" % (self.cpiopath,))
        if self.conf.has('kernel'):
            util.info("Kernel copied into %s" % (os.path.dirname(self.tarpath),) )
<|endoftext|>


<commit_before>#!/usr/bin/python3

import random
import numpy as np
from enum import Enum

class WrongException(Exception):
	pass

class Visibility(Enum):
	hidden = 0
	flag = 1
	visible = 2

	open = visible
	flagged = flag
	unopened = hidden

class Cell(object):
	MINE_STR = 'X'
	ZERO_STR = ' '
	def __init__(self, is_mine, neighbors=None, visible=None):
		#if is_mine is None:
		#	self._is_mine = None
		#else:
		self.is_mine = is_mine

		if neighbors is None:
			self._neighbors = None
		else:
			self.neighbors = neighbors # will go through setter

		if visible is None:
			self._visible = None
		else:
			self._visible = visible

	def __bool__(self):
		return self.is_mine

	__nonzero__ = __bool__

	def __str__(self):
		if self.is_mine:
			return Cell.MINE_STR
		if self.neighbors is None:
			error = NotImplementedError('Incorrectly initialized Cell')
			#error.x, error.y = self. #oops cells dont have their x, y
			raise error
		if self.neighbors is 0:
			return Cell.ZERO_STR
		return str(self.neighbors)

	__repr__ = __str__

	@property
	def neighbors(self):
		return self._neighbors
	@neighbors.setter
	def neighbors(self, value):
		value = int(value)
		if 0 <= value <= 8:
			self._neighbors = value
		else:
			raise ValueError('Neighbors must be between 0 and 8')

	@property
	def visible(self):
		return self._visible
	@visible.setter
	def visible(self, value):
		#if (isinstance(value, Visibility) is False) or (value is None):
		#	raise ValueError('Must be instance of Visibility enum')

		self._visible = Visibility(value) # automatically does checking for Visibility.x or 1, 2, 3

	@property
	def is_mine(self):
		return self._is_mine
	@is_mine.setter
	def is_mine(self, value):
		if value is None:
			raise ValueError('Must not be None. Must be truthy or falsy value.')
		self._is_mine = bool(value)

class Grid(object):
	USE_CELL = True
	def __init__(self, mines):
		try:
			self._mines, self._grid, self.x_size, self.y_size = self._parse_args(mines)
		except (AssertionError, WrongException):
			raise TypeError("Sorry, mines didn't parse correctly.")

		# self._grid = (()*self.y_size,)*x_size

		# print('start reindex')

		self.reindex()

	@staticmethod
	def _parse_args(mines):
		try:
			mines = list(mines)
			'''mines = tuple(
				[
					tuple(
						[
							bool(y) for y in x
							]
					)
					for x in mines
					]
			)'''
			mines = [[bool(y) for y in x] for x in mines]
		except TypeError:
			raise WrongException('TypeError on parsing mines.')

		# ok so now is 2d tuple of True, False
		# check to see if same dimensions

		assert isinstance(mines, list), "Not a tuple"

		# print(mines)

		x_size = len(mines)

		for x in mines:
			assert isinstance(x, list), "Not a list"
			for y in x:
				assert isinstance(y, bool), "Not a bool %s" % str(y)

		try:
			y_size = len(mines[0])
		except IndexError:
			raise WrongException('IndexError on getting "len" so did not tuple correctly')

		for x in mines:
			assert len(x) == y_size, "Not same length"
		#grid = mines[:]
		grid = [[None]*y_size]*x_size
		#if Grid.USE_CELL:
		for x in range(x_size): # TODO FIXME THE BUG IS HERE
			for y in range(y_size):
				temp = mines[x][y]
				temp = bool(temp)
				temp = Cell(temp, None, None)
				grid[x][y] = temp

		return mines, grid, x_size, y_size
	#'''
	@property
	def mines(self):
		return self._mines
	'''
	@mines.setter
	def mines(self, value):
		return self.__class__(value)  # return new instance instead of change self

	# Grid is Immutable

	@property
	def grid(self):
		return self._grid
	'''

	def grid(self, x=None, y=None):
		if None in (x, y):
			return self._grid
		return self._grid[x][y]

	# @property
	def _str_1margin(self):
		s = ''
		for x in range(self.x_size):
			# print(x)
			s += ' ---' * self.y_size
			s += '\n'
			# print('| '*self.y_size)
			for y in range(self.y_size):
				# print(y)
				s += '| '
				now = self.grid(x, y)
				if now is -1:
					s += 'X'
				elif now is 0:
					s += '0'
				else:
					s += str(now)
				s += ' '
			# s = s + str(('X' if self.grid[x][y] is -1 else self.grid[x][y])) + ' '
			s += '|\n'
		s += ' ---' * self.y_size
		return s  # [:-1]

	def _str_0margin(self):
		s = ''
		for x in range(self.x_size):
			# print(x)
			s += ' -' * self.y_size
			s += '\n'
			# print('| '*self.y_size)
			for y in range(self.y_size):
				# print(y)
				s += '|'
				now = self.grid(x, y)
				if now is -1:
					s += 'X'
				elif now is 0:
					s += ' '
				else:
					s += str(now)
				s += ''
			# s = s + str(('X' if self.grid[x][y] is -1 else self.grid[x][y])) + ' '
			s += '|\n'
		s += ' -' * self.y_size
		return s  # [:-1]

	def _str_numpy(self):
		return str(np.array(self.grid()))

	def _str_cell_numpy(self):
		'''#return str(np.array([[('X' for y in x if (y.neighbors == None) else y.neighbors)] for x in self.grid()]))
		return str(
			np.array(
				[
					[
						#'X' if self.mines[x, y] else
						#	(
						#		' ' if not self.grid(x, y).neighbors else self.grid(x, y).neighbors
						#	)
						self.grid(x, y)
					for y in range(self.y_size)  ]
				for x in range(self.x_size)  ]
			)
		)
		#return str(np.array([[str(y) for y in x] for x in self.grid()]))'''
		s = []
		for x in self.grid():
			row = []
			for y in x:
				row.append(str(y))
			s.append(row)

		s = str(s)
		return s

	__str__ = _str_cell_numpy

	__repr__ = __str__

	def reindex(self):
		''' # from before Cell
		grid = ()
		for x in range(self.x_size):
			# print('x=%s'%str(x))
			row = ()
			for y in range(self.y_size):
				# print('y=%s'%str(y))
				if (x, y) in ((1, 0), (1, 4)):
					# print()
					pass
				if self.mines[x][y]:
					row += (-1,)
				else:
					row += (self.get_neighbors(x, y),)  # join 2 tuples
			grid += (row,)

		# Todo do sanity checking o n:
		# 2d tuples, same size
		# all ints, -1 to 8
		# edges have -1 to 5
		# corners have -1 to 3

		# print('end reindex')
		# print(grid)
		self._grid = grid
		'''

		for x in range(self.x_size):
			for y in range(self.x_size):
				if not self.grid(x, y).is_mine:
					self.grid(x, y).neighbors = self.get_neighbors(x, y)

	def get_neighbors(self, x, y) -> list:
		# print('get')
		#assert self.mines[x][y] in (-1, 0)

		neighbors = 0

		for x_ in [-1, 0, 1]:
			# print('_x=%s'%str(x_))
			for y_ in [-1, 0, 1]:
				# print('_y=%s'%str(y_))
				# if not (x_ or y_): # avoid 0, 0 aka center square
				if (x_ is 0) and (y is 0):
					continue
				try:
					X = x_ + x
					Y = y_ + y

					if (X is -1) or (Y is -1):
						raise IndexError  # index is -1, will not
					# throw error normally, but
					#					will wrap-around uninentionally
					#					so stop it from happening

					#neighbor = self.mines[X][Y]
					neighbor = self.grid(X, Y).is_mine
					neighbors += bool(neighbor)
				except IndexError:
					# We are on edge or corner, so no cell here
					pass

		return neighbors
'''
class Mask(object):
	def __init__(self, grid):
		assert isinstance(grid, Grid)
		self._original_grid = grid
		self._mines, self._grid, self.x_size, self.y_size = self.original_grid.mines, self.original_grid.grid, self.original_grid.x_size, self.original_grid.y_size
		self.mask = [[0]*self.y_size]*self.x_size

	@property
	def original_grid(self):
		return self._original_grid

	@property
	def mines(self):
		return self._mines

	@property
	def mask(self):
		return self._grid
'''
def get_random() -> random.Random: # assume random module is imported as random
	return random.random.__self__ # __self__ gets instance a bound method belongs to. Basically, get default random.Random()

def randomize(x_size, y_size, mines, rand=None):
	if rand is None:
		rand = get_random()

	l = []
	grid = []
	for x in range(x_size):
		m = []
		for y in range(y_size):
			m.append(False)
			l.append((x, y))
		grid.append(m)

	#random.shuffle(l)

	l = rand.sample(l, mines)

	#mines = []
	for x in range(mines):
		grid[l[x][0]][l[x][1]] = True

	#for x in grid:
	#	print(x)

	return grid

class RandomGridGenerator(object):
	def __init__(self, x, y, mines, state=None):
		if state:
			self.rand = random.Random(state)
		else:
			self.rand = get_random()

		self.x, self.y, self.mines = x, y, mines

	def __iter__(self):
		return self

	def __next__(self):
		return self.next()

	def next(self):
		#while True:
		#	yield randomize(self.x, self.y, self.mines, self.rand)
		return randomize(self.x, self.y, self.mines, self.rand)

class Game(object):
	def __init__(self, grid, player):
		assert isinstance(player, Grid)
		assert isinstance(player, Player)
		self.grid = grid
		self.player = player
		self.x_size = self.grid.x_size
		self.y_size = self.grid.y_size

		self.mask = Grid(self.grid.mines) # Create copy

class Move(object):
	def __init__(self, move_type, x, y):
		self.type = MoveType(move_type)
		x, y = int(x), int(y)
		if (x < 0) or (y < 0):
			raise ValueError('Out of range ints')
		self.x, self.y = x, y

	def apply(self, grid):
		if not isinstance(grid, Grid):
			raise TypeError('grid must be type Grid')
		if (self.x >= grid.x_size) or (self.y >= grid.y_size):
			raise ValueError('Coordinates out of bounds for Grid')

		cell = grid.grid(self.x, self.y)

		if self.type == MoveType.open:
			cell.visible = Visibility.visible
			if cell.is_mine:
				raise Explosion(self.x, self.y)
		elif self.type == MoveType.flag:
			if cell.visible == Visibility.open:
				return # invalid command. Cannot flag if opened
			elif cell.visible == Visibility.flagged: # unflag if flagged
				cell.visible = Visibility.hidden
			elif cell.visible == Visibility.hidden:
				cell.visible = Visibility.flag
			else:
				raise NotImplementedError()
		else:
			raise NotImplementedError()
		return

class Explosion(Exception):
	def __init__(self, x, y, *a, **k):
		self.x, self.y = x, y
		super().__init__(*a, **k)

class MoveType(Enum):
	open = 1
	flag = 2
	#unflag = 3

class Player(object):
	def __init__(self):
		pass

	def move(self, grid):
		if not isinstance(grid, Grid):
			raise TypeError()

'''
def main():
	import sys
	while True:
		try:
			i = input(">>> ")
			i = "print(" + i + ")"
			exec(i)
		except:
			#print("exception")
			print(sys.exc_info())
'''

def main():
	'''g = Grid([[0, 0, 0, 0, 0, 0, 0, 0, 1], [0, 0, 0, 1, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0, 1, 0],
			  [0, 0, 0, 0, 1, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0, 0, 0],
			  [0, 0, 1, 0, 0, 1, 0, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0]])
	# print(g.mines)
	# print(g.grid)
	print(np.array(g.grid))
	'''
	#g = Grid(randomize(9,9,10))

	while True:
		#i = input("? ")
		#i = int(i)

		#r = RandomGridGenerator(9, 9, 10, state=10)

		#for x in range(3):
		#g = Grid(r.next())			#print(np.array(g.grid()))
		g = Grid([[0,0,1,0],[1,1,0,0],[0,0,0,0],[1,0,0,1]])
		print(g)
		print()
		break

if __name__ == "__main__":
	main()
<commit_msg>Fixed Grid._grid bug. Cleaned up code<commit_after>#!/usr/bin/python3

import random
import numpy as np
from enum import Enum

class WrongException(Exception):
	pass

class Visibility(Enum):
	hidden = 0
	flag = 1
	visible = 2

	open = visible
	flagged = flag
	unopened = hidden

class Cell(object):
	MINE_STR = 'X'
	ZERO_STR = ' '

	def __init__(self, is_mine, neighbors=None, visible=None):
		self.is_mine = is_mine

		if neighbors is None:
			self._neighbors = None
		else:
			self.neighbors = neighbors  # will go through setter

		if visible is None:
			self._visible = None
		else:
			self._visible = visible

	def __bool__(self):
		return self.is_mine

	__nonzero__ = __bool__

	def __str__(self):
		if self.is_mine:
			return Cell.MINE_STR
		if self.neighbors is None:
			error = NotImplementedError('Incorrectly initialized Cell')
			# error.x, error.y = self. #oops cells don't know their x, y
			raise error
		if self.neighbors is 0:
			return Cell.ZERO_STR
		return str(self.neighbors)

	__repr__ = __str__

	@property
	def neighbors(self):
		return self._neighbors

	@neighbors.setter
	def neighbors(self, value):
		value = int(value)
		if 0 <= value <= 8:
			self._neighbors = value
		else:
			raise ValueError('Neighbors must be between 0 and 8')

	@property
	def visible(self):
		return self._visible

	@visible.setter
	def visible(self, value):
		self._visible = Visibility(value)  # automatically does checking for Visibility.x or 1, 2, 3

	@property
	def is_mine(self):
		return self._is_mine

	@is_mine.setter
	def is_mine(self, value):
		if value is None:
			raise ValueError('Must not be None. Must be truthy or falsy value.')
		self._is_mine = bool(value)

class Grid(object):
	USE_CELL = True

	def __init__(self, mines):
		try:
			self._mines, self._grid, self.x_size, self.y_size = self._parse_args(mines)
		except (AssertionError, WrongException):
			raise TypeError("Sorry, mines didn't parse correctly.")

		self.reindex()

	@staticmethod
	def _parse_args(mines):
		try:
			mines = list(mines)
			mines = [[bool(y) for y in x] for x in mines]
		except TypeError:
			raise WrongException('TypeError on parsing mines.')

		# ok so now is 2d tuple of True, False
		# check to see if same dimensions

		assert isinstance(mines, list), "Not a tuple"

		# print(mines)

		x_size = len(mines)

		for x in mines:
			assert isinstance(x, list), "Not a list"
			for y in x:
				assert isinstance(y, bool), "Not a bool %s" % str(y)

		try:
			y_size = len(mines[0])
		except IndexError:
			raise WrongException('IndexError on getting "len" so did not tuple correctly')

		for x in mines:
			assert len(x) == y_size, "Not same length"
		# Done sanity checking # might not be necessary

		grid = []
		for x in range(x_size):
			row = []
			for y in range(y_size):
				temp = mines[x][y]
				temp = bool(temp)
				temp = Cell(temp, None, None)
				row.append(temp)
			# grid[x][y] = temp
			grid.append(row)

		return mines, grid, x_size, y_size

	@property
	def mines(self):
		return self._mines

	def grid(self, x=None, y=None):
		if None in (x, y):
			return self._grid
		return self._grid[x][y]

	def _str_1margin(self):
		s = ''
		for x in range(self.x_size):
			if x is 0:
				s += ' ' + ('----' * self.y_size)[:-1]
			else:
				s += '|' + ('----' * self.y_size)[:-1] + '|'
			s += '\n'

			for y in range(self.y_size):
				s += '| ' + str(self.grid(x, y)) + ' '
			s += '|\n'
		s += ' ' + ('----' * self.y_size)[:-1]
		return s

	def _str_0margin(self):
		s = ''
		for x in range(self.x_size):
			s += ' -' * self.y_size
			s += '\n'
			for y in range(self.y_size):
				s += '|'
				now = str(self.grid(x, y))
				s += now
				s += ''
			s += '|\n'
		s += ' -' * self.y_size
		return s

	def _str_numpy(self):
		return str(np.array(self.grid()))

	__str__ = _str_1margin  # This is changed

	__repr__ = __str__

	def reindex(self):
		for x in range(self.x_size):
			for y in range(self.x_size):
				if not self.grid(x, y).is_mine:
					self.grid(x, y).neighbors = self.get_neighbors(x, y)

	def get_neighbors(self, x, y) -> list:
		neighbors = 0

		for x_ in [-1, 0, 1]:
			# print('_x=%s'%str(x_))
			for y_ in [-1, 0, 1]:

				if (x_ is 0) and (y is 0):  # faster than if not (x_ or y_)
					continue  # avoid 0, 0 aka center square
				try:
					X = x_ + x
					Y = y_ + y

					if (X is -1) or (Y is -1):
						raise IndexError  # -1 index is invalid, so just skip this (x, y) pair

					neighbor = self.grid(X, Y).is_mine
					neighbors += bool(neighbor)
				except IndexError:
					# We are on edge or corner, so no cell here
					pass

		return neighbors

def get_random() -> random.Random:  # assume random module is imported as random
	return random.random.__self__  # __self__ gets instance a bound method belongs to. Basically, get default random.Random()

def randomize(x_size, y_size, mines, rand=None):
	if rand is None:
		rand = get_random()

	l = []
	grid = []
	for x in range(x_size):
		m = []
		for y in range(y_size):
			m.append(False)
			l.append((x, y))
		grid.append(m)

	l = rand.sample(l, mines)  # better than random.shuffle(l); l = l[:mines]

	# mines = []
	for x in range(mines):
		grid[l[x][0]][l[x][1]] = True

	return grid

class RandomGridGenerator(object):
	def __init__(self, x, y, mines, state=None):
		if state:
			self.rand = random.Random(state)
		else:
			self.rand = get_random()

		self.x, self.y, self.mines = x, y, mines

	def __iter__(self):
		return self

	def __next__(self):
		return self.next()

	def next(self):
		return randomize(self.x, self.y, self.mines, self.rand)

class Game(object):  # NOT IMPLEMENTED
	def __init__(self, grid, player):
		raise NotImplementedError()
		assert isinstance(player, Grid)
		assert isinstance(player, Player)
		self.grid = grid
		self.player = player
		self.x_size = self.grid.x_size
		self.y_size = self.grid.y_size

		self.mask = Grid(self.grid.mines)  # Create copy

class Move(object):  # All grid manipulation logic here
	def __init__(self, move_type, x, y):
		self.type = MoveType(move_type)
		x, y = int(x), int(y)
		if (x < 0) or (y < 0):
			raise ValueError('Out of range ints')
		self.x, self.y = x, y

	def apply(self, grid):
		if not isinstance(grid, Grid):
			raise TypeError('grid must be type Grid')
		if (self.x >= grid.x_size) or (self.y >= grid.y_size):
			raise ValueError('Coordinates out of bounds for Grid')

		cell = grid.grid(self.x, self.y)

		if self.type == MoveType.open:
			cell.visible = Visibility.visible
			if cell.is_mine:
				raise Explosion(self.x, self.y)
		elif self.type == MoveType.flag:
			if cell.visible == Visibility.open:
				return  # invalid command. Cannot flag if opened
			elif cell.visible == Visibility.flagged:  # unflag if flagged
				cell.visible = Visibility.hidden
			elif cell.visible == Visibility.hidden:
				cell.visible = Visibility.flag
			else:
				raise NotImplementedError()
		else:
			raise NotImplementedError()
		return

class Explosion(Exception):
	def __init__(self, x, y, *a, **k):
		self.x, self.y = x, y  # Store (x, y) for Game to use
		super().__init__(*a, **k)

class MoveType(Enum):
	open = 1
	flag = 2

class Player(object):
	def __init__(self):  # Override me
		pass

	def _move(self, grid):
		raise NotImplementedError()  # Override me

	def move(self, grid):  # Do Not Override
		if not isinstance(grid, Grid):
			raise TypeError()
		choice = self._move(grid)

		return MoveType(choice)

'''
def main():
	import sys
	while True:
		try:
			i = input(">>> ")
			i = "print(" + i + ")"
			exec(i)
		except:
			#print("exception")
			print(sys.exc_info())
'''

def main():
	'''g = Grid([[0, 0, 0, 0, 0, 0, 0, 0, 1], [0, 0, 0, 1, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0, 1, 0],
			  [0, 0, 0, 0, 1, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0, 0, 0],
			  [0, 0, 1, 0, 0, 1, 0, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0]])
	g = Grid(randomize(9,9,10))
	'''

	while True:
		i = input("? ")
		i = int(i)

		r = RandomGridGenerator(9, 9, 10, state=i)

		for x in range(3):
			g = Grid(r.next())			#print(np.array(g.grid()))
			#g = Grid([[0, 0, 1, 0], [1, 1, 0, 0], [0, 0, 0, 0], [1, 0, 0, 0]])
			print(g)
			print('\n')
		#break

if __name__ == "__main__":
	main()
	input()
<|endoftext|>