<commit_before># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import os
import sys
from _ssl import PROTOCOL_SSLv23
from collections import Mapping
from ssl import SSLContext
from tempfile import NamedTemporaryFile

import flask
from flask import Flask
from werkzeug.contrib.fixers import HeaderRewriterFix
from werkzeug.wrappers import Response

import active_data
import jx_base
from active_data import record_request, cors_wrapper
from active_data.actions import save_query
from active_data.actions.json import get_raw_json
from active_data.actions.jx import jx_query
from active_data.actions.save_query import SaveQueries, find_query
from active_data.actions.sql import sql_query
from active_data.actions.static import download
from jx_base import container
from mo_files import File
from mo_logs import Log
from mo_logs import constants, startup
from mo_threads import Thread
from pyLibrary import convert
from pyLibrary.env import elasticsearch

OVERVIEW = File("active_data/public/index.html").read()

app = Flask(__name__)
config = None

@app.route('/', defaults={'path': ''}, methods=['OPTIONS', 'HEAD'])
@app.route('/<path:path>', methods=['OPTIONS', 'HEAD'])
@cors_wrapper
def _head(path):
    return Response(b'', status=200)

app.add_url_rule('/tools/<path:filename>', None, download)
app.add_url_rule('/find/<path:hash>', None, find_query)
app.add_url_rule('/query', None, jx_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/query/', None, jx_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/sql', None, sql_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/sql/', None, sql_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/query/<path:path>', None, jx_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/json/<path:path>', None, get_raw_json, methods=['GET'])

@app.route('/', defaults={'path': ''}, methods=['GET', 'POST'])
@cors_wrapper
def _default(path):
    record_request(flask.request, None, flask.request.get_data(), None)

    return Response(
        convert.unicode2utf8(OVERVIEW),
        status=200,
        headers={
            "Content-Type": "text/html"
        }
    )

def setup():
    global config

    try:
        config = startup.read_settings(
            defs=[
                {
                    "name": ["--process_num", "--process"],
                    "help": "Additional port offset (for multiple Flask processes",
                    "type": int,
                    "dest": "process_num",
                    "default": 0,
                    "required": False
                },
                {
                    "name": "app_name",
                    "help": "gunicorn supplied argument",
                    "type": str
                }
            ],
            env_filename=os.environ.get('ACTIVEDATA_CONFIG')
        )

        constants.set(config.constants)
        Log.start(config.debug)

        if config.args.process_num and config.flask.port:
            config.flask.port += config.args.process_num

        # PIPE REQUEST LOGS TO ES DEBUG
        if config.request_logs:
            request_logger = elasticsearch.Cluster(config.request_logs).get_or_create_index(config.request_logs)
            active_data.request_log_queue = request_logger.threaded_queue(max_size=2000)

        # SETUP DEFAULT CONTAINER, SO THERE IS SOMETHING TO QUERY
        container.config.default = {
            "type": "elasticsearch",
            "settings": config.elasticsearch.copy()
        }

        # TURN ON /exit FOR WINDOWS DEBUGGING
        if config.flask.debug or config.flask.allow_exit:
            config.flask.allow_exit = None
            Log.warning("ActiveData is in debug mode")
            app.add_url_rule('/exit', 'exit', _exit)

        # TRIGGER FIRST INSTANCE
        if config.saved_queries:
            setattr(save_query, "query_finder", SaveQueries(config.saved_queries))
        HeaderRewriterFix(app, remove_headers=['Date', 'Server'])

        if config.flask.ssl_context:
            if config.args.process_num:
                Log.error("can not serve ssl and multiple Flask instances at once")
            setup_ssl()

    except BaseException as e:  # MUST CATCH BaseException BECAUSE gunicorn LIKES TO EXIT THAT WAY, AND NOT REPORT
        Log.error("Serious problem with ActiveData service construction!  Shutdown!", cause=e)

def setup_ssl():
    config.flask.ssl_context = None

    if not config.flask.ssl_context:
        return

    ssl_flask = config.flask.copy()
    ssl_flask.debug = False
    ssl_flask.port = 443

    if isinstance(config.flask.ssl_context, Mapping):
        # EXPECTED PEM ENCODED FILE NAMES
        # `load_cert_chain` REQUIRES CONCATENATED LIST OF CERTS
        tempfile = NamedTemporaryFile(delete=False, suffix=".pem")
        try:
            tempfile.write(File(ssl_flask.ssl_context.certificate_file).read_bytes())
            if ssl_flask.ssl_context.certificate_chain_file:
                tempfile.write(File(ssl_flask.ssl_context.certificate_chain_file).read_bytes())
            tempfile.flush()
            tempfile.close()

            context = SSLContext(PROTOCOL_SSLv23)
            context.load_cert_chain(tempfile.name, keyfile=File(ssl_flask.ssl_context.privatekey_file).abspath)

            ssl_flask.ssl_context = context
        except Exception, e:
            Log.error("Could not handle ssl context construction", cause=e)
        finally:
            try:
                tempfile.delete()
            except Exception:
                pass

    def runner(please_stop):
        Log.warning("ActiveData listening on encrypted port {{port}}", port=ssl_flask.port)
        app.run(**ssl_flask)

    Thread.run("SSL Server", runner)

    if config.flask.ssl_context and config.flask.port != 80:
        Log.warning("ActiveData has SSL context, but is still listening on non-encrypted http port {{port}}", port=config.flask.port)

    config.flask.ssl_context = None

def _exit():
    Log.note("Got request to shutdown")
    shutdown = flask.request.environ.get('werkzeug.server.shutdown')
    if shutdown:
        shutdown()
    else:
        Log.warning("werkzeug.server.shutdown does not exist")

    return Response(
        convert.unicode2utf8(OVERVIEW),
        status=400,
        headers={
            "Content-Type": "text/html"
        }
    )

print("name is "+__name__)
if __name__ in ("__main__", "active_data.app"):
    try:
        print("setup")
        setup()
        print("flask = "+config.flask)
        if not config.flask:
            print("run app")
            app.run(**config.flask)
    finally:
        Log.stop()

    sys.exit(0)

<commit_msg>try this<commit_after># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import os
import sys
from _ssl import PROTOCOL_SSLv23
from collections import Mapping
from ssl import SSLContext
from tempfile import NamedTemporaryFile

import flask
from flask import Flask
from werkzeug.contrib.fixers import HeaderRewriterFix
from werkzeug.wrappers import Response

import active_data
import jx_base
from active_data import record_request, cors_wrapper
from active_data.actions import save_query
from active_data.actions.json import get_raw_json
from active_data.actions.jx import jx_query
from active_data.actions.save_query import SaveQueries, find_query
from active_data.actions.sql import sql_query
from active_data.actions.static import download
from jx_base import container
from mo_files import File
from mo_logs import Log
from mo_logs import constants, startup
from mo_threads import Thread
from pyLibrary import convert
from pyLibrary.env import elasticsearch

OVERVIEW = File("active_data/public/index.html").read()

app = Flask(__name__)
config = None

@app.route('/', defaults={'path': ''}, methods=['OPTIONS', 'HEAD'])
@app.route('/<path:path>', methods=['OPTIONS', 'HEAD'])
@cors_wrapper
def _head(path):
    return Response(b'', status=200)

app.add_url_rule('/tools/<path:filename>', None, download)
app.add_url_rule('/find/<path:hash>', None, find_query)
app.add_url_rule('/query', None, jx_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/query/', None, jx_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/sql', None, sql_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/sql/', None, sql_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/query/<path:path>', None, jx_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/json/<path:path>', None, get_raw_json, methods=['GET'])

@app.route('/', defaults={'path': ''}, methods=['GET', 'POST'])
@cors_wrapper
def _default(path):
    record_request(flask.request, None, flask.request.get_data(), None)

    return Response(
        convert.unicode2utf8(OVERVIEW),
        status=200,
        headers={
            "Content-Type": "text/html"
        }
    )

def setup():
    global config

    try:
        config = startup.read_settings(
            defs=[
                {
                    "name": ["--process_num", "--process"],
                    "help": "Additional port offset (for multiple Flask processes",
                    "type": int,
                    "dest": "process_num",
                    "default": 0,
                    "required": False
                },
                {
                    "name": "app_name",
                    "help": "gunicorn supplied argument",
                    "type": str
                }
            ],
            env_filename=os.environ.get('ACTIVEDATA_CONFIG')
        )

        constants.set(config.constants)
        Log.start(config.debug)

        if config.args.process_num and config.flask.port:
            config.flask.port += config.args.process_num

        # PIPE REQUEST LOGS TO ES DEBUG
        if config.request_logs:
            request_logger = elasticsearch.Cluster(config.request_logs).get_or_create_index(config.request_logs)
            active_data.request_log_queue = request_logger.threaded_queue(max_size=2000)

        # SETUP DEFAULT CONTAINER, SO THERE IS SOMETHING TO QUERY
        container.config.default = {
            "type": "elasticsearch",
            "settings": config.elasticsearch.copy()
        }

        # TURN ON /exit FOR WINDOWS DEBUGGING
        if config.flask.debug or config.flask.allow_exit:
            config.flask.allow_exit = None
            Log.warning("ActiveData is in debug mode")
            app.add_url_rule('/exit', 'exit', _exit)

        # TRIGGER FIRST INSTANCE
        if config.saved_queries:
            setattr(save_query, "query_finder", SaveQueries(config.saved_queries))
        HeaderRewriterFix(app, remove_headers=['Date', 'Server'])

        if config.flask.ssl_context:
            if config.args.process_num:
                Log.error("can not serve ssl and multiple Flask instances at once")
            setup_ssl()

    except BaseException as e:  # MUST CATCH BaseException BECAUSE gunicorn LIKES TO EXIT THAT WAY, AND NOT REPORT
        Log.error("Serious problem with ActiveData service construction!  Shutdown!", cause=e)

def setup_ssl():
    config.flask.ssl_context = None

    if not config.flask.ssl_context:
        return

    ssl_flask = config.flask.copy()
    ssl_flask.debug = False
    ssl_flask.port = 443

    if isinstance(config.flask.ssl_context, Mapping):
        # EXPECTED PEM ENCODED FILE NAMES
        # `load_cert_chain` REQUIRES CONCATENATED LIST OF CERTS
        tempfile = NamedTemporaryFile(delete=False, suffix=".pem")
        try:
            tempfile.write(File(ssl_flask.ssl_context.certificate_file).read_bytes())
            if ssl_flask.ssl_context.certificate_chain_file:
                tempfile.write(File(ssl_flask.ssl_context.certificate_chain_file).read_bytes())
            tempfile.flush()
            tempfile.close()

            context = SSLContext(PROTOCOL_SSLv23)
            context.load_cert_chain(tempfile.name, keyfile=File(ssl_flask.ssl_context.privatekey_file).abspath)

            ssl_flask.ssl_context = context
        except Exception, e:
            Log.error("Could not handle ssl context construction", cause=e)
        finally:
            try:
                tempfile.delete()
            except Exception:
                pass

    def runner(please_stop):
        Log.warning("ActiveData listening on encrypted port {{port}}", port=ssl_flask.port)
        app.run(**ssl_flask)

    Thread.run("SSL Server", runner)

    if config.flask.ssl_context and config.flask.port != 80:
        Log.warning("ActiveData has SSL context, but is still listening on non-encrypted http port {{port}}", port=config.flask.port)

    config.flask.ssl_context = None

def _exit():
    Log.note("Got request to shutdown")
    shutdown = flask.request.environ.get('werkzeug.server.shutdown')
    if shutdown:
        shutdown()
    else:
        Log.warning("werkzeug.server.shutdown does not exist")

    return Response(
        convert.unicode2utf8(OVERVIEW),
        status=400,
        headers={
            "Content-Type": "text/html"
        }
    )

if __name__ in ("__main__", "active_data.app"):
    try:
        setup()
        if config.flask:
            app.run(**config.flask)
    finally:
        Log.stop()

    sys.exit(0)

<|endoftext|>


<commit_before>###################################################################
#  Numexpr - Fast numerical array expression evaluator for NumPy.
#
#      License: MIT
#      Author:  See AUTHORS.txt
#
#  See LICENSE.txt and LICENSES/*.txt for details about copyright and
#  rights to use.
####################################################################

version='2.0'
release=True

if not release:
    version += '.dev'
    import os
    svn_version_file = os.path.join(os.path.dirname(__file__),
                                    '__svn_version__.py')
    if os.path.isfile(svn_version_file):
        import imp
        svn = imp.load_module('numexpr.__svn_version__',
                              open(svn_version_file),
                              svn_version_file,
                              ('.py','U',1))
        version += svn.version
<commit_msg>Version bumped to 2.1.dev<commit_after>###################################################################
#  Numexpr - Fast numerical array expression evaluator for NumPy.
#
#      License: MIT
#      Author:  See AUTHORS.txt
#
#  See LICENSE.txt and LICENSES/*.txt for details about copyright and
#  rights to use.
####################################################################

version='2.1'
release=False

if not release:
    version += '.dev'
    import os
    svn_version_file = os.path.join(os.path.dirname(__file__),
                                    '__svn_version__.py')
    if os.path.isfile(svn_version_file):
        import imp
        svn = imp.load_module('numexpr.__svn_version__',
                              open(svn_version_file),
                              svn_version_file,
                              ('.py','U',1))
        version += svn.version
<|endoftext|>


<commit_before>from __future__ import absolute_import
import theano
import matplotlib
if 'MACOSX' in matplotlib.get_backend().upper():
  matplotlib.use('TKAgg')
import pylab as py
py.ion() ## Turn on plot visualization

import gzip,pickle
import numpy as np
from PIL import Image
import cv2
import keras.backend as K
K.set_image_dim_ordering('th')
from keras.layers import Input, merge, TimeDistributed,LSTM,GRU,RepeatVector
from keras.models import Sequential,Model
from keras.layers.core import Flatten, Dense, Dropout, Activation, Reshape
from keras.initializers import normal, identity, he_normal,glorot_normal,glorot_uniform,he_uniform
from keras.layers.normalization import BatchNormalization
import threading
import ConfigParser
from tqdm import *
import re,copy

def ReadConfig(File):
    config=ConfigParser.ConfigParser()
    config.read(File)
    section=config.sections()
    Global_Params={}
    Global_Params['num_hidden']=eval(config.get(section[0],'num_hidden'))
    Global_Params['num_recurrent']=eval(config.get(section[0],'num_recurrent'))
    Global_Params['look_back']=eval(config.get(section[0],'look_back'))
    Global_Params['look_forward']=eval(config.get(section[0],'look_forward'))
    Global_Params['batch_size']=eval(config.get(section[0],'batch_size'))
    Global_Params['learning_rate']=eval(config.get(section[0],'learning_rate'))
    Global_Params['epochs']=eval(config.get(section[0],'epochs'))
    Global_Params['cool']=config.get(section[0],'cool')
    Global_Params['weight_decay']=eval(config.get(section[0],'weight_decay'))
    Global_Params['noise_factor']=eval(config.get(section[0],'noise_factor'))
    return Global_Params

############# Define Data Generators ################
class ImageNoiseDataGenerator(object):
    '''Generate minibatches with
    realtime data augmentation.
    '''
    def __init__(self,corruption_level=0.5):

        self.__dict__.update(locals())
        self.p=corruption_level
        self.lock = threading.Lock()

    def _flow_index(self, N, batch_size=32, shuffle=False, seed=None):
        b = 0
        total_b = 0
        while 1:
            if b == 0:
                if seed is not None:
                    np.random.seed(seed + total_b)

                if shuffle:
                    index_array = np.random.permutation(N)
                else:
                    index_array = np.arange(N)

            current_index = (b * batch_size) % N
            if N >= current_index + batch_size:
                current_batch_size = batch_size
            else:
                current_batch_size = N - current_index

            if current_batch_size == batch_size:
                b += 1
            else:
                b = 0
            total_b += 1
            yield index_array[current_index: current_index + current_batch_size], current_index, current_batch_size

    def flow(self, X, y, batch_size=32, shuffle=False, seed=None):
        assert len(X) == len(y)
        self.X = X
        self.y = y
        self.flow_generator = self._flow_index(X.shape[0], batch_size, shuffle, seed)
        return self

    def __iter__(self):
        # needed if we want to do something like for x,y in data_gen.flow(...):
        return self

    def next(self):
        # for python 2.x
        # Keep under lock only the mechainsem which advance the indexing of each batch
        # see # http://anandology.com/blog/using-iterators-and-generators/
        with self.lock:
            index_array, current_index, current_batch_size = next(self.flow_generator)
        # The transformation of images is not under thread lock so it can be done in parallel
        bX = np.zeros(tuple([current_batch_size] + list(self.X.shape)[1:]))
        for i, j in enumerate(index_array):
            x = self.X[j]
            x = self.insertnoise(x,corruption_level=self.p)
            bX[i] = x
        bY = self.y[index_array]
        return bX, bY

    def __next__(self):
        # for python 3.x
        return self.next()

    def insertnoise(self,x,corruption_level=0.5):
        return np.random.binomial(1,1-corruption_level,x.shape)*x

### Generate RNN compatible dataset
def create_dataset(dataset, look_back=1,look_forward=1):
    ## input is np.array of dim T,D
    #output is np.array X: N,look_back,D and Y: N,D
    # where N=T-look_back-1
    assert(look_back>=look_forward)
    dataX, dataY = [], []
    for i in range(len(dataset)-look_back-(look_forward-1)-1):
        a = dataset[i:(i+look_back), :]
        dataX.append(a)
        dataY.append(dataset[i + look_back:i+look_back+look_forward, :])
    dataX=np.array(dataX)
    dataY=np.array(dataY)
    if look_back-look_forward>0:
        dataY_mod=np.zeros((dataY.shape[0],dataX.shape[1],dataY.shape[2]))
        dataY_mod[:,0:dataY.shape[1],:]=dataY
    else:
        dataY_mod=dataY
    return dataX, dataY_mod

def generate_timedistributed_forecast(model,x,prediction_length=10):
    ## to be used when rnn is used for sequence to sequence mapping
    N,T,D=x.shape
    x_data=x[0,:,:].copy()
    x_revise=x.copy()
    for i in range(prediction_length):
        y_pred=model.predict(x_revise[0:1,:,:],batch_size=1)
        yf=y_pred[:,0,:]
        #print 'prediction:',yf
        x_data=np.vstack([x_data,yf])

        #x=x.reshape(T,D) ## assume N=1 ... i.e. one sample
        #print 'data before prediction:\n',x
        x_revise[0,0:T-1,:]=x_revise[0,1:T,:]
        x_revise[0,T-1:T,:]=yf
        #x=x.reshape(1,T,D)
        #print 'data after appending prediction\n',x
    return x_data

##### Define Neural Network Models ###################
def simple_test_rnn(T=1,D=1):
    input_shape=(T,D)
    input_img = Input(shape=input_shape)
    encoder=TimeDistributed(Dense(20,activation='relu'))(input_img)
    rnn=LSTM(10,activation='elu',return_sequences=True, stateful=False)(encoder)
    decoder=TimeDistributed(Dense(20,activation='relu'))(rnn)
    model=Model(input=input_img,output=decoder)
    return model

def dense_auto(weights_path=None,input_shape=(784,),hidden_layers=None,nonlinearity='relu'):
    input_img = Input(shape=input_shape)

    if hidden_layers!=None:
        if type(hidden_layers)!=list:
            hidden_layers=list(hidden_layers)
        for i,l in enumerate(hidden_layers):
            if i==0:
                encoded=Dense(l,activation=nonlinearity)(input_img)
            else:
                encoded=Dense(l,activation=nonlinearity)(encoded)

        for i,l in reversed(list(enumerate(hidden_layers))):
            if i <len(hidden_layers)-1:
                if i==len(hidden_layers)-2:
                    decoded=Dense(l,activation=nonlinearity)(encoded)
                else:
                    decoded=Dense(l,activation=nonlinearity)(decoded)
        decoded=Dense(input_shape[0])(decoded)
    else:
        decoded=Dense(input_shape[0])(input_img)

    model=Model(input=input_img,output=decoded)

    if weights_path:
        print('Loading Model')
        model.load_weights(weights_path)
    return model

def rnn_dense_auto(weights_path=None,T=1,D=1,nonlinearity='relu',hidden_layers=None,recurrent_layers=None):
    input_shape=(T,D)
    input_img = Input(shape=input_shape)

    if hidden_layers!=None:
        if type(hidden_layers)!=list:
            hidden_layers=list(hidden_layers)
        for i,l in enumerate(hidden_layers):
            if i==0:
                encoded=TimeDistributed(Dense(l,activation=nonlinearity))(input_img)
            else:
                encoded=TimeDistributed(Dense(l,activation=nonlinearity))(encoded)
            encoded=TimeDistributed(Dropout(0.2))(encoded)
        for i,l in enumerate(recurrent_layers):
            if i==0:
                rnn=GRU(l,return_sequences=True, stateful=False)(encoded)
            else:
               rnn=GRU(l,return_sequences=True, stateful=False)(rnn)
            encoded=TimeDistributed(Dropout(0.2))(encoded)

        for i,l in reversed(list(enumerate(hidden_layers))):
            if i <len(hidden_layers):
                if i==len(hidden_layers)-1:
                    decoded=TimeDistributed(Dense(l,activation=nonlinearity))(rnn)
                else:
                    decoded=TimeDistributed(Dense(l,activation=nonlinearity))(decoded)
        decoded=TimeDistributed(Dense(D))(decoded)
    else:
        decoded=TimeDistributed(Dense(D))(input_img)

    model=Model(input=input_img,output=decoded)

    if weights_path:
        print('Loading Model')
        model.load_weights(weights_path)
    return model

def get_data(X,case='Full'):
    if case.upper()=='FULL':
        X_train=X.copy().reshape(X.shape[0],np.prod(X.shape[1:]))
    if case.upper()=='CENTER':
        X_train=X.mean(axis=2).reshape(X.shape[0],np.prod(X.mean(axis=2).shape[1:]))
    if case.upper()=='CENTERZ':
        X_train=X.mean(axis=2)[:,:,2].reshape(X.shape[0],np.prod(X.mean(axis=2)[:,:,2].shape[1:]))
    return X_train

class Candle_Train():
    def __init__(self, datagen,model, numpylist,nb_epochs,case='CenterZ',batch_size=32,print_data=True,look_back=10,look_forward=1):
        self.datagen=datagen
        self.numpylist=numpylist
        self.epochs=nb_epochs
        self.case=case
        self.batch_size=batch_size
        self.model=model
        self.look_back=look_back
        self.look_forward=look_forward
        self.print_data=print_data

    def train_ac(self):
        bool_sample=False
        epoch_loss=[]
        for e in range(self.epochs):
            file_loss=[]
            for f in self.numpylist:
                if self.print_data:
                    if e==0:
                        print f
                X=np.load(f)
                data=get_data(X,self.case)
                X_train,y_train=create_dataset(data,look_back=self.look_back,look_forward=self.look_forward)
                imggen=self.datagen.flow(X_train, y_train, batch_size=self.batch_size)
                N_iter=X.shape[0]//self.batch_size

                iter_loss=[]
                for _ in range(N_iter+1):
                    x,y=next(imggen)
                    subset_sample_weight=np.ones((x.shape[0],1))
                    sample_weight=np.zeros((x.shape[0],self.look_back))
                    sample_weight[:,0:1]=subset_sample_weight
                    loss_data=self.model.train_on_batch(x,y,sample_weight=sample_weight)
                    iter_loss.append(loss_data)
                file_loss.append(np.array(iter_loss).mean(axis=0))
            print '\nLoss on epoch %d:'%e, file_loss[-1]
            epoch_loss.append(np.array(file_loss).mean(axis=0))
        return epoch_loss

class autoencoder_preprocess():
    def __init__(self,img_size=(784,),noise_factor=0.):
        self.noise=noise_factor
        self.img_size=img_size
        self.lock = threading.Lock()

    def add_noise(self,X_train):
        ## Add noise to input data
        np.random.seed(100)
        ind=np.where(X_train==0)
        rn=self.noise*np.random.rand(np.shape(ind)[1])
        X_train[ind]=rn
        return X_train

    def renormalize(self,X_train,mu,sigma):
        X_train=(X_train-mu)/sigma
        X_train = X_train.astype("float32")
        return X_train

def get_activations(model, layer, X_batch):
    get_activations = K.function([model.layers[0].input, K.learning_phase()], model.layers[layer].output)
    activations = get_activations([X_batch,0])
    return activations

<commit_msg>Minor mods to p2b2.py code<commit_after>from __future__ import absolute_import
import theano
import matplotlib
if 'MACOSX' in matplotlib.get_backend().upper():
  matplotlib.use('TKAgg')
import pylab as py
py.ion() ## Turn on plot visualization

import gzip,pickle
import numpy as np
from PIL import Image
import cv2
import keras.backend as K
K.set_image_dim_ordering('th')
from keras.layers import Input, merge, TimeDistributed,LSTM,GRU,RepeatVector
from keras.models import Sequential,Model
from keras.layers.core import Flatten, Dense, Dropout, Activation, Reshape
from keras.initializers import normal, identity, he_normal,glorot_normal,glorot_uniform,he_uniform
from keras.layers.normalization import BatchNormalization
import threading
import ConfigParser
from tqdm import *
import re,copy

def ReadConfig(File):
    config=ConfigParser.ConfigParser()
    config.read(File)
    section=config.sections()
    Global_Params={}
    Global_Params['num_hidden']=eval(config.get(section[0],'num_hidden'))
    Global_Params['num_recurrent']=eval(config.get(section[0],'num_recurrent'))
    Global_Params['look_back']=eval(config.get(section[0],'look_back'))
    Global_Params['look_forward']=eval(config.get(section[0],'look_forward'))
    Global_Params['batch_size']=eval(config.get(section[0],'batch_size'))
    Global_Params['learning_rate']=eval(config.get(section[0],'learning_rate'))
    Global_Params['epochs']=eval(config.get(section[0],'epochs'))
    Global_Params['cool']=config.get(section[0],'cool')
    Global_Params['weight_decay']=eval(config.get(section[0],'weight_decay'))
    Global_Params['noise_factor']=eval(config.get(section[0],'noise_factor'))
    return Global_Params

############# Define Data Generators ################
class ImageNoiseDataGenerator(object):
    '''Generate minibatches with
    realtime data augmentation.
    '''
    def __init__(self,corruption_level=0.5):

        self.__dict__.update(locals())
        self.p=corruption_level
        self.lock = threading.Lock()

    def _flow_index(self, N, batch_size=32, shuffle=False, seed=None):
        b = 0
        total_b = 0
        while 1:
            if b == 0:
                if seed is not None:
                    np.random.seed(seed + total_b)

                if shuffle:
                    index_array = np.random.permutation(N)
                else:
                    index_array = np.arange(N)

            current_index = (b * batch_size) % N
            if N >= current_index + batch_size:
                current_batch_size = batch_size
            else:
                current_batch_size = N - current_index

            if current_batch_size == batch_size:
                b += 1
            else:
                b = 0
            total_b += 1
            yield index_array[current_index: current_index + current_batch_size], current_index, current_batch_size

    def flow(self, X, y, batch_size=32, shuffle=False, seed=None):
        assert len(X) == len(y)
        self.X = X
        self.y = y
        self.flow_generator = self._flow_index(X.shape[0], batch_size, shuffle, seed)
        return self

    def __iter__(self):
        # needed if we want to do something like for x,y in data_gen.flow(...):
        return self

    def next(self):
        # for python 2.x
        # Keep under lock only the mechainsem which advance the indexing of each batch
        # see # http://anandology.com/blog/using-iterators-and-generators/
        with self.lock:
            index_array, current_index, current_batch_size = next(self.flow_generator)
        # The transformation of images is not under thread lock so it can be done in parallel
        bX = np.zeros(tuple([current_batch_size] + list(self.X.shape)[1:]))
        for i, j in enumerate(index_array):
            x = self.X[j]
            x = self.insertnoise(x,corruption_level=self.p)
            bX[i] = x
        bY = self.y[index_array]
        return bX, bY

    def __next__(self):
        # for python 3.x
        return self.next()

    def insertnoise(self,x,corruption_level=0.5):
        return np.random.binomial(1,1-corruption_level,x.shape)*x

### Generate RNN compatible dataset
def create_dataset(dataset, look_back=1,look_forward=1):
    ## input is np.array of dim T,D
    #output is np.array X: N,look_back,D and Y: N,D
    # where N=T-look_back-1
    assert(look_back>=look_forward)
    dataX, dataY = [], []
    for i in range(len(dataset)-look_back-(look_forward-1)-1):
        a = dataset[i:(i+look_back), :]
        dataX.append(a)
        dataY.append(dataset[i + look_back:i+look_back+look_forward, :])
    dataX=np.array(dataX)
    dataY=np.array(dataY)
    if look_back-look_forward>0:
        dataY_mod=np.zeros((dataY.shape[0],dataX.shape[1],dataY.shape[2]))
        dataY_mod[:,0:dataY.shape[1],:]=dataY
    else:
        dataY_mod=dataY
    return dataX, dataY_mod

def generate_timedistributed_forecast(model,x,prediction_length=10):
    ## to be used when rnn is used for sequence to sequence mapping
    N,T,D=x.shape
    x_data=x[0,:,:].copy()
    x_revise=x.copy()
    for i in range(prediction_length):
        y_pred=model.predict(x_revise[0:1,:,:],batch_size=1)
        yf=y_pred[:,0,:]
        #print 'prediction:',yf
        x_data=np.vstack([x_data,yf])

        #x=x.reshape(T,D) ## assume N=1 ... i.e. one sample
        #print 'data before prediction:\n',x
        x_revise[0,0:T-1,:]=x_revise[0,1:T,:]
        x_revise[0,T-1:T,:]=yf
        #x=x.reshape(1,T,D)
        #print 'data after appending prediction\n',x
    return x_data

##### Define Neural Network Models ###################
def simple_test_rnn(T=1,D=1):
    input_shape=(T,D)
    input_img = Input(shape=input_shape)
    encoder=TimeDistributed(Dense(20,activation='relu'))(input_img)
    rnn=LSTM(10,activation='elu',return_sequences=True, stateful=False)(encoder)
    decoder=TimeDistributed(Dense(20,activation='relu'))(rnn)
    model=Model(input=input_img,output=decoder)
    return model

def dense_auto(weights_path=None,input_shape=(784,),hidden_layers=None,nonlinearity='relu'):
    input_img = Input(shape=input_shape)

    if hidden_layers!=None:
        if type(hidden_layers)!=list:
            hidden_layers=list(hidden_layers)
        for i,l in enumerate(hidden_layers):
            if i==0:
                encoded=Dense(l,activation=nonlinearity)(input_img)
            else:
                encoded=Dense(l,activation=nonlinearity)(encoded)

        for i,l in reversed(list(enumerate(hidden_layers))):
            if i <len(hidden_layers)-1:
                if i==len(hidden_layers)-2:
                    decoded=Dense(l,activation=nonlinearity)(encoded)
                else:
                    decoded=Dense(l,activation=nonlinearity)(decoded)
        decoded=Dense(input_shape[0])(decoded)
    else:
        decoded=Dense(input_shape[0])(input_img)

    model=Model(input=input_img,output=decoded)

    if weights_path:
        print('Loading Model')
        model.load_weights(weights_path)
    return model

def rnn_dense_auto(weights_path=None,T=1,D=1,nonlinearity='relu',hidden_layers=None,recurrent_layers=None):
    input_shape=(T,D)
    input_img = Input(shape=input_shape)

    if hidden_layers!=None:
        if type(hidden_layers)!=list:
            hidden_layers=list(hidden_layers)
        for i,l in enumerate(hidden_layers):
            if i==0:
                encoded=TimeDistributed(Dense(l,activation=nonlinearity))(input_img)
            else:
                encoded=TimeDistributed(Dense(l,activation=nonlinearity))(encoded)
            encoded=TimeDistributed(Dropout(0.2))(encoded)
        for i,l in enumerate(recurrent_layers):
            if i==0:
                rnn=GRU(l,return_sequences=True, stateful=False)(encoded)
            else:
               rnn=GRU(l,return_sequences=True, stateful=False)(rnn)
            encoded=TimeDistributed(Dropout(0.2))(encoded)

        for i,l in reversed(list(enumerate(hidden_layers))):
            if i <len(hidden_layers):
                if i==len(hidden_layers)-1:
                    decoded=TimeDistributed(Dense(l,activation=nonlinearity))(rnn)
                else:
                    decoded=TimeDistributed(Dense(l,activation=nonlinearity))(decoded)
        decoded=TimeDistributed(Dense(D))(decoded)
    else:
        decoded=TimeDistributed(Dense(D))(input_img)

    model=Model(input=input_img,output=decoded)

    if weights_path:
        print('Loading Model')
        model.load_weights(weights_path)
    return model

def get_data(X,case='Full'):
    if case.upper()=='FULL':
        X_train=X.copy().reshape(X.shape[0],np.prod(X.shape[1:]))
    if case.upper()=='CENTER':
        X_train=X.mean(axis=2).reshape(X.shape[0],np.prod(X.mean(axis=2).shape[1:]))
    if case.upper()=='CENTERZ':
        X_train=X.mean(axis=2)[:,:,2].reshape(X.shape[0],np.prod(X.mean(axis=2)[:,:,2].shape[1:]))
    return X_train

class Candle_Train():
    def __init__(self, datagen,model, numpylist,nb_epochs,case='CenterZ',batch_size=32,print_data=True,look_back=10,look_forward=1):
        self.datagen=datagen
        self.numpylist=numpylist
        self.epochs=nb_epochs
        self.case=case
        self.batch_size=batch_size
        self.model=model
        self.look_back=look_back
        self.look_forward=look_forward
        self.print_data=print_data

    def train_ac(self):
        bool_sample=False
        epoch_loss=[]
        for e in range(self.epochs):
            file_loss=[]
            for f in self.numpylist:
                if self.print_data:
                    if e==0:
                        print f
                X=np.load(f)
                data=get_data(X,self.case)
                X_train,y_train=create_dataset(data,look_back=self.look_back,look_forward=self.look_forward)
                imggen=self.datagen.flow(X_train, y_train, batch_size=self.batch_size)
                N_iter=X.shape[0]//self.batch_size

                iter_loss=[]
                for _ in range(N_iter+1):
                    x,y=next(imggen)
                    subset_sample_weight=np.ones((x.shape[0],1))
                    sample_weight=np.zeros((x.shape[0],self.look_back))
                    sample_weight[:,0:look_forward]=subset_sample_weight
                    loss_data=self.model.train_on_batch(x,y,sample_weight=sample_weight)
                    iter_loss.append(loss_data)
                file_loss.append(np.array(iter_loss).mean(axis=0))
            print '\nLoss on epoch %d:'%e, file_loss[-1]
            epoch_loss.append(np.array(file_loss).mean(axis=0))
        return epoch_loss

class autoencoder_preprocess():
    def __init__(self,img_size=(784,),noise_factor=0.):
        self.noise=noise_factor
        self.img_size=img_size
        self.lock = threading.Lock()

    def add_noise(self,X_train):
        ## Add noise to input data
        np.random.seed(100)
        ind=np.where(X_train==0)
        rn=self.noise*np.random.rand(np.shape(ind)[1])
        X_train[ind]=rn
        return X_train

    def renormalize(self,X_train,mu,sigma):
        X_train=(X_train-mu)/sigma
        X_train = X_train.astype("float32")
        return X_train

def get_activations(model, layer, X_batch):
    get_activations = K.function([model.layers[0].input, K.learning_phase()], model.layers[layer].output)
    activations = get_activations([X_batch,0])
    return activations

<|endoftext|>


<commit_before>import RPi.GPIO as gpio
import time
from sys import argv
import os
import subprocess

script,first = argv

x = 0
y = int(first)
z= y/15
print ('You chose {} seconds which is {} cycles').format(y,z)
gpio.setmode(gpio.BCM)
gpio.setup(17,gpio.OUT)
gpio.setup(4,gpio.OUT)
gpio.setup(22,gpio.OUT)
gpio.setup(18,gpio.OUT)
gpio.setup(23,gpio.OUT)
gpio.setup(24,gpio.OUT)
gpio.setup(25,gpio.OUT)
while x< y:
    x = x+15
    gpio.output(24,gpio.LOW)
    gpio.output(25,gpio.HIGH)
    time.sleep(1)
    gpio.output(17,gpio.LOW)
    gpio.output(22,gpio.HIGH)
    time. sleep(1)
    gpio.output(22,gpio.LOW)
    gpio.output(4,gpio.HIGH)
    time.sleep(1)
    gpio.output(4,gpio.LOW)
    gpio.output(18,gpio.HIGH)
    time.sleep(1)
    gpio.output(18,gpio.LOW)
    gpio.output(17,gpio.HIGH)
    time.sleep(1)
    gpio.output(25,gpio.LOW)
    gpio.output(23,gpio.HIGH)
    time.sleep(1)
    gpio.output(17,gpio.LOW)
    gpio.output(22,gpio.HIGH)
    time. sleep(1)
    gpio.output(22,gpio.LOW)
    gpio.output(4,gpio.HIGH)
    time.sleep(1)
    gpio.output(4,gpio.LOW)
    gpio.output(18,gpio.HIGH)
    time.sleep(1)
    gpio.output(18,gpio.LOW)
    gpio.output(17,gpio.HIGH)
    time.sleep(1)
    gpio.output(23,gpio.LOW)
    gpio.output(24,gpio.HIGH)
    time.sleep(1)
    gpio.output(17,gpio.LOW)
    gpio.output(22,gpio.HIGH)
    time. sleep(1)
    gpio.output(22,gpio.LOW)
    gpio.output(4,gpio.HIGH)
    time.sleep(1)
    gpio.output(4,gpio.LOW)
    gpio.output(18,gpio.HIGH)
    time.sleep(1)
    gpio.output(18,gpio.LOW)
    gpio.output(17,gpio.HIGH)
    time.sleep(1)

else:
    os.system("gpio reset")
<commit_msg>float<commit_after>import RPi.GPIO as gpio
import time
from sys import argv
import os
import subprocess

script,first = argv

x = 0
y = int(first)
z= float(y/15)
print ('You chose {} seconds which is {} cycles').format(y,z)
gpio.setmode(gpio.BCM)
gpio.setup(17,gpio.OUT)
gpio.setup(4,gpio.OUT)
gpio.setup(22,gpio.OUT)
gpio.setup(18,gpio.OUT)
gpio.setup(23,gpio.OUT)
gpio.setup(24,gpio.OUT)
gpio.setup(25,gpio.OUT)
while x< y:
    x = x+15
    gpio.output(24,gpio.LOW)
    gpio.output(25,gpio.HIGH)
    time.sleep(1)
    gpio.output(17,gpio.LOW)
    gpio.output(22,gpio.HIGH)
    time. sleep(1)
    gpio.output(22,gpio.LOW)
    gpio.output(4,gpio.HIGH)
    time.sleep(1)
    gpio.output(4,gpio.LOW)
    gpio.output(18,gpio.HIGH)
    time.sleep(1)
    gpio.output(18,gpio.LOW)
    gpio.output(17,gpio.HIGH)
    time.sleep(1)
    gpio.output(25,gpio.LOW)
    gpio.output(23,gpio.HIGH)
    time.sleep(1)
    gpio.output(17,gpio.LOW)
    gpio.output(22,gpio.HIGH)
    time. sleep(1)
    gpio.output(22,gpio.LOW)
    gpio.output(4,gpio.HIGH)
    time.sleep(1)
    gpio.output(4,gpio.LOW)
    gpio.output(18,gpio.HIGH)
    time.sleep(1)
    gpio.output(18,gpio.LOW)
    gpio.output(17,gpio.HIGH)
    time.sleep(1)
    gpio.output(23,gpio.LOW)
    gpio.output(24,gpio.HIGH)
    time.sleep(1)
    gpio.output(17,gpio.LOW)
    gpio.output(22,gpio.HIGH)
    time. sleep(1)
    gpio.output(22,gpio.LOW)
    gpio.output(4,gpio.HIGH)
    time.sleep(1)
    gpio.output(4,gpio.LOW)
    gpio.output(18,gpio.HIGH)
    time.sleep(1)
    gpio.output(18,gpio.LOW)
    gpio.output(17,gpio.HIGH)
    time.sleep(1)

else:
    os.system("gpio reset")
<|endoftext|>


<commit_before>import os
import sys
import inspect
import types
from numpy.core.numerictypes import obj2sctype, generic
from numpy.core.multiarray import dtype as _dtype
from numpy.core import product, ndarray

__all__ = ['issubclass_', 'get_numpy_include', 'issubsctype',
           'issubdtype', 'deprecate', 'get_numarray_include',
           'get_include', 'info', 'source', 'who',
           'byte_bounds', 'may_share_memory']

def issubclass_(arg1, arg2):
    try:
        return issubclass(arg1, arg2)
    except TypeError:
        return False

def issubsctype(arg1, arg2):
    return issubclass(obj2sctype(arg1), obj2sctype(arg2))

def issubdtype(arg1, arg2):
    if issubclass_(arg2, generic):
        return issubclass(_dtype(arg1).type, arg2)
    mro = _dtype(arg2).type.mro()
    if len(mro) > 1:
        val = mro[1]
    else:
        val = mro[0]
    return issubclass(_dtype(arg1).type, val)

def get_include():
    """Return the directory in the package that contains the numpy/*.h header
    files.

    Extension modules that need to compile against numpy should use this
    function to locate the appropriate include directory. Using distutils:

      import numpy
      Extension('extension_name', ...
                include_dirs=[numpy.get_include()])
    """
    import numpy
    if numpy.show_config is None:
        # running from numpy source directory
        d = os.path.join(os.path.dirname(numpy.__file__), 'core', 'include')
    else:
        # using installed numpy core headers
        import numpy.core as core
        d = os.path.join(os.path.dirname(core.__file__), 'include')
    return d

def get_numarray_include(type=None):
    """Return the directory in the package that contains the numpy/*.h header
    files.

    Extension modules that need to compile against numpy should use this
    function to locate the appropriate include directory. Using distutils:

      import numpy
      Extension('extension_name', ...
                include_dirs=[numpy.get_numarray_include()])
    """
    from numpy.numarray import get_numarray_include_dirs
    include_dirs = get_numarray_include_dirs()
    if type is None:
        return include_dirs[0]
    else:
        return include_dirs + [get_include()]

if sys.version_info < (2, 4):
    # Can't set __name__ in 2.3
    import new
    def _set_function_name(func, name):
        func = new.function(func.func_code, func.func_globals,
                            name, func.func_defaults, func.func_closure)
        return func
else:
    def _set_function_name(func, name):
        func.__name__ = name
        return func

def deprecate(func, oldname, newname):
    import warnings
    def newfunc(*args,**kwds):
        warnings.warn("%s is deprecated, use %s" % (oldname, newname),
                      DeprecationWarning)
        return func(*args, **kwds)
    newfunc = _set_function_name(newfunc, oldname)
    doc = func.__doc__
    depdoc = '%s is DEPRECATED in numpy: use %s instead' % (oldname, newname,)
    if doc is None:
        doc = depdoc
    else:
        doc = '\n'.join([depdoc, doc])
    newfunc.__doc__ = doc
    try:
        d = func.__dict__
    except AttributeError:
        pass
    else:
        newfunc.__dict__.update(d)
    return newfunc

get_numpy_include = deprecate(get_include, 'get_numpy_include', 'get_include')

#--------------------------------------------
# Determine if two arrays can share memory
#--------------------------------------------

def byte_bounds(a):
    """(low, high) are pointers to the end-points of an array

    low is the first byte
    high is just *past* the last byte

    If the array is not single-segment, then it may not actually
    use every byte between these bounds.

    The array provided must conform to the Python-side of the array interface
    """
    ai = a.__array_interface__
    a_data = ai['data'][0]
    astrides = ai['strides']
    ashape = ai['shape']
    nd_a = len(ashape)
    bytes_a = int(ai['typestr'][2:])

    a_low = a_high = a_data
    if astrides is None: # contiguous case
        a_high += product(ashape, dtype=int)*bytes_a
    else:
        for shape, stride in zip(ashape, astrides):
            if stride < 0:
                a_low += (shape-1)*stride
            else:
                a_high += (shape-1)*stride
        a_high += bytes_a
    return a_low, a_high

def may_share_memory(a, b):
    """Determine if two arrays can share memory

    The memory-bounds of a and b are computed.  If they overlap then
    this function returns True.  Otherwise, it returns False.

    A return of True does not necessarily mean that the two arrays
    share any element.  It just means that they *might*.
    """
    a_low, a_high = byte_bounds(a)
    b_low, b_high = byte_bounds(b)
    if b_low >= a_high or a_low >= b_high:
        return False
    return True

#-----------------------------------------------------------------------------
# Function for output and information on the variables used.
#-----------------------------------------------------------------------------

def who(vardict=None):
    """Print the scipy arrays in the given dictionary (or globals() if None).
    """
    if vardict is None:
        frame = sys._getframe().f_back
        vardict = frame.f_globals
    sta = []
    cache = {}
    for name in vardict.keys():
        if isinstance(vardict[name],ndarray):
            var = vardict[name]
            idv = id(var)
            if idv in cache.keys():
                namestr = name + " (%s)" % cache[idv]
                original=0
            else:
                cache[idv] = name
                namestr = name
                original=1
            shapestr = " x ".join(map(str, var.shape))
            bytestr = str(var.itemsize*product(var.shape))
            sta.append([namestr, shapestr, bytestr, var.dtype.name,
                        original])

    maxname = 0
    maxshape = 0
    maxbyte = 0
    totalbytes = 0
    for k in range(len(sta)):
        val = sta[k]
        if maxname < len(val[0]):
            maxname = len(val[0])
        if maxshape < len(val[1]):
            maxshape = len(val[1])
        if maxbyte < len(val[2]):
            maxbyte = len(val[2])
        if val[4]:
            totalbytes += int(val[2])

    if len(sta) > 0:
        sp1 = max(10,maxname)
        sp2 = max(10,maxshape)
        sp3 = max(10,maxbyte)
        prval = "Name %s Shape %s Bytes %s Type" % (sp1*' ', sp2*' ', sp3*' ')
        print prval + "\n" + "="*(len(prval)+5) + "\n"

    for k in range(len(sta)):
        val = sta[k]
        print "%s %s %s %s %s %s %s" % (val[0], ' '*(sp1-len(val[0])+4),
                                        val[1], ' '*(sp2-len(val[1])+5),
                                        val[2], ' '*(sp3-len(val[2])+5),
                                        val[3])
    print "\nUpper bound on total bytes  =       %d" % totalbytes
    return

#-----------------------------------------------------------------------------

# NOTE:  pydoc defines a help function which works simliarly to this
#  except it uses a pager to take over the screen.

# combine name and arguments and split to multiple lines of
#  width characters.  End lines on a comma and begin argument list
#  indented with the rest of the arguments.
def _split_line(name, arguments, width):
    firstwidth = len(name)
    k = firstwidth
    newstr = name
    sepstr = ", "
    arglist = arguments.split(sepstr)
    for argument in arglist:
        if k == firstwidth:
            addstr = ""
        else:
            addstr = sepstr
        k = k + len(argument) + len(addstr)
        if k > width:
            k = firstwidth + 1 + len(argument)
            newstr = newstr + ",\n" + " "*(firstwidth+2) + argument
        else:
            newstr = newstr + addstr + argument
    return newstr

_namedict = None
_dictlist = None

# Traverse all module directories underneath globals
# to see if something is defined
def _makenamedict(module='numpy'):
    module = __import__(module, globals(), locals(), [])
    thedict = {module.__name__:module.__dict__}
    dictlist = [module.__name__]
    totraverse = [module.__dict__]
    while 1:
        if len(totraverse) == 0:
            break
        thisdict = totraverse.pop(0)
        for x in thisdict.keys():
            if isinstance(thisdict[x],types.ModuleType):
                modname = thisdict[x].__name__
                if modname not in dictlist:
                    moddict = thisdict[x].__dict__
                    dictlist.append(modname)
                    totraverse.append(moddict)
                    thedict[modname] = moddict
    return thedict, dictlist

def info(object=None,maxwidth=76,output=sys.stdout,toplevel='numpy'):
    """Get help information for a function, class, or module.

       Example:
          >>> from numpy import *
          >>> info(polyval) # doctest: +SKIP

          polyval(p, x)

            Evaluate the polymnomial p at x.

            Description:
                If p is of length N, this function returns the value:
                p[0]*(x**N-1) + p[1]*(x**N-2) + ... + p[N-2]*x + p[N-1]
    """
    global _namedict, _dictlist
    import pydoc

    if hasattr(object,'_ppimport_importer') or \
       hasattr(object, '_ppimport_module'):
        object = object._ppimport_module
    elif hasattr(object, '_ppimport_attr'):
        object = object._ppimport_attr

    if object is None:
        info(info)
    elif isinstance(object, ndarray):
        import numpy.numarray as nn
        nn.info(object, output=output, numpy=1)
    elif isinstance(object, str):
        if _namedict is None:
            _namedict, _dictlist = _makenamedict(toplevel)
        numfound = 0
        objlist = []
        for namestr in _dictlist:
            try:
                obj = _namedict[namestr][object]
                if id(obj) in objlist:
                    print >> output, "\n     *** Repeat reference found in %s *** " % namestr
                else:
                    objlist.append(id(obj))
                    print >> output, "     *** Found in %s ***" % namestr
                    info(obj)
                    print >> output, "-"*maxwidth
                numfound += 1
            except KeyError:
                pass
        if numfound == 0:
            print >> output, "Help for %s not found." % object
        else:
            print >> output, "\n     *** Total of %d references found. ***" % numfound

    elif inspect.isfunction(object):
        name = object.func_name
        arguments = apply(inspect.formatargspec, inspect.getargspec(object))

        if len(name+arguments) > maxwidth:
            argstr = _split_line(name, arguments, maxwidth)
        else:
            argstr = name + arguments

        print >> output, " " + argstr + "\n"
        print >> output, inspect.getdoc(object)

    elif inspect.isclass(object):
        name = object.__name__
        arguments = "()"
        try:
            if hasattr(object, '__init__'):
                arguments = apply(inspect.formatargspec, inspect.getargspec(object.__init__.im_func))
                arglist = arguments.split(', ')
                if len(arglist) > 1:
                    arglist[1] = "("+arglist[1]
                    arguments = ", ".join(arglist[1:])
        except:
            pass

        if len(name+arguments) > maxwidth:
            argstr = _split_line(name, arguments, maxwidth)
        else:
            argstr = name + arguments

        print >> output, " " + argstr + "\n"
        doc1 = inspect.getdoc(object)
        if doc1 is None:
            if hasattr(object,'__init__'):
                print >> output, inspect.getdoc(object.__init__)
        else:
            print >> output, inspect.getdoc(object)

        methods = pydoc.allmethods(object)
        if methods != []:
            print >> output, "\n\nMethods:\n"
            for meth in methods:
                if meth[0] == '_':
                    continue
                thisobj = getattr(object, meth, None)
                if thisobj is not None:
                    methstr, other = pydoc.splitdoc(inspect.getdoc(thisobj) or "None")
                print >> output, "  %s  --  %s" % (meth, methstr)

    elif type(object) is types.InstanceType: ## check for __call__ method
        print >> output, "Instance of class: ", object.__class__.__name__
        print >> output
        if hasattr(object, '__call__'):
            arguments = apply(inspect.formatargspec, inspect.getargspec(object.__call__.im_func))
            arglist = arguments.split(', ')
            if len(arglist) > 1:
                arglist[1] = "("+arglist[1]
                arguments = ", ".join(arglist[1:])
            else:
                arguments = "()"

            if hasattr(object,'name'):
                name = "%s" % object.name
            else:
                name = "<name>"
            if len(name+arguments) > maxwidth:
                argstr = _split_line(name, arguments, maxwidth)
            else:
                argstr = name + arguments

            print >> output, " " + argstr + "\n"
            doc = inspect.getdoc(object.__call__)
            if doc is not None:
                print >> output, inspect.getdoc(object.__call__)
            print >> output, inspect.getdoc(object)

        else:
            print >> output, inspect.getdoc(object)

    elif inspect.ismethod(object):
        name = object.__name__
        arguments = apply(inspect.formatargspec, inspect.getargspec(object.im_func))
        arglist = arguments.split(', ')
        if len(arglist) > 1:
            arglist[1] = "("+arglist[1]
            arguments = ", ".join(arglist[1:])
        else:
            arguments = "()"

        if len(name+arguments) > maxwidth:
            argstr = _split_line(name, arguments, maxwidth)
        else:
            argstr = name + arguments

        print >> output, " " + argstr + "\n"
        print >> output, inspect.getdoc(object)

    elif hasattr(object, '__doc__'):
        print >> output, inspect.getdoc(object)

def source(object, output=sys.stdout):
    """Write source for this object to output.
    """
    try:
        print >> output,  "In file: %s\n" % inspect.getsourcefile(object)
        print >> output,  inspect.getsource(object)
    except:
        print >> output,  "Not available for this object."
<commit_msg>Change scipy -> numpy in who() docstring<commit_after>import os
import sys
import inspect
import types
from numpy.core.numerictypes import obj2sctype, generic
from numpy.core.multiarray import dtype as _dtype
from numpy.core import product, ndarray

__all__ = ['issubclass_', 'get_numpy_include', 'issubsctype',
           'issubdtype', 'deprecate', 'get_numarray_include',
           'get_include', 'info', 'source', 'who',
           'byte_bounds', 'may_share_memory']

def issubclass_(arg1, arg2):
    try:
        return issubclass(arg1, arg2)
    except TypeError:
        return False

def issubsctype(arg1, arg2):
    return issubclass(obj2sctype(arg1), obj2sctype(arg2))

def issubdtype(arg1, arg2):
    if issubclass_(arg2, generic):
        return issubclass(_dtype(arg1).type, arg2)
    mro = _dtype(arg2).type.mro()
    if len(mro) > 1:
        val = mro[1]
    else:
        val = mro[0]
    return issubclass(_dtype(arg1).type, val)

def get_include():
    """Return the directory in the package that contains the numpy/*.h header
    files.

    Extension modules that need to compile against numpy should use this
    function to locate the appropriate include directory. Using distutils:

      import numpy
      Extension('extension_name', ...
                include_dirs=[numpy.get_include()])
    """
    import numpy
    if numpy.show_config is None:
        # running from numpy source directory
        d = os.path.join(os.path.dirname(numpy.__file__), 'core', 'include')
    else:
        # using installed numpy core headers
        import numpy.core as core
        d = os.path.join(os.path.dirname(core.__file__), 'include')
    return d

def get_numarray_include(type=None):
    """Return the directory in the package that contains the numpy/*.h header
    files.

    Extension modules that need to compile against numpy should use this
    function to locate the appropriate include directory. Using distutils:

      import numpy
      Extension('extension_name', ...
                include_dirs=[numpy.get_numarray_include()])
    """
    from numpy.numarray import get_numarray_include_dirs
    include_dirs = get_numarray_include_dirs()
    if type is None:
        return include_dirs[0]
    else:
        return include_dirs + [get_include()]

if sys.version_info < (2, 4):
    # Can't set __name__ in 2.3
    import new
    def _set_function_name(func, name):
        func = new.function(func.func_code, func.func_globals,
                            name, func.func_defaults, func.func_closure)
        return func
else:
    def _set_function_name(func, name):
        func.__name__ = name
        return func

def deprecate(func, oldname, newname):
    import warnings
    def newfunc(*args,**kwds):
        warnings.warn("%s is deprecated, use %s" % (oldname, newname),
                      DeprecationWarning)
        return func(*args, **kwds)
    newfunc = _set_function_name(newfunc, oldname)
    doc = func.__doc__
    depdoc = '%s is DEPRECATED in numpy: use %s instead' % (oldname, newname,)
    if doc is None:
        doc = depdoc
    else:
        doc = '\n'.join([depdoc, doc])
    newfunc.__doc__ = doc
    try:
        d = func.__dict__
    except AttributeError:
        pass
    else:
        newfunc.__dict__.update(d)
    return newfunc

get_numpy_include = deprecate(get_include, 'get_numpy_include', 'get_include')

#--------------------------------------------
# Determine if two arrays can share memory
#--------------------------------------------

def byte_bounds(a):
    """(low, high) are pointers to the end-points of an array

    low is the first byte
    high is just *past* the last byte

    If the array is not single-segment, then it may not actually
    use every byte between these bounds.

    The array provided must conform to the Python-side of the array interface
    """
    ai = a.__array_interface__
    a_data = ai['data'][0]
    astrides = ai['strides']
    ashape = ai['shape']
    nd_a = len(ashape)
    bytes_a = int(ai['typestr'][2:])

    a_low = a_high = a_data
    if astrides is None: # contiguous case
        a_high += product(ashape, dtype=int)*bytes_a
    else:
        for shape, stride in zip(ashape, astrides):
            if stride < 0:
                a_low += (shape-1)*stride
            else:
                a_high += (shape-1)*stride
        a_high += bytes_a
    return a_low, a_high

def may_share_memory(a, b):
    """Determine if two arrays can share memory

    The memory-bounds of a and b are computed.  If they overlap then
    this function returns True.  Otherwise, it returns False.

    A return of True does not necessarily mean that the two arrays
    share any element.  It just means that they *might*.
    """
    a_low, a_high = byte_bounds(a)
    b_low, b_high = byte_bounds(b)
    if b_low >= a_high or a_low >= b_high:
        return False
    return True

#-----------------------------------------------------------------------------
# Function for output and information on the variables used.
#-----------------------------------------------------------------------------

def who(vardict=None):
    """Print the Numpy arrays in the given dictionary (or globals() if None).
    """
    if vardict is None:
        frame = sys._getframe().f_back
        vardict = frame.f_globals
    sta = []
    cache = {}
    for name in vardict.keys():
        if isinstance(vardict[name],ndarray):
            var = vardict[name]
            idv = id(var)
            if idv in cache.keys():
                namestr = name + " (%s)" % cache[idv]
                original=0
            else:
                cache[idv] = name
                namestr = name
                original=1
            shapestr = " x ".join(map(str, var.shape))
            bytestr = str(var.itemsize*product(var.shape))
            sta.append([namestr, shapestr, bytestr, var.dtype.name,
                        original])

    maxname = 0
    maxshape = 0
    maxbyte = 0
    totalbytes = 0
    for k in range(len(sta)):
        val = sta[k]
        if maxname < len(val[0]):
            maxname = len(val[0])
        if maxshape < len(val[1]):
            maxshape = len(val[1])
        if maxbyte < len(val[2]):
            maxbyte = len(val[2])
        if val[4]:
            totalbytes += int(val[2])

    if len(sta) > 0:
        sp1 = max(10,maxname)
        sp2 = max(10,maxshape)
        sp3 = max(10,maxbyte)
        prval = "Name %s Shape %s Bytes %s Type" % (sp1*' ', sp2*' ', sp3*' ')
        print prval + "\n" + "="*(len(prval)+5) + "\n"

    for k in range(len(sta)):
        val = sta[k]
        print "%s %s %s %s %s %s %s" % (val[0], ' '*(sp1-len(val[0])+4),
                                        val[1], ' '*(sp2-len(val[1])+5),
                                        val[2], ' '*(sp3-len(val[2])+5),
                                        val[3])
    print "\nUpper bound on total bytes  =       %d" % totalbytes
    return

#-----------------------------------------------------------------------------

# NOTE:  pydoc defines a help function which works simliarly to this
#  except it uses a pager to take over the screen.

# combine name and arguments and split to multiple lines of
#  width characters.  End lines on a comma and begin argument list
#  indented with the rest of the arguments.
def _split_line(name, arguments, width):
    firstwidth = len(name)
    k = firstwidth
    newstr = name
    sepstr = ", "
    arglist = arguments.split(sepstr)
    for argument in arglist:
        if k == firstwidth:
            addstr = ""
        else:
            addstr = sepstr
        k = k + len(argument) + len(addstr)
        if k > width:
            k = firstwidth + 1 + len(argument)
            newstr = newstr + ",\n" + " "*(firstwidth+2) + argument
        else:
            newstr = newstr + addstr + argument
    return newstr

_namedict = None
_dictlist = None

# Traverse all module directories underneath globals
# to see if something is defined
def _makenamedict(module='numpy'):
    module = __import__(module, globals(), locals(), [])
    thedict = {module.__name__:module.__dict__}
    dictlist = [module.__name__]
    totraverse = [module.__dict__]
    while 1:
        if len(totraverse) == 0:
            break
        thisdict = totraverse.pop(0)
        for x in thisdict.keys():
            if isinstance(thisdict[x],types.ModuleType):
                modname = thisdict[x].__name__
                if modname not in dictlist:
                    moddict = thisdict[x].__dict__
                    dictlist.append(modname)
                    totraverse.append(moddict)
                    thedict[modname] = moddict
    return thedict, dictlist

def info(object=None,maxwidth=76,output=sys.stdout,toplevel='numpy'):
    """Get help information for a function, class, or module.

       Example:
          >>> from numpy import *
          >>> info(polyval) # doctest: +SKIP

          polyval(p, x)

            Evaluate the polymnomial p at x.

            Description:
                If p is of length N, this function returns the value:
                p[0]*(x**N-1) + p[1]*(x**N-2) + ... + p[N-2]*x + p[N-1]
    """
    global _namedict, _dictlist
    import pydoc

    if hasattr(object,'_ppimport_importer') or \
       hasattr(object, '_ppimport_module'):
        object = object._ppimport_module
    elif hasattr(object, '_ppimport_attr'):
        object = object._ppimport_attr

    if object is None:
        info(info)
    elif isinstance(object, ndarray):
        import numpy.numarray as nn
        nn.info(object, output=output, numpy=1)
    elif isinstance(object, str):
        if _namedict is None:
            _namedict, _dictlist = _makenamedict(toplevel)
        numfound = 0
        objlist = []
        for namestr in _dictlist:
            try:
                obj = _namedict[namestr][object]
                if id(obj) in objlist:
                    print >> output, "\n     *** Repeat reference found in %s *** " % namestr
                else:
                    objlist.append(id(obj))
                    print >> output, "     *** Found in %s ***" % namestr
                    info(obj)
                    print >> output, "-"*maxwidth
                numfound += 1
            except KeyError:
                pass
        if numfound == 0:
            print >> output, "Help for %s not found." % object
        else:
            print >> output, "\n     *** Total of %d references found. ***" % numfound

    elif inspect.isfunction(object):
        name = object.func_name
        arguments = apply(inspect.formatargspec, inspect.getargspec(object))

        if len(name+arguments) > maxwidth:
            argstr = _split_line(name, arguments, maxwidth)
        else:
            argstr = name + arguments

        print >> output, " " + argstr + "\n"
        print >> output, inspect.getdoc(object)

    elif inspect.isclass(object):
        name = object.__name__
        arguments = "()"
        try:
            if hasattr(object, '__init__'):
                arguments = apply(inspect.formatargspec, inspect.getargspec(object.__init__.im_func))
                arglist = arguments.split(', ')
                if len(arglist) > 1:
                    arglist[1] = "("+arglist[1]
                    arguments = ", ".join(arglist[1:])
        except:
            pass

        if len(name+arguments) > maxwidth:
            argstr = _split_line(name, arguments, maxwidth)
        else:
            argstr = name + arguments

        print >> output, " " + argstr + "\n"
        doc1 = inspect.getdoc(object)
        if doc1 is None:
            if hasattr(object,'__init__'):
                print >> output, inspect.getdoc(object.__init__)
        else:
            print >> output, inspect.getdoc(object)

        methods = pydoc.allmethods(object)
        if methods != []:
            print >> output, "\n\nMethods:\n"
            for meth in methods:
                if meth[0] == '_':
                    continue
                thisobj = getattr(object, meth, None)
                if thisobj is not None:
                    methstr, other = pydoc.splitdoc(inspect.getdoc(thisobj) or "None")
                print >> output, "  %s  --  %s" % (meth, methstr)

    elif type(object) is types.InstanceType: ## check for __call__ method
        print >> output, "Instance of class: ", object.__class__.__name__
        print >> output
        if hasattr(object, '__call__'):
            arguments = apply(inspect.formatargspec, inspect.getargspec(object.__call__.im_func))
            arglist = arguments.split(', ')
            if len(arglist) > 1:
                arglist[1] = "("+arglist[1]
                arguments = ", ".join(arglist[1:])
            else:
                arguments = "()"

            if hasattr(object,'name'):
                name = "%s" % object.name
            else:
                name = "<name>"
            if len(name+arguments) > maxwidth:
                argstr = _split_line(name, arguments, maxwidth)
            else:
                argstr = name + arguments

            print >> output, " " + argstr + "\n"
            doc = inspect.getdoc(object.__call__)
            if doc is not None:
                print >> output, inspect.getdoc(object.__call__)
            print >> output, inspect.getdoc(object)

        else:
            print >> output, inspect.getdoc(object)

    elif inspect.ismethod(object):
        name = object.__name__
        arguments = apply(inspect.formatargspec, inspect.getargspec(object.im_func))
        arglist = arguments.split(', ')
        if len(arglist) > 1:
            arglist[1] = "("+arglist[1]
            arguments = ", ".join(arglist[1:])
        else:
            arguments = "()"

        if len(name+arguments) > maxwidth:
            argstr = _split_line(name, arguments, maxwidth)
        else:
            argstr = name + arguments

        print >> output, " " + argstr + "\n"
        print >> output, inspect.getdoc(object)

    elif hasattr(object, '__doc__'):
        print >> output, inspect.getdoc(object)

def source(object, output=sys.stdout):
    """Write source for this object to output.
    """
    try:
        print >> output,  "In file: %s\n" % inspect.getsourcefile(object)
        print >> output,  inspect.getsource(object)
    except:
        print >> output,  "Not available for this object."
<|endoftext|>


<commit_before>import sys, threading, traceback
from collections import OrderedDict
import json
import types

import multiconf, envs
from .config_errors import InvalidUsageException

class NestedJsonCallError(Exception):
    pass

def _class_tuple(obj, obj_info=""):
    return ('__class__', obj.__class__.__name__ + obj_info)

class ConfigItemEncoder(json.JSONEncoder):
    recursion_check = threading.local()
    recursion_check.in_default = False

    def __init__(self, filter_callable=None, compact=False, property_methods=True, builders=False, **kwargs):
        """
        filter_callable: func(obj, key, value)
        - filter_callable is called for each key/value pair of attributes on each ConfigItem obj.
        - It it must return a tuple of (key, value). If key is False, the key/value pair is removed from the json output
        compact: Set compact to true if dumping for debug, false for machine readable output.
        property_methods: call @property methods and insert values in output, including a comment that the value is calculated.
        """
        super(ConfigItemEncoder, self).__init__(**kwargs)
        self.filter_out_keys = ('env', 'valid_envs', 'contained_in', 'root_conf', 'attributes', 'frozen')
        self.user_filter_callable = filter_callable
        self.compact = compact
        self.property_methods = property_methods
        self.seen = {}
        self.start_obj = None
        self.builders = builders

    def _class_dict(self, obj):
        if self.compact:
            return OrderedDict((_class_tuple(obj, ' #id: ' + repr(id(obj))),))
        return OrderedDict((_class_tuple(obj), ('__id__', id(obj))))

    def _mc_class_dict(self, obj):
        not_frozen_msg = "" if obj.frozen else ", not-frozen"
        if self.compact:
            msg = " #as: '" + obj.named_as() + "', id: " + str(id(obj)) + not_frozen_msg
            return OrderedDict((_class_tuple(obj, msg),))
        return OrderedDict(( _class_tuple(obj, not_frozen_msg), ('__id__', id(obj))))

    def _check_already_dumped(self, objval):
        # Return (new)objval, done
        # Check for reference to already dumped objects
        return ("#ref id: " + repr(id(objval)), True) if self.seen.get(id(objval)) else (objval, False)

    def _set_already_dumped(self, obj):
        self.seen[id(obj)] = obj

    def _check_nesting(self, obj, child_obj):
        # Returns (new)val
        # Check that object being dumped is actually contained in self
        # We dont want to display an outer/sibling object as nested under an inner object
        # Check for reference to parent or sibling object (in case we dump from a lower level than root)
        if child_obj is obj:
            return "#ref self, id: " + repr(id(child_obj))

        child_obj, done = self._check_already_dumped(child_obj)

        if not done and isinstance(child_obj, multiconf._ConfigBase):
            contained_in = child_obj.contained_in
            if contained_in is obj:
                return child_obj

            while contained_in:
                if contained_in is self.start_obj:
                    return "#ref later, id: " + repr(id(child_obj))
                contained_in = contained_in.contained_in
            else:
                id_msg = ": id: " + repr(child_obj.id) if hasattr(child_obj, 'id') else ''
                name_msg = ", name: " + repr(child_obj.name) if hasattr(child_obj, 'name') else ''
                child_obj = "#outside-ref: " + child_obj.__class__.__name__ + id_msg + name_msg

        return child_obj

    def encode(self, obj, **kwargs):
        #print self.__class__.__name__, "encode: type(obj)", type(obj)
        return super(ConfigItemEncoder, self).encode(obj, **kwargs)

    def iterencode(self, obj, **kwargs):
        #print self.__class__.__name__, "iterencode: type(obj)", type(obj)
        return super(ConfigItemEncoder, self).iterencode(obj, **kwargs)

    # pylint: disable=E0202
    def default(self, obj):
        if ConfigItemEncoder.recursion_check.in_default:
            # print >> sys.stderr, self.__class__.__name__, "json_output.default: type(obj)", type(obj)
            raise NestedJsonCallError("Nested json calls detected. Maybe a @property method calls json or repr (implicitly)?")

        try:
            ConfigItemEncoder.recursion_check.in_default = True
            if not self.start_obj:
                self.start_obj = obj

            obj, dumped = self._check_already_dumped(obj)
            if dumped:
                return obj
            self._set_already_dumped(obj)

            if isinstance(obj, multiconf._ConfigBase):
                #print "# Handle ConfigItems", type(obj)
                dd = self._mc_class_dict(obj)

                entries = ()
                try:
                    entries = dir(obj)
                except Exception as ex:
                    print >> sys.stderr, "Error in json generation:"
                    traceback.print_exception(*sys.exc_info())
                    dd['__json_error__ # trying to list property methods, failed call to dir(), @properties will not be included'] = repr(ex)

                # Order 'env' first on root object
                if isinstance(obj, multiconf.ConfigRoot):
                    dd['env'] = obj.env

                # Handle attributes
                for key, val in obj.iteritems():
                    if self.user_filter_callable:
                        key, val = self.user_filter_callable(obj, key, val)
                        if key is False:
                            continue

                    if not self.builders and isinstance(val, multiconf.ConfigBuilder):
                        continue

                    val = self._check_nesting(obj, val)
                    if key in entries:
                        dd[key + ' #shadowed'] = val
                        continue
                    dd[key] = val

                if not self.property_methods:
                    return dd

                # Handle @property methods (defined in subclasses)
                for key in entries:
                    if key.startswith('_') or key in self.filter_out_keys:
                        continue

                    try:
                        val = getattr(obj, key)
                    except InvalidUsageException as ex:
                        dd[key + ' #invalid usage context'] = repr(ex)
                        continue
                    except Exception as ex:
                        print >> sys.stderr, "Error in json generation:"
                        traceback.print_exception(*sys.exc_info())
                        dd[repr(key) + ' # json_error trying to handle property method'] = repr(ex)
                        continue

                    if type(val) == types.MethodType:
                        continue

                    if not self.builders and isinstance(val, multiconf.ConfigBuilder):
                        continue

                    if self.user_filter_callable:
                        key, val = self.user_filter_callable(obj, key, val)
                        if key is False:
                            continue

                    val = self._check_nesting(obj, val)

                    if self.compact and isinstance(val, (str, int, long, float)):
                        dd[key] = str(val) + ' #calculated'
                        continue

                    if isinstance(val, (list, tuple)):
                        new_list = []
                        for item in val:
                            if not self.builders and isinstance(item, multiconf.ConfigBuilder):
                                continue
                            new_list.append(self._check_nesting(obj, item))
                        dd[key] = new_list
                        dd[key + ' #calculated'] = True
                        continue

                    if isinstance(val, dict):
                        new_dict = OrderedDict()
                        for item_key, item in val.iteritems():
                            if not self.builders and isinstance(item, multiconf.ConfigBuilder):
                                continue
                            new_dict[item_key] = self._check_nesting(obj, item)
                        dd[key] = new_dict
                        dd[key + ' #calculated'] = True
                        continue

                    dd[key] = val
                    dd[key + ' #calculated'] = True
                return dd

            if isinstance(obj, envs.BaseEnv):
                #print "# Handle Env objects", type(obj)
                dd = OrderedDict((_class_tuple(obj),))
                for eg in obj.all():
                    dd['name'] = eg.name
                return dd

            try:
                iterable = iter(obj)
            except TypeError:
                pass
            else:
                #print "# Handle iterable objects", type(obj)
                return list(iterable)

            if isinstance(obj, types.InstanceType):
                #print "# Handle instances of old style classes", type(obj)
                # Note that new style class instances are practically indistinguishable from other types of objects
                dd = self._class_dict(obj)
                for key, val in obj.__dict__.iteritems():
                    if key[0] != '_':
                        dd[key], _dumped = self._check_already_dumped(val)
                return dd

            return "__json_error__ # don't know how to handle obj of type: " + repr(type(obj))

        finally:
            ConfigItemEncoder.recursion_check.in_default = False
<commit_msg>JSON: Optional dump of builders, removed check for builder being returned from a @property method. If this was really done, it should be shown.<commit_after>import sys, threading, traceback
from collections import OrderedDict
import json
import types

import multiconf, envs
from .config_errors import InvalidUsageException

class NestedJsonCallError(Exception):
    pass

def _class_tuple(obj, obj_info=""):
    return ('__class__', obj.__class__.__name__ + obj_info)

class ConfigItemEncoder(json.JSONEncoder):
    recursion_check = threading.local()
    recursion_check.in_default = False

    def __init__(self, filter_callable=None, compact=False, property_methods=True, builders=False, **kwargs):
        """
        filter_callable: func(obj, key, value)
        - filter_callable is called for each key/value pair of attributes on each ConfigItem obj.
        - It it must return a tuple of (key, value). If key is False, the key/value pair is removed from the json output
        compact: Set compact to true if dumping for debug, false for machine readable output.
        property_methods: call @property methods and insert values in output, including a comment that the value is calculated.
        """
        super(ConfigItemEncoder, self).__init__(**kwargs)
        self.filter_out_keys = ('env', 'valid_envs', 'contained_in', 'root_conf', 'attributes', 'frozen')
        self.user_filter_callable = filter_callable
        self.compact = compact
        self.property_methods = property_methods
        self.seen = {}
        self.start_obj = None
        self.builders = builders

    def _class_dict(self, obj):
        if self.compact:
            return OrderedDict((_class_tuple(obj, ' #id: ' + repr(id(obj))),))
        return OrderedDict((_class_tuple(obj), ('__id__', id(obj))))

    def _mc_class_dict(self, obj):
        not_frozen_msg = "" if obj.frozen else ", not-frozen"
        if self.compact:
            msg = " #as: '" + obj.named_as() + "', id: " + str(id(obj)) + not_frozen_msg
            return OrderedDict((_class_tuple(obj, msg),))
        return OrderedDict(( _class_tuple(obj, not_frozen_msg), ('__id__', id(obj))))

    def _check_already_dumped(self, objval):
        # Return (new)objval, done
        # Check for reference to already dumped objects
        return ("#ref id: " + repr(id(objval)), True) if self.seen.get(id(objval)) else (objval, False)

    def _set_already_dumped(self, obj):
        self.seen[id(obj)] = obj

    def _check_nesting(self, obj, child_obj):
        # Returns (new)val
        # Check that object being dumped is actually contained in self
        # We dont want to display an outer/sibling object as nested under an inner object
        # Check for reference to parent or sibling object (in case we dump from a lower level than root)
        if child_obj is obj:
            return "#ref self, id: " + repr(id(child_obj))

        child_obj, done = self._check_already_dumped(child_obj)

        if not done and isinstance(child_obj, multiconf._ConfigBase):
            contained_in = child_obj.contained_in
            if contained_in is obj:
                return child_obj

            while contained_in:
                if contained_in is self.start_obj:
                    return "#ref later, id: " + repr(id(child_obj))
                contained_in = contained_in.contained_in
            else:
                id_msg = ": id: " + repr(child_obj.id) if hasattr(child_obj, 'id') else ''
                name_msg = ", name: " + repr(child_obj.name) if hasattr(child_obj, 'name') else ''
                child_obj = "#outside-ref: " + child_obj.__class__.__name__ + id_msg + name_msg

        return child_obj

    def encode(self, obj, **kwargs):
        #print self.__class__.__name__, "encode: type(obj)", type(obj)
        return super(ConfigItemEncoder, self).encode(obj, **kwargs)

    def iterencode(self, obj, **kwargs):
        #print self.__class__.__name__, "iterencode: type(obj)", type(obj)
        return super(ConfigItemEncoder, self).iterencode(obj, **kwargs)

    # pylint: disable=E0202
    def default(self, obj):
        if ConfigItemEncoder.recursion_check.in_default:
            # print >> sys.stderr, self.__class__.__name__, "json_output.default: type(obj)", type(obj)
            raise NestedJsonCallError("Nested json calls detected. Maybe a @property method calls json or repr (implicitly)?")

        try:
            ConfigItemEncoder.recursion_check.in_default = True
            if not self.start_obj:
                self.start_obj = obj

            obj, dumped = self._check_already_dumped(obj)
            if dumped:
                return obj
            self._set_already_dumped(obj)

            if isinstance(obj, multiconf._ConfigBase):
                #print "# Handle ConfigItems", type(obj)
                dd = self._mc_class_dict(obj)

                entries = ()
                try:
                    entries = dir(obj)
                except Exception as ex:
                    print >> sys.stderr, "Error in json generation:"
                    traceback.print_exception(*sys.exc_info())
                    dd['__json_error__ # trying to list property methods, failed call to dir(), @properties will not be included'] = repr(ex)

                # Order 'env' first on root object
                if isinstance(obj, multiconf.ConfigRoot):
                    dd['env'] = obj.env

                # Handle attributes
                for key, val in obj.iteritems():
                    if self.user_filter_callable:
                        key, val = self.user_filter_callable(obj, key, val)
                        if key is False:
                            continue

                    if not self.builders and isinstance(val, multiconf.ConfigBuilder):
                        continue

                    val = self._check_nesting(obj, val)
                    if key in entries:
                        dd[key + ' #shadowed'] = val
                        continue
                    dd[key] = val

                if not self.property_methods:
                    return dd

                # Handle @property methods (defined in subclasses)
                for key in entries:
                    if key.startswith('_') or key in self.filter_out_keys:
                        continue

                    try:
                        val = getattr(obj, key)
                    except InvalidUsageException as ex:
                        dd[key + ' #invalid usage context'] = repr(ex)
                        continue
                    except Exception as ex:
                        print >> sys.stderr, "Error in json generation:"
                        traceback.print_exception(*sys.exc_info())
                        dd[repr(key) + ' # json_error trying to handle property method'] = repr(ex)
                        continue

                    if type(val) == types.MethodType:
                        continue

                    if self.user_filter_callable:
                        key, val = self.user_filter_callable(obj, key, val)
                        if key is False:
                            continue

                    val = self._check_nesting(obj, val)

                    if self.compact and isinstance(val, (str, int, long, float)):
                        dd[key] = str(val) + ' #calculated'
                        continue

                    if isinstance(val, (list, tuple)):
                        new_list = []
                        for item in val:
                            new_list.append(self._check_nesting(obj, item))
                        dd[key] = new_list
                        dd[key + ' #calculated'] = True
                        continue

                    if isinstance(val, dict):
                        new_dict = OrderedDict()
                        for item_key, item in val.iteritems():
                            new_dict[item_key] = self._check_nesting(obj, item)
                        dd[key] = new_dict
                        dd[key + ' #calculated'] = True
                        continue

                    dd[key] = val
                    dd[key + ' #calculated'] = True
                return dd

            if isinstance(obj, envs.BaseEnv):
                #print "# Handle Env objects", type(obj)
                dd = OrderedDict((_class_tuple(obj),))
                for eg in obj.all():
                    dd['name'] = eg.name
                return dd

            try:
                iterable = iter(obj)
            except TypeError:
                pass
            else:
                #print "# Handle iterable objects", type(obj)
                return list(iterable)

            if isinstance(obj, types.InstanceType):
                #print "# Handle instances of old style classes", type(obj)
                # Note that new style class instances are practically indistinguishable from other types of objects
                dd = self._class_dict(obj)
                for key, val in obj.__dict__.iteritems():
                    if key[0] != '_':
                        dd[key], _dumped = self._check_already_dumped(val)
                return dd

            return "__json_error__ # don't know how to handle obj of type: " + repr(type(obj))

        finally:
            ConfigItemEncoder.recursion_check.in_default = False
<|endoftext|>