<commit_before>import os
from scipy import misc
import numpy as np
SUPPORTED_FORMATS=("png","jpg","PNG","JPG")
IMG_SAVE_PATH = "~/Desktop/ML/results/"
def checkfit(X,Y):
	if not isinstance(X,np.ndarray):
		X=np.array(X)
	if not isinstance(Y,np.ndarray):
		Y=np.array(Y)
	if len(X)!=len(Y):
		raise ValueError("The length of the features vector and target vector does not match")
	elif len(X.shape)!=2:
			raise ValueError("all the values in the feature vector do not have same dimensions")
	else:return X,Y
def one_hot_encoding(y):
	"""creates one-hot vector for the target variables
	for example,if a training set consists of multiple classes y=(0,1,2...n)
	one-hot vector for y(0)=[1,0,0,..n],y(2)=[0,0,1,0,..n]"""
	no_=y.shape[0]
	class_set=set([_[0] for _ in y if isinstance(_,(list,np.ndarray))])
	temp=np.full([no_,len(class_set)],0.0)
	for i in range(len(y)):
		temp[i][y[i]]=1
	return temp
def normalize(X):
	"""	This function normalizes the feature values:
		new_x=(x-min(x))/(max(x)-min(x))"""
	mi=np.min(X,axis=0)
	ma=np.max(X,axis=0)
	r=np.zeros(X.shape)
	r= np.divide((X-mi),(mi-ma))
	return r
def split_train_test(X,Y,per=20):

	"""splits training and testing data
	X=feature vector
	Y=target vector
	per=percentage of data the training set should take
	Example:
		from sklearn.datasets import load_iris
		a=load_iris()
		train_f,train_l,test_f,test_l=split_train_test(a.data,a.target)
	"""
	#if checkfit(X,Y):
	class_set=set(Y)
	l=list(Y)
	count=l.count(list(class_set)[0])
	dim_1=int((per/100)*count)*len(list(class_set))
	dim_2=X.shape[1]
	n_train_f=np.full([dim_1,dim_2],np.inf)
	n_test_f=np.full([len(X)-dim_1,dim_2],np.inf)
	n_train_t=np.full([dim_1,],0)
	n_test_t=np.full([len(X)-dim_1,],0)
	f_i=0
	ins=f_i
	s_i=int((per/100)*count)
	ind=s_i
	temp=0
	for i in range(len(class_set)):
		n_train_f[ins:ins+ind]=X[f_i:s_i]
		n_test_f[temp:(count-ind)+temp]=X[s_i:f_i+count]
		n_train_t[ins:ins+ind]=Y[f_i:s_i]
		n_test_t[temp:(count-ind)+temp]=Y[s_i:f_i+count]
		f_i+=count
		s_i+=count
		ins+=ind
		temp+=count-ind
		return n_train_f,n_train_t,n_test_f,n_test_t

class nx:
	"""a class for custom methods in numpy"""
	def __init__(self,ob):
		if not isinstance(ob,np.ndarray):
			raise "expected np.array object"
		else:
			self.ob=ob
	def add_col(self,col_no,val=np.inf):
		"""this method returns an numpy.ndarray
		which consists of an extra column (with a same value) to the original array object passed in __init__.
		col_no=column number to add
		val=value to be passed"""
		try: new_shape=self.ob.shape[0],self.ob.shape[1]+1
		except IndexError:
			new_shape=self.ob.shape[0],1
		b=np.full(new_shape,np.inf)
		for i in range(len(self.ob)):
			temp=np.insert(self.ob[i],col_no,val)
			b[i]=temp
		return b
class Preprocess:
	"""
	Class for preprocessing, mainly deals with images and their conversion to arrays,checks for valid directory and other paths

	INPUT : root path(optional)
	"""
	def __init__(self,root=None):
		self.root_path=root
	def check_dir(self,path):
		"""check on the directory

		path=root path to the directory

		"""
		p=path
		if not path.endswith("/"):
			raise ValueError("Provided Invalid Path")
		if not os.path.exists(path):
			raise ValueError("Path does not exists")
		else:
			file_l=os.listdir(path)
			self.formats=checkformat(file_l)
			class_list=list(self.formats.keys())
			self.classes=class_list
		self.root_path=p
	def checkformat(self,path=None):
		"""check format of the images in the directory
		INPUT PARAMS:
			path= path to class directories
		"""
		if not path:raise ValueError("Path not provided")
		formats={}
		dir_list=os.listdir(self.root_path)
		for i in dir_list:
			new_path=os.path.join(self.root_path,i)
			j=os.listdir(new_path)
			if not j[0].endswith(SUPPORTED_FORMATS):
				raise ValueError("FILE FORMAT NOT SUPPORTED")
			else:formats[i]=j[0][-3:]
			return True
	def direc_to_array(self,path=None,reshape=True):
		"""This function converts images present in the specified path to arrays.

		INPUT:
		path   =    path to the directory of the images(if not given while creating an instance)

		OUTPUT:

		arr,tar   =    where	arr=features vector [instance of np.ndarray]
					tar=Target vecor	[instance of np.ndarray]
		EXAMPLE:
			>>>path="path_to/image_dir/"
			>>>i=Preprocess(root=path)
			>>>feat,target  =  i.direc_to_array()
				or
			>>>i=Preprocess()
			>>>i.direc_to_array(path=path)
			>>>feat,target  =  i.img_to_array()
		"""
		arr=[]
		tar=[]
		self.class_dict={}
		if path:
			if not os.path.exists(path):
				raise ValueError("Provided invalid path")
			else:self.root_path=path
		if self.checkformat(self.root_path):
			class_list=sorted(os.listdir(self.root_path))
			for i in range(len(class_list)):
				self.class_dict[i]=class_list[i]
				new_path=os.path.join(self.root_path,class_list[i])
				new_dir_list=os.listdir(new_path)
				for j in range(len(new_dir_list)):
					img=self.rgb2grey(self.img_to_array(os.path.join(new_path,new_dir_list[j]),cond=reshape))
					img_f= img.flatten()
					arr.append(img_f)
				target=categorical(np.full([len(new_dir_list),],i),len(class_list))
				tar.extend(target)
		return np.asarray(arr),np.asarray(tar)

	def img_to_array(self,path,cond=False):
		"""This method converts single image to array

		INPUT	:

			path  	:  path to the image

		OUTPUT	:
			array form of the image(matrix of pixels)

		EXAMPLE:
			This is a method of the class Preprocess so we need to create an instance.
			>>>i=Preprocess()
			>>>path="Desktop/images/../.jpg"
			>>>X=i.img_to_array(path)
		"""

		if not os.path.exists(path):
				raise ValueError("Provided invalid path")
		if cond:return np.resize(misc.imread(path),[224,224,3])
		else:return misc.imread(path)

	def rgb2grey(self,img):
		""" INPUT : Array of rgb image
		    OUTPUT :Array of Greyscale image
		"""
		r,g,b = img[:,:,0],img[:,:,1],img[:,:,2]
		grey = 0.2989 * r + 0.5870 * g + 0.1140 * b
		return grey

def sub_mean_ch(X):
	"""
		removes per channel mean
	"""
	temp_images=[]
	X=X.astype(np.float32)
	for i in X:
		i[:,:,0] -= 103.939
		i[:,:,1] -= 116.779
		i[:,:,2] -= 123.68
		i = i.transpose((2,0,1))
		temp_images.append(i)
	return np.array(temp_images).astype(np.float32)
def categorical(target,no_classes):
	"""
		assigning binary vectors for each class

		INPUT:
			target=target vector
			no_class=number of classes

	"""
	target=np.asarray(target,dtype="int32")
	if not no_classes:no_classes=np.max(target)+1
	T=np.zeros((len(target),no_classes))
	T[np.arange(len(target)),target]=1
	return T

<commit_msg>added kwarg<commit_after>import os
from scipy import misc
import numpy as np
SUPPORTED_FORMATS=("png","jpg","PNG","JPG")
IMG_SAVE_PATH = "~/Desktop/ML/results/"
def checkfit(X,Y):
	if not isinstance(X,np.ndarray):
		X=np.array(X)
	if not isinstance(Y,np.ndarray):
		Y=np.array(Y)
	if len(X)!=len(Y):
		raise ValueError("The length of the features vector and target vector does not match")
	elif len(X.shape)!=2:
			raise ValueError("all the values in the feature vector do not have same dimensions")
	else:return X,Y
def one_hot_encoding(y):
	"""creates one-hot vector for the target variables
	for example,if a training set consists of multiple classes y=(0,1,2...n)
	one-hot vector for y(0)=[1,0,0,..n],y(2)=[0,0,1,0,..n]"""
	no_=y.shape[0]
	class_set=set([_[0] for _ in y if isinstance(_,(list,np.ndarray))])
	temp=np.full([no_,len(class_set)],0.0)
	for i in range(len(y)):
		temp[i][y[i]]=1
	return temp
def normalize(X):
	"""	This function normalizes the feature values:
		new_x=(x-min(x))/(max(x)-min(x))"""
	mi=np.min(X,axis=0)
	ma=np.max(X,axis=0)
	r=np.zeros(X.shape)
	r= np.divide((X-mi),(mi-ma))
	return r
def split_train_test(X,Y,per=20):

	"""splits training and testing data
	X=feature vector
	Y=target vector
	per=percentage of data the training set should take
	Example:
		from sklearn.datasets import load_iris
		a=load_iris()
		train_f,train_l,test_f,test_l=split_train_test(a.data,a.target)
	"""
	#if checkfit(X,Y):
	class_set=set(Y)
	l=list(Y)
	count=l.count(list(class_set)[0])
	dim_1=int((per/100)*count)*len(list(class_set))
	dim_2=X.shape[1]
	n_train_f=np.full([dim_1,dim_2],np.inf)
	n_test_f=np.full([len(X)-dim_1,dim_2],np.inf)
	n_train_t=np.full([dim_1,],0)
	n_test_t=np.full([len(X)-dim_1,],0)
	f_i=0
	ins=f_i
	s_i=int((per/100)*count)
	ind=s_i
	temp=0
	for i in range(len(class_set)):
		n_train_f[ins:ins+ind]=X[f_i:s_i]
		n_test_f[temp:(count-ind)+temp]=X[s_i:f_i+count]
		n_train_t[ins:ins+ind]=Y[f_i:s_i]
		n_test_t[temp:(count-ind)+temp]=Y[s_i:f_i+count]
		f_i+=count
		s_i+=count
		ins+=ind
		temp+=count-ind
		return n_train_f,n_train_t,n_test_f,n_test_t

class nx:
	"""a class for custom methods in numpy"""
	def __init__(self,ob):
		if not isinstance(ob,np.ndarray):
			raise "expected np.array object"
		else:
			self.ob=ob
	def add_col(self,col_no,val=np.inf):
		"""this method returns an numpy.ndarray
		which consists of an extra column (with a same value) to the original array object passed in __init__.
		col_no=column number to add
		val=value to be passed"""
		try: new_shape=self.ob.shape[0],self.ob.shape[1]+1
		except IndexError:
			new_shape=self.ob.shape[0],1
		b=np.full(new_shape,np.inf)
		for i in range(len(self.ob)):
			temp=np.insert(self.ob[i],col_no,val)
			b[i]=temp
		return b
class Preprocess:
	"""
	Class for preprocessing, mainly deals with images and their conversion to arrays,checks for valid directory and other paths

	INPUT : root path(optional)
	"""
	def __init__(self,root=None):
		self.root_path=root
	def check_dir(self,path):
		"""check on the directory

		path=root path to the directory

		"""
		p=path
		if not path.endswith("/"):
			raise ValueError("Provided Invalid Path")
		if not os.path.exists(path):
			raise ValueError("Path does not exists")
		else:
			file_l=os.listdir(path)
			self.formats=checkformat(file_l)
			class_list=list(self.formats.keys())
			self.classes=class_list
		self.root_path=p
	def checkformat(self,path=None):
		"""check format of the images in the directory
		INPUT PARAMS:
			path= path to class directories
		"""
		if not path:raise ValueError("Path not provided")
		formats={}
		dir_list=os.listdir(self.root_path)
		for i in dir_list:
			new_path=os.path.join(self.root_path,i)
			j=os.listdir(new_path)
			if not j[0].endswith(SUPPORTED_FORMATS):
				raise ValueError("FILE FORMAT NOT SUPPORTED")
			else:formats[i]=j[0][-3:]
			return True
	def direc_to_array(self,path=None,reshape=True,grey_scale=True):
		"""This function converts images present in the specified path to arrays.

		INPUT:
		path   =    path to the directory of the images(if not given while creating an instance)

		OUTPUT:

		arr,tar   =    where	arr=features vector [instance of np.ndarray]
					tar=Target vecor	[instance of np.ndarray]
		EXAMPLE:
			>>>path="path_to/image_dir/"
			>>>i=Preprocess(root=path)
			>>>feat,target  =  i.direc_to_array()
				or
			>>>i=Preprocess()
			>>>i.direc_to_array(path=path)
			>>>feat,target  =  i.img_to_array()
		"""
		arr=[]
		tar=[]
		self.class_dict={}
		if path:
			if not os.path.exists(path):
				raise ValueError("Provided invalid path")
			else:self.root_path=path
		if self.checkformat(self.root_path):
			class_list=sorted(os.listdir(self.root_path))
			for i in range(len(class_list)):
				self.class_dict[i]=class_list[i]
				new_path=os.path.join(self.root_path,class_list[i])
				new_dir_list=os.listdir(new_path)
				for j in range(len(new_dir_list)):
					img=self.img_to_array(os.path.join(new_path,new_dir_list[j]),cond=True)
					if grey_scale:
						img=self.rgb2grey(img).flatten()
						arr.append(img)
					else:arr.append(img)
				target=categorical(np.full([len(new_dir_list),],i),len(class_list))
				tar.extend(target)
		return np.asarray(arr),np.asarray(tar)

	def img_to_array(self,path,cond=False):
		"""This method converts single image to array

		INPUT	:

			path  	:  path to the image

		OUTPUT	:
			array form of the image(matrix of pixels)

		EXAMPLE:
			This is a method of the class Preprocess so we need to create an instance.
			>>>i=Preprocess()
			>>>path="Desktop/images/../.jpg"
			>>>X=i.img_to_array(path)
		"""

		if not os.path.exists(path):
				raise ValueError("Provided invalid path")
		if cond:return np.resize(misc.imread(path),[28,28,3])
		else:return misc.imread(path)

	def rgb2grey(self,img):
		""" INPUT : Array of rgb image
		    OUTPUT :Array of Greyscale image
		"""
		r,g,b = img[:,:,0],img[:,:,1],img[:,:,2]
		grey = 0.2989 * r + 0.5870 * g + 0.1140 * b
		return grey

def sub_mean_ch(X):
	"""
		removes per channel mean
	"""
	temp_images=[]
	X=X.astype(np.float32)
	for i in X:
		i[:,:,0] -= 103.939
		i[:,:,1] -= 116.779
		i[:,:,2] -= 123.68
		i = i.transpose((2,0,1))
		temp_images.append(i)
	return np.array(temp_images).astype(np.float32)
def categorical(target,no_classes):
	"""
		assigning binary vectors for each class

		INPUT:
			target=target vector
			no_class=number of classes

	"""
	target=np.asarray(target,dtype="int32")
	if not no_classes:no_classes=np.max(target)+1
	T=np.zeros((len(target),no_classes))
	T[np.arange(len(target)),target]=1
	return T

<|endoftext|>


<commit_before># QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from clr import AddReference
AddReference("System")
AddReference("QuantConnect.Algorithm")
AddReference("QuantConnect.Algorithm.Framework")
AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Orders import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework import *

from Selection.FundamentalUniverseSelectionModel import FundamentalUniverseSelectionModel
from Selection.UncorrelatedToSPYUniverseSelectionModel import UncorrelatedToSPYUniverseSelectionModel

from datetime import datetime, timedelta

class UncorrelatedToSPYFrameworkAlgorithm(QCAlgorithmFramework):

    def Initialize(self):

        self.UniverseSettings.Resolution = Resolution.Daily

        self.SetStartDate(2019,2,2)   # Set Start Date
        self.SetEndDate(2019,3,15)    # Set End Date
        self.SetCash(100000)          # Set Strategy Cash

        self.SetUniverseSelection(UncorrelatedToSPYUniverseSelectionModel())
        self.SetAlpha(UncorrelatedToSPYAlphaModel())
        self.SetPortfolioConstruction(EqualWeightingPortfolioConstructionModel())
        self.SetExecution(ImmediateExecutionModel())

class UncorrelatedToSPYAlphaModel(AlphaModel):
    '''Uses ranking of intraday percentage difference between open price and close price to create magnitude and direction prediction for insights'''

    def __init__(self, *args, **kwargs):
        self.lookback = kwargs['lookback'] if 'lookback' in kwargs else 1
        self.numberOfStocks = kwargs['numberOfStocks'] if 'numberOfStocks' in kwargs else 10
        self.resolution = kwargs['resolution'] if 'resolution' in kwargs else Resolution.Daily
        self.predictionInterval = Time.Multiply(Extensions.ToTimeSpan(self.resolution), self.lookback)
        self.symbolDataBySymbol = {}

    def Update(self, algorithm, data):

        insights = []
        ret = []
        symbols = []

        activeSec = [x.Key for x in algorithm.ActiveSecurities]

        for symbol in activeSec:
            if algorithm.ActiveSecurities[symbol].HasData:
                open = algorithm.Securities[symbol].Open
                close = algorithm.Securities[symbol].Close
                if open != 0:
                    openCloseReturn = close/open - 1
                    ret.append(openCloseReturn)
                    symbols.append(symbol)

        # Intraday price change
        symbolsRet = dict(zip(symbols,ret))

        # Rank on price change
        symbolsRanked = dict(sorted(symbolsRet.items(), key=lambda kv: kv[1],reverse=False)[:self.numberOfStocks])

        # Emit "up" insight if the price change is positive and "down" insight if the price change is negative
        for key,value in symbolsRanked.items():
            if value > 0:
                insights.append(Insight.Price(key, self.predictionInterval, InsightDirection.Up, value, None))
            else:
                insights.append(Insight.Price(key, self.predictionInterval, InsightDirection.Down, value, None))

        return insights
<commit_msg>Update UncorrelatedToSPYFrameworkAlgorithm.py<commit_after># QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from clr import AddReference
AddReference("System")
AddReference("QuantConnect.Algorithm")
AddReference("QuantConnect.Algorithm.Framework")
AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Orders import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework import *
from QuantConnect.Algorithm.Framework.Alphas import *
from QuantConnect.Algorithm.Framework.Portfolio import EqualWeightingPortfolioConstructionModel
from QuantConnect.Algorithm.Framework.Execution import ImmediateExecutionModel
from Selection.UncorrelatedToBenchmarkUniverseSelectionModel import UncorrelatedToBenchmarkUniverseSelectionModel
from datetime import timedelta

class UncorrelatedToBenchmarkFrameworkAlgorithm(QCAlgorithmFramework):

    def Initialize(self):

        self.UniverseSettings.Resolution = Resolution.Daily

        self.SetStartDate(2017,1,1)   # Set Start Date
        self.SetEndDate(2017,3,1)     # Set End Date
        self.SetCash(1000000)         # Set Strategy Cash

        benchmark = Symbol.Create("SPY", SecurityType.Equity, Market.USA)
        self.SetUniverseSelection(UncorrelatedToBenchmarkUniverseSelectionModel(benchmark))
        self.SetAlpha(UncorrelatedToBenchmarkAlphaModel())
        self.SetPortfolioConstruction(EqualWeightingPortfolioConstructionModel())
        self.SetExecution(ImmediateExecutionModel())

class UncorrelatedToBenchmarkAlphaModel(AlphaModel):
    '''Uses ranking of intraday percentage difference between open price and close price to create magnitude and direction prediction for insights'''

    def __init__(self, numberOfStocks = 10, predictionInterval = timedelta(1)):
        self.predictionInterval = predictionInterval
        self.numberOfStocks = numberOfStocks

    def Update(self, algorithm, data):
        symbolsRet = dict()

        for kvp in algorithm.ActiveSecurities:
            security = kvp.Value
            if security.HasData:
                open = security.Open
                if open != 0:
                    symbolsRet[security.Symbol] = security.Close / open - 1

        # Rank on the absolute value of price change
        symbolsRet = dict(sorted(symbolsRet.items(), key=lambda kvp: abs(kvp[1]),reverse=True)[:self.numberOfStocks])

        insights = []
        for symbol, price_change in symbolsRet.items():
            # Emit "up" insight if the price change is positive and "down" otherwise
            direction = InsightDirection.Up if price_change > 0 else InsightDirection.Down
            insights.append(Insight.Price(symbol, self.predictionInterval, direction, abs(price_change), None))

        return insights
<|endoftext|>


<commit_before>from google.appengine.ext import ndb
from endpoints_proto_datastore.ndb import EndpointsModel
from google.appengine.api import search

import model

class Phone (EndpointsModel):
	_message_fields_schema =('id')
	owner = ndb.StringProperty()
	organization = ndb.KeyProperty()
	number = ndb.StringProperty()
	type_number = ndb.StringProperty()
	creationTime = ndb.DateTimeProperty(auto_now_add=True)
    lastmodification = ndb.DateTimeProperty(auto_now=True)

    def put(self,**kwargs):
    	ndb.Model.put(self, **kwargs)
    	self.put_index()

    def put_index():
    	""" index the element at each"""
        empty_string = lambda x: x if x else ""
        organization = str(self.organization.id())
        my_document = search.Document(
        	doc_id=str(self.key.id()),
        	fields=[
        	search.TextField(name=u'type', value=u'Phone'),
        	search.TextField(name='title', value = empty_string(self.number),
        	search.TextField(name='type_number',value=empty_string(self.type_number)),
        	search.TextField(name='owner', value = empty_string(self.owner) ),
        	search.DateField(name='creationTime', value = self.creationTime),
        	search.DateField(name='lastmodification', value = self.lastmodification)
        	])
        my_index = search.Index(name="GlobalIndex")
        my_index.put(my_document)

<commit_msg>fix a wrong indentation<commit_after>from google.appengine.ext import ndb
from endpoints_proto_datastore.ndb import EndpointsModel
from google.appengine.api import search

import model

class Phone (EndpointsModel):
    _message_fields_schema =('id')
    owner = ndb.StringProperty()
    organization = ndb.KeyProperty()
    number = ndb.StringProperty()
    type_number = ndb.StringProperty()
    creationTime = ndb.DateTimeProperty(auto_now_add=True)
    lastmodification = ndb.DateTimeProperty(auto_now=True)

    def put(self,**kwargs):
    	ndb.Model.put(self, **kwargs)
    	self.put_index()

    def put_index():
    	""" index the element at each"""
        empty_string = lambda x: x if x else ""
        organization = str(self.organization.id())
        my_document = search.Document(
        	doc_id=str(self.key.id()),
        	fields=[
        	search.TextField(name=u'type', value=u'Phone'),
        	search.TextField(name='title', value = empty_string(self.number),
        	search.TextField(name='type_number',value=empty_string(self.type_number)),
        	search.TextField(name='owner', value = empty_string(self.owner) ),
        	search.DateField(name='creationTime', value = self.creationTime),
        	search.DateField(name='lastmodification', value = self.lastmodification)
        	])
        my_index = search.Index(name="GlobalIndex")
        my_index.put(my_document)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# mxdatareduction.py
#
#    Copyright (C) 2014 Diamond Light Source, Karl Levik
#
# 2014-09-24
#
# Methods to store data from MX reduction pipelines
#

try:
  import cx_Oracle
except ImportError, e:
  print 'Oracle API module not found'
  raise e

import string
import logging
import time
import os
import sys
import datetime
from logging.handlers import RotatingFileHandler
from collections import OrderedDict
import copy
from ispyb_api.ExtendedOrderedDict import ExtendedOrderedDict

class MXDataReduction:
  '''MXDataReduction provides methods to store reduced MX data.'''

  def __init__(self):
    pass

  _program_params = ExtendedOrderedDict([('id',None), ('cmd_line',None), ('programs',None),
    ('status',None), ('message',None), ('starttime',None), ('endtime',None), ('environment',None),
    ('filename1',None), ('filepath1',None), ('filetype1',None),
    ('filename2',None), ('filepath2',None), ('filetype2',None),
    ('filename3',None), ('filepath3',None), ('filetype3',None)])

  _processing_params = ExtendedOrderedDict([('parentid',None), ('spacegroup',None),
    ('refinedcell_a',None), ('refinedcell_b',None), ('refinedcell_c',None),
    ('refinedcell_alpha',None), ('refinedcell_beta',None), ('refinedcell_gamma',None)])

  _scaling_params = ExtendedOrderedDict([
    ('type',None), ('comments',None), ('res_lim_low', None), ('res_lim_high',None), ('r_merge',None),
	('r_meas_within_iplusi_minus',None), ('r_meas_all_iplusi_minus',None), ('r_pim_within_iplusi_minus',None), ('r_pim_all_iplusi_minus',None), ('fract_partial_bias',None), ('n_tot_obs',None),
	('n_tot_unique_obs',None), ('mean_i_sig_i',None), ('completeness',None), ('multiplicity',None), ('anom','0'), ('anom_completeness',None), ('anom_multiplicity',None),
	('cc_half',None), ('cc_anom',None)])

  _integration_params = ExtendedOrderedDict([('parentid',None), ('datacollectionid',None), ('programid',None),
    ('start_image_no',None), ('end_image_no',None), ('refined_detector_dist',None),
    ('refined_xbeam',None), ('refined_ybeam',None), ('rot_axis_x',None), ('rot_axis_y',None), ('rot_axis_z',None),
    ('beam_vec_x',None), ('beam_vec_y',None), ('beam_vec_z',None),
    ('cell_a',None), ('cell_b',None), ('cell_c',None), ('cell_alpha',None), ('cell_beta',None), ('cell_gamma',None),
    ('anom', '0')])

  def get_program_params(self):
    return copy.deepcopy(self._program_params)

  def get_processing_params(self):
    return copy.deepcopy(self._processing_params)

  def get_inner_shell_scaling_params(self):
    sp = copy.deepcopy(self._scaling_params)
    sp['type'] = 'innerShell'
    return sp

  def get_outer_shell_scaling_params(self):
    sp = copy.deepcopy(self._scaling_params)
    sp['type'] = 'outerShell'
    return sp

  def get_overall_scaling_params(self):
    sp = copy.deepcopy(self._scaling_params)
    sp['type'] = 'overall'
    return sp

  def get_integration_params(self):
    return copy.deepcopy(self._integration_params)

  def insert_program(self, cursor, values):
    cursor.execute('select ispyb.upsert_program_run(%s)' % ','.join(['%s'] * len(values)), values)
    rs = cursor.fetchone()
    if len(rs) > 0:
        return int(rs[0])
    return None

  def update_program(self, cursor, values):
    id = cursor.execute('select ispyb.upsert_program_run(%s)' % ','.join(['%s'] * len(values)), values)
    rs = cursor.fetchone()
    if len(rs) > 0:
        return int(rs[0])
    return None

  def put_program(self, cursor, values):
    id = None
    if values[0] is None:
        id = self.insert_program(cursor, values)
    else:
        self.update_program(cursor, values)
        id = values[0]

    if id != None:
      return int(id)
    return None

  def insert_processing(self, cursor, values):
    cursor.execute('select ispyb.upsert_processing(%s)' % ','.join(['%s'] * len(values)), values)
    rs = cursor.fetchone()
    if len(rs) > 0:
        return int(rs[0])
    return None

  def insert_scaling(self, cursor, parent_id, values1, values2, values3):
    values = [parent_id] + values1 + values2 + values3
    cursor.execute('select ispyb.insert_scaling(%s)' % ','.join(['%s'] * len(values)), values)
    rs = cursor.fetchone()
    if len(rs) > 0:
        return int(rs[0])
    return None

  def insert_integration(self, cursor, values):
    id = cursor.callfunc('select ispyb.upsert_integration(%s)' % ','.join(['%s'] * len(values)), values)
    rs = cursor.fetchone()
    if len(rs) > 0:
        return int(rs[0])
    return None

mxdatareduction = MXDataReduction()

<commit_msg>Added file ID params to the programs dict.<commit_after>#!/usr/bin/env python
# mxdatareduction.py
#
#    Copyright (C) 2014 Diamond Light Source, Karl Levik
#
# 2014-09-24
#
# Methods to store data from MX reduction pipelines
#

try:
  import cx_Oracle
except ImportError, e:
  print 'Oracle API module not found'
  raise e

import string
import logging
import time
import os
import sys
import datetime
from logging.handlers import RotatingFileHandler
from collections import OrderedDict
import copy
from ispyb_api.ExtendedOrderedDict import ExtendedOrderedDict

class MXDataReduction:
  '''MXDataReduction provides methods to store reduced MX data.'''

  def __init__(self):
    pass

  _program_params = ExtendedOrderedDict([('id',None), ('cmd_line',None), ('programs',None),
    ('status',None), ('message',None), ('starttime',None), ('endtime',None), ('environment',None),
    ('fileid1',None), ('filename1',None), ('filepath1',None), ('filetype1',None),
    ('fileid2',None), ('filename2',None), ('filepath2',None), ('filetype2',None),
    ('fileid3',None), ('filename3',None), ('filepath3',None), ('filetype3',None)])

  _processing_params = ExtendedOrderedDict([('parentid',None), ('spacegroup',None),
    ('refinedcell_a',None), ('refinedcell_b',None), ('refinedcell_c',None),
    ('refinedcell_alpha',None), ('refinedcell_beta',None), ('refinedcell_gamma',None)])

  _scaling_params = ExtendedOrderedDict([
    ('type',None), ('comments',None), ('res_lim_low', None), ('res_lim_high',None), ('r_merge',None),
	('r_meas_within_iplusi_minus',None), ('r_meas_all_iplusi_minus',None), ('r_pim_within_iplusi_minus',None), ('r_pim_all_iplusi_minus',None), ('fract_partial_bias',None), ('n_tot_obs',None),
	('n_tot_unique_obs',None), ('mean_i_sig_i',None), ('completeness',None), ('multiplicity',None), ('anom','0'), ('anom_completeness',None), ('anom_multiplicity',None),
	('cc_half',None), ('cc_anom',None)])

  _integration_params = ExtendedOrderedDict([('parentid',None), ('datacollectionid',None), ('programid',None),
    ('start_image_no',None), ('end_image_no',None), ('refined_detector_dist',None),
    ('refined_xbeam',None), ('refined_ybeam',None), ('rot_axis_x',None), ('rot_axis_y',None), ('rot_axis_z',None),
    ('beam_vec_x',None), ('beam_vec_y',None), ('beam_vec_z',None),
    ('cell_a',None), ('cell_b',None), ('cell_c',None), ('cell_alpha',None), ('cell_beta',None), ('cell_gamma',None),
    ('anom', '0')])

  def get_program_params(self):
    return copy.deepcopy(self._program_params)

  def get_processing_params(self):
    return copy.deepcopy(self._processing_params)

  def get_inner_shell_scaling_params(self):
    sp = copy.deepcopy(self._scaling_params)
    sp['type'] = 'innerShell'
    return sp

  def get_outer_shell_scaling_params(self):
    sp = copy.deepcopy(self._scaling_params)
    sp['type'] = 'outerShell'
    return sp

  def get_overall_scaling_params(self):
    sp = copy.deepcopy(self._scaling_params)
    sp['type'] = 'overall'
    return sp

  def get_integration_params(self):
    return copy.deepcopy(self._integration_params)

  def insert_program(self, cursor, values):
    cursor.execute('select ispyb.upsert_program_run(%s)' % ','.join(['%s'] * len(values)), values)
    rs = cursor.fetchone()
    if len(rs) > 0:
        return int(rs[0])
    return None

  def update_program(self, cursor, values):
    id = cursor.execute('select ispyb.upsert_program_run(%s)' % ','.join(['%s'] * len(values)), values)
    rs = cursor.fetchone()
    if len(rs) > 0:
        return int(rs[0])
    return None

  def put_program(self, cursor, values):
    id = None
    if values[0] is None:
        id = self.insert_program(cursor, values)
    else:
        self.update_program(cursor, values)
        id = values[0]

    if id != None:
      return int(id)
    return None

  def insert_processing(self, cursor, values):
    cursor.execute('select ispyb.upsert_processing(%s)' % ','.join(['%s'] * len(values)), values)
    rs = cursor.fetchone()
    if len(rs) > 0:
        return int(rs[0])
    return None

  def insert_scaling(self, cursor, parent_id, values1, values2, values3):
    values = [parent_id] + values1 + values2 + values3
    cursor.execute('select ispyb.insert_scaling(%s)' % ','.join(['%s'] * len(values)), values)
    rs = cursor.fetchone()
    if len(rs) > 0:
        return int(rs[0])
    return None

  def insert_integration(self, cursor, values):
    id = cursor.callfunc('select ispyb.upsert_integration(%s)' % ','.join(['%s'] * len(values)), values)
    rs = cursor.fetchone()
    if len(rs) > 0:
        return int(rs[0])
    return None

mxdatareduction = MXDataReduction()

<|endoftext|>


<commit_before>#
# Jasy - Web Tooling Framework
# Copyright 2013-2014 Sebastian Werner
#

import copy, random, string

import jasy.core.Console as Console
import jasy.style.parse.Node as Node
import jasy.style.Util as Util

def processExtends(tree):
    """
    Processes all requests for mixin extends
    """

    Console.info("Processing extend requests...")
    Console.indent()
    modified = __extend(tree)
    Console.debug("Processed %s selectors", modified)
    Console.outdent()

    return modified

def processMixins(tree):
    """
    Processes all mixin includes inside mixins
    """

    Console.info("Merging mixins with each other...")
    Console.indent()
    modified = __process(tree, scanMixins=True)
    Console.debug("Merged %s mixins", modified)
    Console.outdent()

    return modified

def processSelectors(tree):
    """
    Processes all mixin includes inside selectors
    """

    Console.info("Merging mixins into selectors")
    Console.indent()
    modified = __process(tree, scanMixins=False)
    Console.debug("Merged %s mixins", modified)
    Console.outdent()

    return modified

def isExtendCall(node):
    return (node.type == "call" and (not hasattr(node, "params") or len(node.params) == 0)) or (node.type == "variable" and node.parent.type == "block")

def isMixinCall(node):
    return (node.type == "call" or (node.type == "variable" and node.parent.type == "block"))

def __extend(node, scanMixins=False):
    """
    Finds extend requests for mixins aka

    - mixins calls without params
    - simple variables in a block

    For all found extend requests it detects the flattened selector and appends
    the selector section of the extendable mixin accordingly. After that it
    removes the original mixin request.
    """

    modified = 0

    for child in reversed(node):
        # Ignore all mixin declarations. Can't operate inside them.
        # For these things to work we have to wait for the include mechanics to resolve them first
        # (which actually just remove these mixin declarations though)
        if child is not None and (scanMixins or child.type != "mixin"):
            modified += __extend(child)

    if isExtendCall(node):

        name = node.name

        Console.debug("Extend request to mixin %s at: %s", name, node.line)
        Console.indent()

        mixin = __findMixin(node.parent, name)
        if not mixin:
            raise Exception("Could not find mixin %s as required by extend request at line %s" % (node.name, node.line))

        Console.debug("Found matching mixin declaration at line: %s", mixin.line)

        selector, media, supports = Util.combineSelector(node.parent, stop=mixin.parent)

        # There is no possibility to handle this in a series of CSS selectors. This is why
        # we have to use an include like approach instead of extend to correctly deal
        # with the situation. This should work well, but is not as efficient regarding
        # output file size.
        if media or supports:
            Console.warn("Extending inside a @media/@support structure behaves like including (larger result size): %s %s + %s", media, supports, ", ".join(selector))

            replacements = __resolveMixin(mixin, None)

            Console.debug("Replacing call %s at line %s with mixin from line %s" % (name, node.line, replacements.line))

            # Reverse inject all children of that block
            # at the same position as the original call
            parent = node.parent
            pos = parent.index(node)
            parent.insertAll(pos, replacements)

        else:
            Console.debug("Extending selector of mixin by: %s", ", ".join(selector))

            if hasattr(mixin, "selector"):
                # We iterate from in inverse mode, so add new selectors to the front
                mixin.selector[0:0] = selector

            else:
                mixin.selector = selector

            virtualBlock = Node.Node(type="block")
            __extendContent(mixin.rules, node, virtualBlock, mixin)

            if len(virtualBlock) > 0:
                callSelector, callMedia, callSupports = Util.combineSelector(node)

                if callSelector:
                    virtualSelector = Node.Node(type="selector")
                    virtualSelector.name = callSelector

                if callMedia:
                    virtualMedia = Node.Node(type="media")
                    virtualMedia.name = callMedia

                if callSelector:
                    virtualSelector.append(virtualBlock, "rules")
                elif callMedia:
                    virtualMedia.append(virtualBlock, "rules")

                if callMedia:
                    virtualTop = virtualMedia
                elif callSelector:
                    virtualTop = virtualSelector

                pos = mixin.parent.index(mixin)
                mixin.parent.insert(pos+1, virtualTop)

        node.parent.remove(node)
        Console.outdent()

        modified += 1

    return modified

def __process(node, scanMixins=False, active=None):
    """
    Recursively processes the given node.

    - scanMixins: Whether mixins or selectors should be processed (phase1 vs. phase2)
    - active: Whether replacements should happen
    """

    modified = 0

    if active is None:
        active = not scanMixins

    for child in reversed(node):
        if child is not None:
            if child.type == "mixin":
                if scanMixins:
                    modified += __process(child, scanMixins=scanMixins, active=True)

            else:
                # Only process non mixin childs
                modified += __process(child, scanMixins=scanMixins, active=active)

    if active and isMixinCall(node) and not isExtendCall(node):
        name = node.name

        mixin = __findMixin(node.parent, name)
        replacements = __resolveMixin(mixin, getattr(node, "params", None))

        Console.debug("Replacing call %s at line %s with mixin from line %s" % (name, node.line, replacements.line))

        __injectContent(replacements, node)

        # Reverse inject all children of that block
        # at the same position as the original call
        parent = node.parent
        pos = parent.index(node)
        for child in reversed(replacements):
            parent.insert(pos, child)

        # Finally remove original node
        parent.remove(node)

        modified += 1

    return modified

def __injectContent(node, call):
    """
    Inserts content section of call into prepared content area of mixin clone
    """

    for child in reversed(node):
        if child:
            __injectContent(child, call)

    if node.type == "content":
        if hasattr(call, "rules"):
            Console.debug("Inserting content section from call into mixin clone")
            node.parent.insertAllReplace(node, copy.deepcopy(call.rules))
        else:
            Console.debug("Removing unused content section from mixin clone")
            node.parent.remove(node)

def __extendContent(node, call, targetBlock, stopCombineAt):
    """
    Builds up a list of selector/@media/@support to insert after
    the extend to produce the @content sections on the intended selectors.
    """

    for child in reversed(node):
        if child:
            __extendContent(child, call, targetBlock, stopCombineAt)

    if node.type == "content" and hasattr(call, "rules"):
        # Extends support @content as well. In this case we produce a new selector
        # which matches the position of the content section and append it after
        # the original extended mixin on return

        Console.debug("Inserting content section into new virtual selector")

        selector, media, supports = Util.combineSelector(node, stop=stopCombineAt)

        selectorNode = Node.Node(type="selector")
        selectorNode.name = selector

        selectorNode.append(copy.deepcopy(call.rules), "rules")

        # Support media queries, too
        if media:
            mediaNode = Node.Node(type="media")
            mediaNode.name = media
            mediaNode.append(selectorNode)
            targetBlock.append(mediaNode)

        else:
            targetBlock.append(selectorNode)

def __findMixin(node, name):
    """
    Reverse scanning loop-engine for figuring out first position of given mixin
    """

    for child in reversed(node):
        if child is not None:
            # Sheets are just fragments with a special origin,
            # but otherwise the content is valid on the same level
            # as other siblings of the sheet.
            if child.type == "sheet":
                for subChild in reversed(child):
                    if subChild is not None:
                        if subChild.type == "mixin" and subChild.name == name:
                            return subChild

            elif child.type == "mixin" and child.name == name:
                return child

    parent = getattr(node, "parent", None)
    if parent:
        return __findMixin(parent, name)
    else:
        return None

def __resolveMixin(mixin, params):
    """
    Returns a clone of the given mixin and applies optional parameters to it
    """

    # Generate random prefix for variables and parameters
    chars = string.ascii_letters + string.digits
    prefix = ''.join(random.sample(chars*6, 6))

    # Data base of all local variable and parameter name mappings
    variables = {}

    # Generate full recursive clone of mixin rules
    clone = copy.deepcopy(mixin.rules)

    if hasattr(mixin, "params"):
        for pos, param in enumerate(mixin.params):
            # We have to copy over the parameter value as a local variable declaration
            paramAsDeclaration = Node.Node(type="declaration")

            if param.type == "variable":
                paramAsDeclaration.name = param.name
            elif param.type == "assign" and param[0].type == "variable":
                paramAsDeclaration.name = param[0].name
            else:
                raise Exception("Unsupported param structure for mixin resolver at line %s! Expected type variable or assignment and got: %s!" % (mixin.line, param.type));

            # Copy over actual param value
            if len(params) > pos:
                paramAsDeclaration.append(copy.deepcopy(params[pos]), "initializer")
            elif param.type == "assign" and param[0].type == "variable":
                paramAsDeclaration.append(copy.deepcopy(param[1]), "initializer")

            clone.insert(0, paramAsDeclaration)

    __renameRecurser(clone, variables, prefix)

    return clone

def __renameRecurser(node, variables, prefix):
    """
    Resursive engine to rename all local variables to prefixed ones for protecting
    the scope of the mixin vs. the place it is injected to.
    """

    for child in node:
        if child is not None:
            __renameRecurser(child, variables, prefix)

    # Set variable
    if node.type == "declaration":
        if not node.name in variables:
            variables[node.name] = "%s-%s" % (prefix, node.name)
            Console.debug("Renaming variable: %s to %s at line %s", node.name, variables[node.name], node.line)

        node.name = variables[node.name]

    # Access variable
    elif node.type == "variable" and node.name in variables:
        node.name = variables[node.name]

<commit_msg>Improved @supports support<commit_after>#
# Jasy - Web Tooling Framework
# Copyright 2013-2014 Sebastian Werner
#

import copy, random, string

import jasy.core.Console as Console
import jasy.style.parse.Node as Node
import jasy.style.Util as Util

def processExtends(tree):
    """
    Processes all requests for mixin extends
    """

    Console.info("Processing extend requests...")
    Console.indent()
    modified = __extend(tree)
    Console.debug("Processed %s selectors", modified)
    Console.outdent()

    return modified

def processMixins(tree):
    """
    Processes all mixin includes inside mixins
    """

    Console.info("Merging mixins with each other...")
    Console.indent()
    modified = __process(tree, scanMixins=True)
    Console.debug("Merged %s mixins", modified)
    Console.outdent()

    return modified

def processSelectors(tree):
    """
    Processes all mixin includes inside selectors
    """

    Console.info("Merging mixins into selectors")
    Console.indent()
    modified = __process(tree, scanMixins=False)
    Console.debug("Merged %s mixins", modified)
    Console.outdent()

    return modified

def isExtendCall(node):
    return (node.type == "call" and (not hasattr(node, "params") or len(node.params) == 0)) or (node.type == "variable" and node.parent.type == "block")

def isMixinCall(node):
    return (node.type == "call" or (node.type == "variable" and node.parent.type == "block"))

def __extend(node, scanMixins=False):
    """
    Finds extend requests for mixins aka

    - mixins calls without params
    - simple variables in a block

    For all found extend requests it detects the flattened selector and appends
    the selector section of the extendable mixin accordingly. After that it
    removes the original mixin request.
    """

    modified = 0

    for child in reversed(node):
        # Ignore all mixin declarations. Can't operate inside them.
        # For these things to work we have to wait for the include mechanics to resolve them first
        # (which actually just remove these mixin declarations though)
        if child is not None and (scanMixins or child.type != "mixin"):
            modified += __extend(child)

    if isExtendCall(node):

        name = node.name

        Console.debug("Extend request to mixin %s at: %s", name, node.line)
        Console.indent()

        mixin = __findMixin(node.parent, name)
        if not mixin:
            raise Exception("Could not find mixin %s as required by extend request at line %s" % (node.name, node.line))

        Console.debug("Found matching mixin declaration at line: %s", mixin.line)

        selector, media, supports = Util.combineSelector(node.parent, stop=mixin.parent)

        # There is no possibility to handle this in a series of CSS selectors. This is why
        # we have to use an include like approach instead of extend to correctly deal
        # with the situation. This should work well, but is not as efficient regarding
        # output file size.
        if media or supports:
            Console.warn("Extending inside a @media/@support structure behaves like including (larger result size): %s %s + %s", media, supports, ", ".join(selector))

            replacements = __resolveMixin(mixin, None)

            Console.debug("Replacing call %s at line %s with mixin from line %s" % (name, node.line, replacements.line))

            # Reverse inject all children of that block
            # at the same position as the original call
            parent = node.parent
            pos = parent.index(node)
            parent.insertAll(pos, replacements)

        else:
            Console.debug("Extending selector of mixin by: %s", ", ".join(selector))

            if hasattr(mixin, "selector"):
                # We iterate from in inverse mode, so add new selectors to the front
                mixin.selector[0:0] = selector

            else:
                mixin.selector = selector

            virtualBlock = Node.Node(type="block")
            __extendContent(mixin.rules, node, virtualBlock, mixin)

            if len(virtualBlock) > 0:
                callSelector, callMedia, callSupports = Util.combineSelector(node)

                if callSelector:
                    virtualSelector = Node.Node(type="selector")
                    virtualSelector.name = callSelector

                if callMedia:
                    virtualMedia = Node.Node(type="media")
                    virtualMedia.name = callMedia

                if callSelector:
                    virtualSelector.append(virtualBlock, "rules")
                elif callMedia:
                    virtualMedia.append(virtualBlock, "rules")

                if callMedia:
                    virtualTop = virtualMedia
                elif callSelector:
                    virtualTop = virtualSelector

                pos = mixin.parent.index(mixin)
                mixin.parent.insert(pos+1, virtualTop)

        node.parent.remove(node)
        Console.outdent()

        modified += 1

    return modified

def __process(node, scanMixins=False, active=None):
    """
    Recursively processes the given node.

    - scanMixins: Whether mixins or selectors should be processed (phase1 vs. phase2)
    - active: Whether replacements should happen
    """

    modified = 0

    if active is None:
        active = not scanMixins

    for child in reversed(node):
        if child is not None:
            if child.type == "mixin":
                if scanMixins:
                    modified += __process(child, scanMixins=scanMixins, active=True)

            else:
                # Only process non mixin childs
                modified += __process(child, scanMixins=scanMixins, active=active)

    if active and isMixinCall(node) and not isExtendCall(node):
        name = node.name

        mixin = __findMixin(node.parent, name)
        replacements = __resolveMixin(mixin, getattr(node, "params", None))

        Console.debug("Replacing call %s at line %s with mixin from line %s" % (name, node.line, replacements.line))

        __injectContent(replacements, node)

        # Reverse inject all children of that block
        # at the same position as the original call
        parent = node.parent
        pos = parent.index(node)
        for child in reversed(replacements):
            parent.insert(pos, child)

        # Finally remove original node
        parent.remove(node)

        modified += 1

    return modified

def __injectContent(node, call):
    """
    Inserts content section of call into prepared content area of mixin clone
    """

    for child in reversed(node):
        if child:
            __injectContent(child, call)

    if node.type == "content":
        if hasattr(call, "rules"):
            Console.debug("Inserting content section from call into mixin clone")
            node.parent.insertAllReplace(node, copy.deepcopy(call.rules))
        else:
            Console.debug("Removing unused content section from mixin clone")
            node.parent.remove(node)

def __extendContent(node, call, targetBlock, stopCombineAt):
    """
    Builds up a list of selector/@media/@support to insert after
    the extend to produce the @content sections on the intended selectors.
    """

    for child in reversed(node):
        if child:
            __extendContent(child, call, targetBlock, stopCombineAt)

    if node.type == "content" and hasattr(call, "rules"):
        # Extends support @content as well. In this case we produce a new selector
        # which matches the position of the content section and append it after
        # the original extended mixin on return

        Console.debug("Inserting content section into new virtual selector")

        selector, media, supports = Util.combineSelector(node, stop=stopCombineAt)

        selectorNode = Node.Node(type="selector")
        selectorNode.name = selector

        selectorNode.append(copy.deepcopy(call.rules), "rules")

        # Support @supports
        if supports:
            supportsNode = Node.Node(type="supports")
            supportsNode.name = supports

            supportsBlock = Node.Node(type="block")
            supportsBlock.append(selectorNode)
            supportsNode.append(supportsBlock, "rules")

            # Update reference
            selectorNode = supportsNode

        # Support @media
        if media:
            mediaNode = Node.Node(type="media")
            mediaNode.name = media

            mediaBlock = Node.Node(type="block")
            mediaBlock.append(selectorNode)
            mediaNode.append(mediaBlock, "rules")

            # Update reference
            selectorNode = mediaNode

        # Insert selectorNode (or media node or supports node when updated)
        # If all kinds are used we should have the following structure:
        # @media->@supports->selector
        targetBlock.append(selectorNode)

def __findMixin(node, name):
    """
    Reverse scanning loop-engine for figuring out first position of given mixin
    """

    for child in reversed(node):
        if child is not None:
            # Sheets are just fragments with a special origin,
            # but otherwise the content is valid on the same level
            # as other siblings of the sheet.
            if child.type == "sheet":
                for subChild in reversed(child):
                    if subChild is not None:
                        if subChild.type == "mixin" and subChild.name == name:
                            return subChild

            elif child.type == "mixin" and child.name == name:
                return child

    parent = getattr(node, "parent", None)
    if parent:
        return __findMixin(parent, name)
    else:
        return None

def __resolveMixin(mixin, params):
    """
    Returns a clone of the given mixin and applies optional parameters to it
    """

    # Generate random prefix for variables and parameters
    chars = string.ascii_letters + string.digits
    prefix = ''.join(random.sample(chars*6, 6))

    # Data base of all local variable and parameter name mappings
    variables = {}

    # Generate full recursive clone of mixin rules
    clone = copy.deepcopy(mixin.rules)

    if hasattr(mixin, "params"):
        for pos, param in enumerate(mixin.params):
            # We have to copy over the parameter value as a local variable declaration
            paramAsDeclaration = Node.Node(type="declaration")

            if param.type == "variable":
                paramAsDeclaration.name = param.name
            elif param.type == "assign" and param[0].type == "variable":
                paramAsDeclaration.name = param[0].name
            else:
                raise Exception("Unsupported param structure for mixin resolver at line %s! Expected type variable or assignment and got: %s!" % (mixin.line, param.type));

            # Copy over actual param value
            if len(params) > pos:
                paramAsDeclaration.append(copy.deepcopy(params[pos]), "initializer")
            elif param.type == "assign" and param[0].type == "variable":
                paramAsDeclaration.append(copy.deepcopy(param[1]), "initializer")

            clone.insert(0, paramAsDeclaration)

    __renameRecurser(clone, variables, prefix)

    return clone

def __renameRecurser(node, variables, prefix):
    """
    Resursive engine to rename all local variables to prefixed ones for protecting
    the scope of the mixin vs. the place it is injected to.
    """

    for child in node:
        if child is not None:
            __renameRecurser(child, variables, prefix)

    # Set variable
    if node.type == "declaration":
        if not node.name in variables:
            variables[node.name] = "%s-%s" % (prefix, node.name)
            Console.debug("Renaming variable: %s to %s at line %s", node.name, variables[node.name], node.line)

        node.name = variables[node.name]

    # Access variable
    elif node.type == "variable" and node.name in variables:
        node.name = variables[node.name]

<|endoftext|>


<commit_before>import logging
from . import log
import traceback
from .exception import StopUser, RescheduleTask, RescheduleTaskImmediately, InterruptTaskSet

class EventHook:
    """
    Simple event class used to provide hooks for different types of events in Locust.

    Here's how to use the EventHook class::

        my_event = EventHook()
        def on_my_event(a, b, **kw):
            print("Event was fired with arguments: %s, %s" % (a, b))
        my_event.add_listener(on_my_event)
        my_event.fire(a="foo", b="bar")

    If reverse is True, then the handlers will run in the reverse order
    that they were inserted
    """

    def __init__(self):
        self._handlers = []

    def add_listener(self, handler):
        self._handlers.append(handler)
        return handler

    def remove_listener(self, handler):
        self._handlers.remove(handler)

    def fire(self, *, reverse=False, **kwargs):
        if reverse:
            handlers = reversed(self._handlers)
        else:
            handlers = self._handlers
        for handler in handlers:
            try:
                handler(**kwargs)
            except (StopUser, RescheduleTask, RescheduleTaskImmediately, InterruptTaskSet):
                # These exceptions could be thrown by, for example, a request_failure handler,
                # in which case they are entirely appropriate and should not be caught
                raise
            except Exception:
                logging.error("Uncaught exception in event handler: \n%s", traceback.format_exc())
                log.unhandled_greenlet_exception = True

class Events:
    request_success = EventHook
    """
    Fired when a request is completed successfully. This event is typically used to report requests
    when writing custom clients for locust.

    Event arguments:

    :param request_type: Request type method used
    :param name: Path to the URL that was called (or override name if it was used in the call to the client)
    :param response_time: Response time in milliseconds
    :param response_length: Content-length of the response
    """

    request_failure = EventHook
    """
    Fired when a request fails. This event is typically used to report failed requests when writing
    custom clients for locust.

    Event arguments:

    :param request_type: Request type method used
    :param name: Path to the URL that was called (or override name if it was used in the call to the client)
    :param response_time: Time in milliseconds until exception was thrown
    :param response_length: Content-length of the response
    :param exception: Exception instance that was thrown
    """

    user_error = EventHook
    """
    Fired when an exception occurs inside the execution of a User class.

    Event arguments:

    :param user_instance: User class instance where the exception occurred
    :param exception: Exception that was thrown
    :param tb: Traceback object (from e.__traceback__)
    """

    report_to_master = EventHook
    """
    Used when Locust is running in --worker mode. It can be used to attach
    data to the dicts that are regularly sent to the master. It's fired regularly when a report
    is to be sent to the master server.

    Note that the keys "stats" and "errors" are used by Locust and shouldn't be overridden.

    Event arguments:

    :param client_id: The client id of the running locust process.
    :param data: Data dict that can be modified in order to attach data that should be sent to the master.
    """

    worker_report = EventHook
    """
    Used when Locust is running in --master mode and is fired when the master
    server receives a report from a Locust worker server.

    This event can be used to aggregate data from the locust worker servers.

    Event arguments:

    :param client_id: Client id of the reporting worker
    :param data: Data dict with the data from the worker node
    """

    spawning_complete = EventHook
    """
    Fired when all simulated users has been spawned.

    Event arguments:

    :param user_count: Number of users that were spawned
    """

    quitting = EventHook
    """
    Fired when the locust process is exiting

    Event arguments:

    :param environment: Environment instance
    """

    init = EventHook
    """
    Fired when Locust is started, once the Environment instance and locust runner instance
    have been created. This hook can be used by end-users' code to run code that requires access to
    the Environment. For example to register listeners to request_success, request_failure
    or other events.

    Event arguments:

    :param environment: Environment instance
    """

    init_command_line_parser = EventHook
    """
    Event that can be used to add command line options to Locust

    Event arguments:

    :param parser: ArgumentParser instance
    """

    test_start = EventHook
    """
    Fired when a new load test is started. It's not fired again if the number of
    users change during a test. When running locust distributed the event is only fired
    on the master node and not on each worker node.
    """

    test_stop = EventHook
    """
    Fired when a load test is stopped. When running locust distributed the event
    is only fired on the master node and not on each worker node.
    """

    reset_stats = EventHook
    """
    Fired when the Reset Stats button is clicked in the web UI.
    """

    def __init__(self):
        for name, value in vars(type(self)).items():
            if value == EventHook:
                setattr(self, name, value())
<commit_msg>fix type hinting on Events<commit_after>import logging
from . import log
import traceback
from .exception import StopUser, RescheduleTask, RescheduleTaskImmediately, InterruptTaskSet

class EventHook:
    """
    Simple event class used to provide hooks for different types of events in Locust.

    Here's how to use the EventHook class::

        my_event = EventHook()
        def on_my_event(a, b, **kw):
            print("Event was fired with arguments: %s, %s" % (a, b))
        my_event.add_listener(on_my_event)
        my_event.fire(a="foo", b="bar")

    If reverse is True, then the handlers will run in the reverse order
    that they were inserted
    """

    def __init__(self):
        self._handlers = []

    def add_listener(self, handler):
        self._handlers.append(handler)
        return handler

    def remove_listener(self, handler):
        self._handlers.remove(handler)

    def fire(self, *, reverse=False, **kwargs):
        if reverse:
            handlers = reversed(self._handlers)
        else:
            handlers = self._handlers
        for handler in handlers:
            try:
                handler(**kwargs)
            except (StopUser, RescheduleTask, RescheduleTaskImmediately, InterruptTaskSet):
                # These exceptions could be thrown by, for example, a request_failure handler,
                # in which case they are entirely appropriate and should not be caught
                raise
            except Exception:
                logging.error("Uncaught exception in event handler: \n%s", traceback.format_exc())
                log.unhandled_greenlet_exception = True

class Events:
    request_success: EventHook
    """
    Fired when a request is completed successfully. This event is typically used to report requests
    when writing custom clients for locust.

    Event arguments:

    :param request_type: Request type method used
    :param name: Path to the URL that was called (or override name if it was used in the call to the client)
    :param response_time: Response time in milliseconds
    :param response_length: Content-length of the response
    """

    request_failure: EventHook
    """
    Fired when a request fails. This event is typically used to report failed requests when writing
    custom clients for locust.

    Event arguments:

    :param request_type: Request type method used
    :param name: Path to the URL that was called (or override name if it was used in the call to the client)
    :param response_time: Time in milliseconds until exception was thrown
    :param response_length: Content-length of the response
    :param exception: Exception instance that was thrown
    """

    user_error: EventHook
    """
    Fired when an exception occurs inside the execution of a User class.

    Event arguments:

    :param user_instance: User class instance where the exception occurred
    :param exception: Exception that was thrown
    :param tb: Traceback object (from e.__traceback__)
    """

    report_to_master: EventHook
    """
    Used when Locust is running in --worker mode. It can be used to attach
    data to the dicts that are regularly sent to the master. It's fired regularly when a report
    is to be sent to the master server.

    Note that the keys "stats" and "errors" are used by Locust and shouldn't be overridden.

    Event arguments:

    :param client_id: The client id of the running locust process.
    :param data: Data dict that can be modified in order to attach data that should be sent to the master.
    """

    worker_report: EventHook
    """
    Used when Locust is running in --master mode and is fired when the master
    server receives a report from a Locust worker server.

    This event can be used to aggregate data from the locust worker servers.

    Event arguments:

    :param client_id: Client id of the reporting worker
    :param data: Data dict with the data from the worker node
    """

    spawning_complete: EventHook
    """
    Fired when all simulated users has been spawned.

    Event arguments:

    :param user_count: Number of users that were spawned
    """

    quitting: EventHook
    """
    Fired when the locust process is exiting

    Event arguments:

    :param environment: Environment instance
    """

    init: EventHook
    """
    Fired when Locust is started, once the Environment instance and locust runner instance
    have been created. This hook can be used by end-users' code to run code that requires access to
    the Environment. For example to register listeners to request_success, request_failure
    or other events.

    Event arguments:

    :param environment: Environment instance
    """

    init_command_line_parser: EventHook
    """
    Event that can be used to add command line options to Locust

    Event arguments:

    :param parser: ArgumentParser instance
    """

    test_start: EventHook
    """
    Fired when a new load test is started. It's not fired again if the number of
    users change during a test. When running locust distributed the event is only fired
    on the master node and not on each worker node.
    """

    test_stop: EventHook
    """
    Fired when a load test is stopped. When running locust distributed the event
    is only fired on the master node and not on each worker node.
    """

    reset_stats: EventHook
    """
    Fired when the Reset Stats button is clicked in the web UI.
    """

    def __init__(self):
        for name, value in vars(type(self)).items():
            if self.__annotations__.get(name) == EventHook or value == EventHook:
                setattr(self, name, value())
<|endoftext|>


<commit_before>from rapidsms.webui.utils import render_to_response
from models import *
from datetime import datetime, timedelta
from utilities.export import export

def index(req):
    template_name="logger/index.html"
    incoming = IncomingMessage.objects.order_by('received')
    outgoing = OutgoingMessage.objects.order_by('sent')
    all = []
    [ all.append(msg) for msg in incoming ]
    [ all.append(msg) for msg in outgoing]
    # sort by date, descending
    all.sort(lambda x, y: cmp(y.date, x.date))
    return render_to_response(req, template_name, { "messages": all})

def csv_in(req, format='csv'):
    context = {}
    if req.user.is_authenticated():
        return export(IncomingMessage.objects.all())
    return export(IncomingMessage.objects.all(), ['id','text','backend','domain','received'])

def csv_out(req, format='csv'):
    context = {}
    if req.user.is_authenticated():
        return export(OutgoingMessage.objects.all())
    return export(OutgoingMessage.objects.all(), ['id','text','backend','domain','sent'])
<commit_msg>Paginate message log<commit_after>from rapidsms.webui.utils import render_to_response
from models import *
from datetime import datetime, timedelta
from utilities.export import export
from rapidsms.webui.utils import paginated

def index(req):
    template_name="logger/index.html"
    incoming = IncomingMessage.objects.order_by('received')
    outgoing = OutgoingMessage.objects.order_by('sent')
    all = []
    [ all.append(msg) for msg in incoming ]
    [ all.append(msg) for msg in outgoing]
    # sort by date, descending
    all.sort(lambda x, y: cmp(y.date, x.date))
    context = {}
    context['messages'] = paginated(req, all, per_page=50)
    return render_to_response(req, template_name, context )

def csv_in(req, format='csv'):
    context = {}
    if req.user.is_authenticated():
        return export(IncomingMessage.objects.all())
    return export(IncomingMessage.objects.all(), ['id','text','backend','domain','received'])

def csv_out(req, format='csv'):
    context = {}
    if req.user.is_authenticated():
        return export(OutgoingMessage.objects.all())
    return export(OutgoingMessage.objects.all(), ['id','text','backend','domain','sent'])
<|endoftext|>


<commit_before>import tensorflow as tf
import numpy as np
from tensorflow.contrib import rnn
os.chdir('..')
from data import Data

LEARNING_RATE = 1e-3
EPOCH = 10000
NUM_CLASSES = 5
NUM_HIDDEN = 128 # commonly chosen hyperparam
DROPOUT_RATE = 0.5

data = Data("imdb_data", "dictionary.txt", MODEL_NAME)

sentence = tf.placeholder("float", [None, EMBEDDING_SIZE, data.maxLength])
label = tf.placeholder("int32", [None, NUM_CLASSES])

weights = {
	"weight": tf.Variable(tf.random_normal([2 * NUM_HIDDEN, NUM_CLASSES]))
}

biases = {
	"biases": tf.Variable(tf.random_normal([NUM_CLASSES]))
}

<commit_msg>started movement from tf to pytorch<commit_after>import numpy as np
import torch
import torch.nn.functional as F
from torch.autograd import Variable

os.chdir('..')
from data import Data

LEARNING_RATE = 1e-3
EPOCH = 10000
NUM_CLASSES = 5
NUM_HIDDEN = 128 # commonly chosen hyperparam
DROPOUT_RATE = 0.5
EMBEDDING_SIZE = 5

class Net(nn.Module):
    def __init__(self, maxLength, batchSize, embedDim, inputSize, weights = None):
        super(Net, self).__init__()
        self.input = Variable(torch.tensor())
        self.batchSize = batchSize
        self.inputSize = inputSize
        self.embedDim = embedDim
        self.maxLength = maxLength

        self.setBLSTM()

        if not weights:
           self.setBLSTM()

        else:
            weights = weights

    def setBLSTM(self):
        self.lstm = nn.LSTM(input_size=self.inputSize,hidden_size=NUM_HIDDEN,bidirectional=True,
                        dropout=DROPOUT_RATE, bias=True)

        <|endoftext|>


<commit_before>import sys
import os

from keras.models import load_model
from keras.utils import plot_model
from custom_objects import CustomObjects
from model_def import ModelDef
from run_config import RunConfig
import datetime

Train=1
Generate=2

class ModelUtils(object):

  iteration = 0
  config = None
  mode = Train
  model_filename = ""
  model_tag = ""
  testdata_filename = ""
  output_dir = ""
  h5_model_filename = ""
  h5_weights_filename = ""
  output_fn = ""
  output_file = None
  csv_logger_fn = ""
  csv_logger = None
  logfile_fn = ""
  logfile = None
  iteration_counter_fn = None
  model_def = None
  one_off_generate_len = None
  load_weights = None

  def __init__(self):
    self.buffered_logs = []
    self.log("====================================================")
    self.log("Started New Run at:",  datetime.datetime.now())
    self.log("PID:", os.getpid())
    self.log("====================================================")

    if len(sys.argv) < 2:
      print("training usage: lstm_c2_generation <tagname> [test data filename>] [load model filename]")
      print("for example\n lstm_c2_generation test1 test/LDC97S44-8k.c2cb")
      print("if test data filename or load model filename are excluded, the settings in config.json will be used if it exists")
      print("if load model filename is set to 'none' then the coded model definition will be used, regardless of what is set in the config.json file.\n")
      print("generator usage: lstm_c2_generation [tagname] --generate=<base filename> [--seed_index=<'random'|frame num|time in seconds>] [--generate-len=<frames>] <test data filename> <load model filename>")
      PRINT("loading weights: lstm_c2_generation [tagname] --load-weights=<path to .h5 file> <test data filename> <load model filename>")
      print("for example\n lstm_c2_generation --generate=audiofile --seed_index=60s --generate-len=500 test/LDC97S44-8k.c2cb out/realmodel/model-600.h5")
      exit()

    named_args = {}
    basic_args = []

    self.named_args = named_args
    self.basic_args = basic_args
    for i, arg in enumerate(sys.argv[1:]):

      self.log('arg', 1, arg)

      if arg[0:2] == "--":
        a = arg.split("=")
        key = a[0][2:]
        named_args[key] = a[1]
      else:
        basic_args.append(arg)

    if named_args.get('generate', None):
      self.generate_name = named_args['generate']
      self.mode = Generate
      self.model_tag = basic_args[0]
      self.log("mode: Generate")
    else:
      self.model_tag = basic_args[0]
      self.mode = Train
      self.log("mode: Train")

    if self.training_mode():
      self.output_dir="out/"+str(self.model_tag)+"/"
      self.output_fn=self.output_dir+"out-c2cb-"
      try:
        os.makedirs(self.output_dir)
      except OSError:
        print("the tag ", self.model_tag, " has been used")
        print("continuing where we left off")
    else:
      self.output_dir="out/"+str(self.model_tag)+"/"
      self.output_fn="generated/"+str(self.generate_name)

    exit()

    self.config = RunConfig(self)

    if len(basic_args) > 1:
      self.testdata_filename = basic_args[1]
      self.config.test_data_fn = self.testdata_filename
      self.log("using command line test data filename:", self.config.test_data_fn)

    if len(basic_args) > 2:
      self.model_filename = basic_args[2]
      self.log("using command line model filename:",self.model_filename)
    else:
      self.model_filename = self.config.model_filename
      self.log("using configured model_filename:",self.config.model_filename)

    if named_args.get('generate-len', None):
      self.one_off_generate_len = int(named_args['generate-len'])

    if named_args.get('load-weights', None):
      self.load_weights = named_args['load-weights']
      self.log("loading weights from a weights file:", self.load_weights)
    else:
      self.log("not loading weights from a weights file")

    self.h5_model_filename=self.output_dir+"model-"
    self.h5_weights_filename=self.output_dir+"weights-"

    if self.training_mode():
      from keras.callbacks import CSVLogger
      self.csv_logger_fn = self.output_dir + 'training.log'
      self.csv_logger = CSVLogger(self.csv_logger_fn, append=True)
      self.iteration_counter_fn = self.output_dir + "iteration_counter"
      self.gen_counter_fn = self.output_dir + "gen_counter"
    self.logfile_fn = self.output_dir + "log"
    self.logfile = open(self.logfile_fn, "a", 1)

  def setup_seed_start(self, generator):
    if self.named_args.get('seed_index', None):
      seed_index = self.named_args['seed_index']
      if seed_index == 'random':
        self.log("Setting seed start index to 'random'")
        generator.set_random_seed_start_index()
      elif seed_index.find('s') > 0:
        self.log("Setting seed start index to:", seed_index)
        seed_index = seed_index[0:-1]
        generator.set_time_seed_start_index(int(seed_index))
      else:
        self.log("Setting seed start index to:", seed_index)
        generator.set_frame_seed_start_index(int(seed_index))

  def load_model(self):

    self.log("loading model: " + self.model_filename)
    self.model_def.model = load_model(self.model_filename, custom_objects=self.custom_objects())

    if self.training_mode():
      self.log("saving config after loading model")
      self.config.model_filename = self.model_filename
      self.config.save_config()
    else:
      self.log("not saving config after loading model")
    self.log_model_summary()
    return self.model_def.model

  def save_json_model(self, update_num=0):
    model = self.model_def.model
    json_string = model.to_json()
    print("saving json model")
    n = "jmodel-"+str(update_num)+".json"
    mfile= open(self.output_dir + n, "w")
    mfile.write(json_string)
    mfile.close

  def save_h5_model(self, iteration):
    model = self.model_def.model
    fn = self.h5_model_filename+str(iteration)+".h5"
    res = model.save(fn)

    self.config.model_filename = fn
    self.write_iteration_count(iteration)
    self.config.save_config()

    #plot_model(model, to_file=self.output_dir+'vis-model-'+str(iteration)+'.png')

    return res

  def save_weights(self, iteration):
    model = self.model_def.model
    return model.save_weights(self.h5_weights_filename+str(iteration)+".h5")

  def open_output_file(self, iteration):
    if self.training_mode():
      output_fn = self.output_fn+str(iteration)
    else:
      output_fn = self.output_fn
    self.output_file = open(output_fn, "wb")
    return output_fn

  def after_iteration(self, iteration):
    #self.write_iteration_count(self, iteration)
    return

  def write_iteration_count(self, iteration):
    self.config.start_iteration = iteration
    with open(self.iteration_counter_fn, "w") as f:
      f.write(str(iteration))

  def write_gen_count(self, iteration):
    with open(self.gen_counter_fn, "w") as f:
      f.write(str(iteration))

  def read_iteration_count(self):
    res = []
    if self.iteration_counter_fn and os.path.isfile(self.iteration_counter_fn):
      with open(self.iteration_counter_fn) as f:
        res = f.readlines()

    if len(res) == 1:
      i = int(res[0])
      self.iteration = i
      self.log("Continuing from a previous run at iteration: ", i)
      return i
    else:
      self.iteration = 0
      self.log("No iteration file found. Setting to 0.")
      return 0

  def log(self, *inargs):

    if self.logfile == None:
      args = []
      for a in inargs:
        args.append(str(a))
      self.buffered_logs.append(str.join(" ", args) + "\n")
      return
    elif len(self.buffered_logs) > 0:
      for s in self.buffered_logs:
        print(s)
        self.logfile.write(s)
        self.logfile.flush()
        self.buffered_logs = []

    try:
      for arg in inargs:
        self.logfile.write(str(arg)+" ")
        print(str(arg)),
      print

      self.logfile.write("\n")
      self.logfile.flush()
    except IOError:
      print("* Logging Failed *")
      for arg in inargs:
        print(str(arg)),
      print

  def signal_handler(self, signal, frame):

    self.log('Interrupt signal caught. Closing gracefully.')
    self.write_iteration_count(self.iteration)

    print("saving .h5 model file")
    self.save_h5_model(self.iteration)
    print("saving .h5 weights file")
    self.save_weights(self.iteration)
    print("exiting now")
    self.logfile.close()
    sys.exit(0)

  def custom_objects(self):
    return {"CustomObjects": CustomObjects,
      "codec2_param_error":CustomObjects.codec2_param_error,
      "codec2_param_error_td":CustomObjects.codec2_param_error}

  def test_seed_data(self, all_frames, start_index):
    self.open_output_file(0)
    seed_frame_seq = all_frames[start_index: start_index + frame_seq_len]

    for frame in seed_frame_seq:
      self.output_file.write(sample(frame))

    self.output_file.close()

  def define_or_load_model(self, frame_seq_len, framelen, num_frame_seqs):
    self.model_def = ModelDef(self, self.config)

    if len(self.model_filename) > 0 and self.model_filename != 'none' and self.model_filename != 'None':
      model = self.load_model()
      self.save_json_model()
    else:
      self.log("creating new model")
      model = self.model_def.define_model(frame_seq_len, framelen, num_frame_seqs)
      self.save_json_model()

    if self.load_weights != None:
      self.model_def.load_weights(self.load_weights, by_name=True)

    return self.model_def

  def training_mode(self):
      return self.mode == Train

  def generate_mode(self):
      return self.mode == Generate

  def setup_config(self):
    return self.config

  def log_model_summary(self):
    self.model_def.model.summary()
<commit_msg>Learning func-25-1-1 ./test/criticalpracticalreason.c2-3200 none<commit_after>import sys
import os

from keras.models import load_model
from keras.utils import plot_model
from custom_objects import CustomObjects
from model_def import ModelDef
from run_config import RunConfig
import datetime

Train=1
Generate=2

class ModelUtils(object):

  iteration = 0
  config = None
  mode = Train
  model_filename = ""
  model_tag = ""
  testdata_filename = ""
  output_dir = ""
  h5_model_filename = ""
  h5_weights_filename = ""
  output_fn = ""
  output_file = None
  csv_logger_fn = ""
  csv_logger = None
  logfile_fn = ""
  logfile = None
  iteration_counter_fn = None
  model_def = None
  one_off_generate_len = None
  load_weights = None

  def __init__(self):
    self.buffered_logs = []
    self.log("====================================================")
    self.log("Started New Run at:",  datetime.datetime.now())
    self.log("PID:", os.getpid())
    self.log("====================================================")

    if len(sys.argv) < 2:
      print("training usage: lstm_c2_generation <tagname> [test data filename>] [load model filename]")
      print("for example\n lstm_c2_generation test1 test/LDC97S44-8k.c2cb")
      print("if test data filename or load model filename are excluded, the settings in config.json will be used if it exists")
      print("if load model filename is set to 'none' then the coded model definition will be used, regardless of what is set in the config.json file.\n")
      print("generator usage: lstm_c2_generation [tagname] --generate=<base filename> [--seed_index=<'random'|frame num|time in seconds>] [--generate-len=<frames>] <test data filename> <load model filename>")
      PRINT("loading weights: lstm_c2_generation [tagname] --load-weights=<path to .h5 file> <test data filename> <load model filename>")
      print("for example\n lstm_c2_generation --generate=audiofile --seed_index=60s --generate-len=500 test/LDC97S44-8k.c2cb out/realmodel/model-600.h5")
      exit()

    named_args = {}
    basic_args = []

    self.named_args = named_args
    self.basic_args = basic_args
    for i, arg in enumerate(sys.argv[1:]):

      self.log('arg', 1, arg)
      print('arg', 1, arg)
      if arg[0:2] == "--":
        a = arg.split("=")
        key = a[0][2:]
        named_args[key] = a[1]
      else:
        basic_args.append(arg)

    if named_args.get('generate', None):
      self.generate_name = named_args['generate']
      self.mode = Generate
      self.model_tag = basic_args[0]
      self.log("mode: Generate")
    else:
      self.model_tag = basic_args[0]
      self.mode = Train
      self.log("mode: Train")

    if self.training_mode():
      self.output_dir="out/"+str(self.model_tag)+"/"
      self.output_fn=self.output_dir+"out-c2cb-"
      try:
        os.makedirs(self.output_dir)
      except OSError:
        print("the tag ", self.model_tag, " has been used")
        print("continuing where we left off")
    else:
      self.output_dir="out/"+str(self.model_tag)+"/"
      self.output_fn="generated/"+str(self.generate_name)

    exit()

    self.config = RunConfig(self)

    if len(basic_args) > 1:
      self.testdata_filename = basic_args[1]
      self.config.test_data_fn = self.testdata_filename
      self.log("using command line test data filename:", self.config.test_data_fn)

    if len(basic_args) > 2:
      self.model_filename = basic_args[2]
      self.log("using command line model filename:",self.model_filename)
    else:
      self.model_filename = self.config.model_filename
      self.log("using configured model_filename:",self.config.model_filename)

    if named_args.get('generate-len', None):
      self.one_off_generate_len = int(named_args['generate-len'])

    if named_args.get('load-weights', None):
      self.load_weights = named_args['load-weights']
      self.log("loading weights from a weights file:", self.load_weights)
    else:
      self.log("not loading weights from a weights file")

    self.h5_model_filename=self.output_dir+"model-"
    self.h5_weights_filename=self.output_dir+"weights-"

    if self.training_mode():
      from keras.callbacks import CSVLogger
      self.csv_logger_fn = self.output_dir + 'training.log'
      self.csv_logger = CSVLogger(self.csv_logger_fn, append=True)
      self.iteration_counter_fn = self.output_dir + "iteration_counter"
      self.gen_counter_fn = self.output_dir + "gen_counter"
    self.logfile_fn = self.output_dir + "log"
    self.logfile = open(self.logfile_fn, "a", 1)

  def setup_seed_start(self, generator):
    if self.named_args.get('seed_index', None):
      seed_index = self.named_args['seed_index']
      if seed_index == 'random':
        self.log("Setting seed start index to 'random'")
        generator.set_random_seed_start_index()
      elif seed_index.find('s') > 0:
        self.log("Setting seed start index to:", seed_index)
        seed_index = seed_index[0:-1]
        generator.set_time_seed_start_index(int(seed_index))
      else:
        self.log("Setting seed start index to:", seed_index)
        generator.set_frame_seed_start_index(int(seed_index))

  def load_model(self):

    self.log("loading model: " + self.model_filename)
    self.model_def.model = load_model(self.model_filename, custom_objects=self.custom_objects())

    if self.training_mode():
      self.log("saving config after loading model")
      self.config.model_filename = self.model_filename
      self.config.save_config()
    else:
      self.log("not saving config after loading model")
    self.log_model_summary()
    return self.model_def.model

  def save_json_model(self, update_num=0):
    model = self.model_def.model
    json_string = model.to_json()
    print("saving json model")
    n = "jmodel-"+str(update_num)+".json"
    mfile= open(self.output_dir + n, "w")
    mfile.write(json_string)
    mfile.close

  def save_h5_model(self, iteration):
    model = self.model_def.model
    fn = self.h5_model_filename+str(iteration)+".h5"
    res = model.save(fn)

    self.config.model_filename = fn
    self.write_iteration_count(iteration)
    self.config.save_config()

    #plot_model(model, to_file=self.output_dir+'vis-model-'+str(iteration)+'.png')

    return res

  def save_weights(self, iteration):
    model = self.model_def.model
    return model.save_weights(self.h5_weights_filename+str(iteration)+".h5")

  def open_output_file(self, iteration):
    if self.training_mode():
      output_fn = self.output_fn+str(iteration)
    else:
      output_fn = self.output_fn
    self.output_file = open(output_fn, "wb")
    return output_fn

  def after_iteration(self, iteration):
    #self.write_iteration_count(self, iteration)
    return

  def write_iteration_count(self, iteration):
    self.config.start_iteration = iteration
    with open(self.iteration_counter_fn, "w") as f:
      f.write(str(iteration))

  def write_gen_count(self, iteration):
    with open(self.gen_counter_fn, "w") as f:
      f.write(str(iteration))

  def read_iteration_count(self):
    res = []
    if self.iteration_counter_fn and os.path.isfile(self.iteration_counter_fn):
      with open(self.iteration_counter_fn) as f:
        res = f.readlines()

    if len(res) == 1:
      i = int(res[0])
      self.iteration = i
      self.log("Continuing from a previous run at iteration: ", i)
      return i
    else:
      self.iteration = 0
      self.log("No iteration file found. Setting to 0.")
      return 0

  def log(self, *inargs):

    if self.logfile == None:
      args = []
      for a in inargs:
        args.append(str(a))
      self.buffered_logs.append(str.join(" ", args) + "\n")
      return
    elif len(self.buffered_logs) > 0:
      for s in self.buffered_logs:
        print(s)
        self.logfile.write(s)
        self.logfile.flush()
        self.buffered_logs = []

    try:
      for arg in inargs:
        self.logfile.write(str(arg)+" ")
        print(str(arg)),
      print

      self.logfile.write("\n")
      self.logfile.flush()
    except IOError:
      print("* Logging Failed *")
      for arg in inargs:
        print(str(arg)),
      print

  def signal_handler(self, signal, frame):

    self.log('Interrupt signal caught. Closing gracefully.')
    self.write_iteration_count(self.iteration)

    print("saving .h5 model file")
    self.save_h5_model(self.iteration)
    print("saving .h5 weights file")
    self.save_weights(self.iteration)
    print("exiting now")
    self.logfile.close()
    sys.exit(0)

  def custom_objects(self):
    return {"CustomObjects": CustomObjects,
      "codec2_param_error":CustomObjects.codec2_param_error,
      "codec2_param_error_td":CustomObjects.codec2_param_error}

  def test_seed_data(self, all_frames, start_index):
    self.open_output_file(0)
    seed_frame_seq = all_frames[start_index: start_index + frame_seq_len]

    for frame in seed_frame_seq:
      self.output_file.write(sample(frame))

    self.output_file.close()

  def define_or_load_model(self, frame_seq_len, framelen, num_frame_seqs):
    self.model_def = ModelDef(self, self.config)

    if len(self.model_filename) > 0 and self.model_filename != 'none' and self.model_filename != 'None':
      model = self.load_model()
      self.save_json_model()
    else:
      self.log("creating new model")
      model = self.model_def.define_model(frame_seq_len, framelen, num_frame_seqs)
      self.save_json_model()

    if self.load_weights != None:
      self.model_def.load_weights(self.load_weights, by_name=True)

    return self.model_def

  def training_mode(self):
      return self.mode == Train

  def generate_mode(self):
      return self.mode == Generate

  def setup_config(self):
    return self.config

  def log_model_summary(self):
    self.model_def.model.summary()
<|endoftext|>