<commit_before>#!/usr/bin/env python
#
#   Requirements:
#   numpy, scipy.optimize, matplotlib, multiprocessing
#
#   Python v2.5 or later (not compatible with Python 3)
#
#   Version History:
#
#   v0.1 2013-05-21: First more or less fully functional version
#   v0.2 2013-06-16: Clean up and restructuring of the code

from __future__ import division
import numpy as np
from numpy import sqrt, array, sum
import matplotlib.pylab as plt
from matplotlib.mlab import normpdf
import matplotlib.gridspec as gs
from scipy.optimize import leastsq
from sys import stderr
import sys
from traceback import print_exc

class lmfit(object):
    """
    Class providing access to Scipy's leastsq function for function fitting
    employing the Levenberg-Marqurdt algorithm.

    Resulting fitting parameters can be accessed via self.pvec or individually
    via self.pvec['name of parameter'].
    """

    # CONSTRUCTOR

    def __init__(self, func, xdata, ydata, p0, yerror=None, lm_options={},\
                     verbose=True, plot=False, plot_options={}):
        """
        Constructor of class lmfit.
        Parameters:
        func: 		  Pointer on the testfunction used to fit the data with
        xdata, ydata: Lists or 1D-Arrays of the same length containing the \\
        	data points
        yerror:  	  List or 1D-Array of the same length as ydata containing\\
        	weights to the individual ydata points. None (default) weights \\
        	every y value equally
        p0:			  Dictionary of the parameters in func to optimize
        lmoptions:	  Options passed to scipy.optimize.leastsq function, cf. \\
        	Scipy manual for available options
        verbose:	  Prints a full report on every fit (default: True)
        plot:		  Generates Plots to analyze the fit
        """
        # Check input
        self.__reclength = len(xdata)
        if self.__reclength != len(ydata):
            stderr.write('ERROR: Inconsistent number of data points in xdata and ydata!\n')
            return

        if yerror == None:
            self.__ToMinimize = self.__Residuals
        elif self.__reclength == len(yerror):
            self.__ToMinimize = self.__WeightedResiduals
        else:
            stderr.write('ERROR: Inconsistent number of data points in data and yerror!\n')
            return

        self.__x = xdata
        self.__y = ydata
        self.__yerror = yerror
        self.__ndf = self.__reclength - len(p0) - 1
        self.__func = func
        params, self.__pNames = self.__getParams(p0)
        self.__p0 = dict(zip(self.__pNames, params))
        self.fit(params, lm_options=lm_options, verbose=verbose, plot=plot,\
                     plot_options = plot_options)

    # PRIVATE METHODS

    def __getParams(self, p0):
        VarNames = self.__getVarNames(self.__func)
        i = 1
        if VarNames[0] == 'self':
            i += 1
        pNames = VarNames[i:]
        if type(p0) == type({}):
            p0List = self.__pDictToList(p0, pNames)
            return p0List, pNames
        else:
            return p0, pNames

    def __getVarNames(self, func):
        return func.func_code.co_varnames

    def __pDictToList(self, pdict, PNames):
        PList = []
        for PName in PNames:
            PList.append(pdict[PName])
        return PList

    def __ToMinimize(self, params):
        pass

    def __Residuals(self, params):
        return(self.__y - self.__func(self.__x,  *params))

    def __WeightedResiduals(self, params):
        return(self.__y - self.__func(self.__x, *params)/self.__yerror)

    # SPECIAL PYTHON METHODS

    def __call__(self, x):
    	"""
    	Evaluates the test function at x with the current set of parameters
        """
        return self.__func(x, *self.__pfinal)

    # PUBLIC METHODS

    @property
    def P(self):
        return self.__pfinalDict

    @property
    def StdDev(self):
        return self.__StdDev

    def fit(self, p0, lm_options={}, verbose=True, plot=False, plot_options={}):
        """
  	Carries out the least squares optimization.
        """
        params, self.__pNames = self.__getParams(p0)
        self.__p0 = dict(zip(self.__pNames, params))
        np.seterr(divide='raise')
        try:
            self.__func(self.__x, *params)
        except FloatingPointError as FPE:
            stderr.write('\nERROR: Testfunction could not be evaluated using the\
 given initial parameters!\n(%s)\n\n' %FPE)
            print_exc(limit=2)
            return

        try:
            self.__pfinal, covx, infodict, msg, ier =\
                leastsq(func=self.__ToMinimize, x0=params, full_output=1, **lm_options)
        except:
            raise Exception("An unknown error has occured in the fitting process!")
            return

        if covx == None:
            print """
Warning: Fit did not converge properly!

Message provided by MINPACK:
%s
""" % msg
            return

        self.__pfinalDict = dict(zip(self.__pNames, self.__pfinal))
        Chi2 = sum(self(self.__x)**2)
        VarRes = Chi2 / self.__ndf
        RMSChi2 = sqrt(VarRes)
        CovMatrix = covx * VarRes
        StdDev = dict(zip(self.__pNames, [sqrt(i) for i in np.diag(CovMatrix)]))
        Residuals = self.__Residuals(self.__pfinal)
        self.__StdDev = StdDev
        self.__Res = Residuals
        self.__results = {\
            'Parameters': self.__pfinalDict, 'CovMatr': CovMatrix, 'Chi2': Chi2,\
            'VarRes': VarRes, 'RMSChi2': RMSChi2, 'StdDev': StdDev, \
            'Residuals': Residuals, 'nfev': infodict['nfev'], 'fvec': infodict['fvec'],\
            'MINPACKMsg': msg}
        if verbose:
        	self.report()
        if plot:
        	self.plot(**plot_options)

    @property
    def xdata(self):
    	return self.__x

    @property
    def ydata(self):
		return self.__y

    @property
    def CovMatrix(self):
        return self.__results['CovMatr']

    @property
    def Chi2(self):
        return self.__results['Chi2']

    @property
    def RMSChi2(self):
        return self.__results['RMSChi2']

    @property
    def Residuals(self):
        return self.__results['Residuals']

    @property
    def full_results(self):
        return self.__results

    @property
    def func(self):
        return self.__func

    def plot(self, residuals=True, acf=True, lagplot=True, histogramm=True):
    	"""
    	Creates a plot of the data and the test function using current parameters.
    	Options:
    		residuals  (bool):	Plot the residuals
    		acf        (bool):		Plot the autocorrelogramm
    		lagplot    (bool):	Show a lagplot
    		histogramm (bool): Plot histogramm of residuals
    	"""
    	# determine geometry
    	rows = 1
    	cols = 2
    	height_ratios=[2]
    	width_ratios=[1,1]
    	if residuals:
    		rows += 1
    		height_ratios.append(1)
        if lagplot:
            rows += 2
            height_ratios.append(1)
            height_ratios.append(1)
        elif histogramm or acf:
            rows += 1
            height_ratios.append(1)
    	# set geometry
    	grid = gs.GridSpec(rows, cols, width_ratios=width_ratios,\
    		 height_ratios=height_ratios)
    	# main plot
    	fig = plt.figure(figsize=(16,10), dpi=100)
    	fitplot = plt.subplot(grid[0, :])
    	fitplot.set_xlabel(r'$x$')
    	fitplot.set_ylabel(r'$y$')
    	row = 1
    	if self.__yerror != None:
            fitplot.errorbar(self.__x, self.__y, self.__yerror,\
                             fmt='o')
        else:
            fitplot.plot(self.__x, self.__y, 'o', label='Data')
        x = np.linspace(self.__x[0], self.__x[-1], 1000)
        fitplot.plot(x, self(x), 'r-', label='Fit')
        fitplot.legend(loc='best', numpoints=1)
        # additional plots
        col = 0
    	if residuals:
        	residualplot = plt.subplot(grid[row, :])
        	row += 1
        	residualplot.plot(self.__x, self.__Res, 'go')
        	residualplot.set_title(u'Plot of Residuals')
        	residualplot.set_xlabel(r'$x$')
        	residualplot.set_ylabel(r'Residuals')
    	if acf:
    		col = 0
    		acfplot = plt.subplot(grid[row, col])
    		col = 1
        	lags, corrs, line, xaxis =\
                    acfplot.acorr(self.__Res, maxlags=None)
        	acfplot.set_xlim((-(len(lags)-1)/200, (len(lags)-1)/2))
        	acfplot.set_title(u'Correlogram of Residuals')
        	acfplot.set_xlabel(u'Lag')
        	acfplot.set_ylabel(u'Autocorrelation')
        if lagplot:
        	lagplot = plt.subplot(grid[row:, col])
        	if col == 1:
        		col = 0
        	row += 1
        	lagplot.plot(self.__Res[1:], self.__Res[:-1], 'g.')
        	lagplot.set_title(u'Lag Plot of Residuals')
        	lagplot.set_xlabel(r'$Y_{i-1}$')
        	lagplot.set_ylabel(r'$Y_i$')
       	if histogramm:
        	hist = plt.subplot(grid[row, col])
        	n, bins, patches = plt.hist(self.__Res, bins=20, normed=1,\
                                                histtype='stepfilled')
        	data = array(self.__Res)
    		mu = data.mean()
    		sigma = data.std()
    		gauss = normpdf(bins,mu,sigma)
    		plt.plot(bins,gauss,'ro-')
        	hist.set_title(u'Histogramm')
        	hist.set_xlabel(r'')
        	hist.set_ylabel(r'')
        grid.tight_layout(fig)
        self.fig = fig
        plt.show(fig)

    def report(self):
    	"""
    	Prints a report about the results of the last fitting procedure
    	"""
        if self.__results == None:
            print 'No results to report!'
            return
        pstring = ''
        for item in self.__pfinalDict:
            pstring += "\n%s = %f +/- %f"\
                %(item, self.__pfinalDict[item], self.__StdDev[item])
        print """
===========================================
Report:
-------------------------------------------
Initial set of parameters:
%s
Number of degrees of freedom (ndf): %d

Results:
Number of function evaluations: %d
      Sum of residuals (Chi^2): %f
 Variance of Chi^2 (Chi^2/ndf): %f
RMS of Chi^2 (sqrt(Chi^2/ndf)): %f

Covariance Matrix:
%s

Final set of parameters: %s
===========================================
""" % (self.__p0, self.__ndf, self.__results['nfev'], self.__results['Chi2'], \
           self.__results['VarRes'], self.__results['RMSChi2'],\
           self.__results['CovMatr'], pstring)

    def bootstrap(self, n=500, plot=False):
        """
        BETA!
	Resamples the residuals and fits the data to the fitted function + resampled residuals. Then calculates the statistics for the fitting parameters.
	n: Number of resamplings
        n_cpu: Number of processes used by this method
        Returns:
            Mean: Dictionary of Mean values found for all fit parameters
            StdDev: Dictionary of Standard Deviations for all fit parameters
            outlist: A list with dictionaries of fit parameters from all fits
	"""

        outlist = [i for i in range(n)]
        print 'Bootstrapping:'
        for i in outlist:
            NewY = self(self.__x) + np.random.permutation(self.__Res)
            ToMinimize = lambda params: NewY - self.__func(self.__x, *params)
            pfinal, covx, infodict, msg, ier =\
                    leastsq(func=ToMinimize, x0=self.__pfinal, full_output=1)
            outlist[i] = dict(zip(self.__pNames, pfinal))
        Results = {}
        for varname in self.__pNames:
            Results[varname] = [outdict[varname] for outdict in outlist]
        # Calculate Mean and Standard Deviation
        Mean = {}
        StdDev = {}
        for varname in Results.keys():
            Mean[varname] = np.array(Results[varname]).mean()
            StdDev[varname] = np.array(Results[varname]).std()
        if plot: # plot fits if requested
            plt.plot(self.__x, self.__y, 'bo')
            plt.title(r'Bootstrapping Fits')
            x = np.linspace(self.__x[0], self.__x[-1], 1000)
            for finalDict in outlist:
                plt.plot(x, self.__func(x, **finalDict))
            plt.show()
        return Mean, StdDev, outlist

# TESTCODE
if __name__ == '__main__':
    from numpy import exp, linspace, array
    from numpy.random import normal
    # Signal generating function
    sig = lambda x, alpha, x0: x**3/(x0**3) * exp(-((x-x0)/alpha)**2) + normal(0,0.1,len(x))
    # Test function
    testfunc = lambda x, alpha, x0: x**3.0/(x0**3.0) * exp(-((x-x0)/alpha)**2)
    # Creating a synthetic dataset
    x = array(linspace(0, 100, 1000))
    y = sig(x=x, alpha=9.67, x0=18.47)
    # Fitting the data with the testfunction by creating an instance of
    # the lmfit class
    testfit = lmfit(testfunc, xdata=x, ydata=y, p0={'x0':20, 'alpha':10}, plot=True)
    Mean, StdDev, outlist = testfit.bootstrap(500, plot=True)
    print Mean, StdDev
<commit_msg>From now on fitfunctions are accepted which use local variables not declared in the function call<commit_after>#!/usr/bin/env python
#
#   Requirements:
#   numpy, scipy.optimize, matplotlib, multiprocessing
#
#   Python v2.5 or later (not compatible with Python 3)
#
#   Version History:
#
#   v0.1 2013-05-21: First more or less fully functional version
#   v0.2 2013-06-16: Clean up and restructuring of the code

from __future__ import division
import numpy as np
from numpy import sqrt, array, sum
import matplotlib.pylab as plt
from matplotlib.mlab import normpdf
import matplotlib.gridspec as gs
from scipy.optimize import leastsq
from sys import stderr
import sys
from traceback import print_exc

class lmfit(object):
    """
    Class providing access to Scipy's leastsq function for function fitting
    employing the Levenberg-Marqurdt algorithm.

    Resulting fitting parameters can be accessed via self.P or individually
    via self.P['name of parameter'].
    """

    # CONSTRUCTOR

    def __init__(self, func, xdata, ydata, p0, yerror=None, lm_options={},\
                     verbose=True, plot=False, plot_options={}):
        """
        Constructor of class lmfit.
        Parameters:
        func: 		  Pointer on the testfunction used to fit the data with
        xdata, ydata: Lists or 1D-Arrays of the same length containing the \\
        	data points
        yerror:  	  List or 1D-Array of the same length as ydata containing\\
        	weights to the individual ydata points. None (default) weights \\
        	every y value equally
        p0:			  Dictionary of the parameters in func to optimize
        lmoptions:	  Options passed to scipy.optimize.leastsq function, cf. \\
        	Scipy manual for available options
        verbose:	  Prints a full report on every fit (default: True)
        plot:		  Generates Plots to analyze the fit
        """
        # Check input
        self.__reclength = len(xdata)
        if self.__reclength != len(ydata):
            stderr.write('ERROR: Inconsistent number of data points in xdata and ydata!\n')
            return

        if yerror == None:
            self.__ToMinimize = self.__Residuals
        elif self.__reclength == len(yerror):
            self.__ToMinimize = self.__WeightedResiduals
        else:
            stderr.write('ERROR: Inconsistent number of data points in data and yerror!\n')
            return

        self.__x = xdata
        self.__y = ydata
        self.__yerror = yerror
        self.__ndf = self.__reclength - len(p0) - 1
        self.__func = func
        params, self.__pNames = self.__getParams(p0)
        self.__p0 = dict(zip(self.__pNames, params))
        self.fit(params, lm_options=lm_options, verbose=verbose, plot=plot,\
                     plot_options = plot_options)

    # PRIVATE METHODS

    def __getParams(self, p0):
        VarNames = self.__getVarNames(self.__func)
        i = 1
        if VarNames[0] == 'self':
            i += 1
        pNames = VarNames[i:]
        if type(p0) == type({}):
            p0List = self.__pDictToList(p0, pNames)
            return p0List, pNames
        else:
            return p0, pNames

    def __getVarNames(self, func):
        return func.func_code.co_varnames[:func.func_code.co_argcount]

    def __pDictToList(self, pdict, PNames):
        PList = []
        for PName in PNames:
            PList.append(pdict[PName])
        return PList

    def __ToMinimize(self, params):
        pass

    def __Residuals(self, params):
        return(self.__y - self.__func(self.__x,  *params))

    def __WeightedResiduals(self, params):
        return(self.__y - self.__func(self.__x, *params)/self.__yerror)

    # SPECIAL PYTHON METHODS

    def __call__(self, x):
    	"""
    	Evaluates the test function at x with the current set of parameters
        """
        return self.__func(x, *self.__pfinal)

    # PUBLIC METHODS

    @property
    def P(self):
        return self.__pfinalDict

    @property
    def StdDev(self):
        return self.__StdDev

    def fit(self, p0, lm_options={}, verbose=True, plot=False, plot_options={}):
        """
  	Carries out the least squares optimization.
        """
        params, self.__pNames = self.__getParams(p0)
        self.__p0 = dict(zip(self.__pNames, params))
        np.seterr(divide='raise')
        try:
            self.__func(self.__x, *params)
        except FloatingPointError as FPE:
            stderr.write('\nERROR: Testfunction could not be evaluated using the\
 given initial parameters!\n(%s)\n\n' %FPE)
            print_exc(limit=2)
            return

        try:
            self.__pfinal, covx, infodict, msg, ier =\
                leastsq(func=self.__ToMinimize, x0=params, full_output=1, **lm_options)
        except:
            raise Exception("An unknown error has occured in the fitting process!")
            return

        if covx == None:
            print """
Warning: Fit did not converge properly!

Message provided by MINPACK:
%s
""" % msg
            return

        self.__pfinalDict = dict(zip(self.__pNames, self.__pfinal))
        Chi2 = sum(self(self.__x)**2)
        VarRes = Chi2 / self.__ndf
        RMSChi2 = sqrt(VarRes)
        CovMatrix = covx * VarRes
        StdDev = dict(zip(self.__pNames, [sqrt(i) for i in np.diag(CovMatrix)]))
        Residuals = self.__Residuals(self.__pfinal)
        self.__StdDev = StdDev
        self.__Res = Residuals
        self.__results = {\
            'Parameters': self.__pfinalDict, 'CovMatr': CovMatrix, 'Chi2': Chi2,\
            'VarRes': VarRes, 'RMSChi2': RMSChi2, 'StdDev': StdDev, \
            'Residuals': Residuals, 'nfev': infodict['nfev'], 'fvec': infodict['fvec'],\
            'MINPACKMsg': msg}
        if verbose:
        	self.report()
        if plot:
        	self.plot(**plot_options)

    @property
    def xdata(self):
    	return self.__x

    @property
    def ydata(self):
		return self.__y

    @property
    def CovMatrix(self):
        return self.__results['CovMatr']

    @property
    def Chi2(self):
        return self.__results['Chi2']

    @property
    def RMSChi2(self):
        return self.__results['RMSChi2']

    @property
    def Residuals(self):
        return self.__results['Residuals']

    @property
    def full_results(self):
        return self.__results

    @property
    def func(self):
        return self.__func

    def plot(self, residuals=True, acf=True, lagplot=True, histogramm=True):
    	"""
    	Creates a plot of the data and the test function using current parameters.
    	Options:
    		residuals  (bool):	Plot the residuals
    		acf        (bool):		Plot the autocorrelogramm
    		lagplot    (bool):	Show a lagplot
    		histogramm (bool): Plot histogramm of residuals
    	"""
    	# determine geometry
    	rows = 1
    	cols = 2
    	height_ratios=[2]
    	width_ratios=[1,1]
    	if residuals:
    		rows += 1
    		height_ratios.append(1)
        if lagplot:
            rows += 2
            height_ratios.append(1)
            height_ratios.append(1)
        elif histogramm or acf:
            rows += 1
            height_ratios.append(1)
    	# set geometry
    	grid = gs.GridSpec(rows, cols, width_ratios=width_ratios,\
    		 height_ratios=height_ratios)
    	# main plot
    	fig = plt.figure(figsize=(16,10), dpi=100)
    	fitplot = plt.subplot(grid[0, :])
    	fitplot.set_xlabel(r'$x$')
    	fitplot.set_ylabel(r'$y$')
    	row = 1
    	if self.__yerror != None:
            fitplot.errorbar(self.__x, self.__y, self.__yerror,\
                             fmt='o')
        else:
            fitplot.plot(self.__x, self.__y, 'o', label='Data')
        x = np.linspace(self.__x[0], self.__x[-1], 1000)
        fitplot.plot(x, self(x), 'r-', label='Fit')
        fitplot.legend(loc='best', numpoints=1)
        # additional plots
        col = 0
    	if residuals:
        	residualplot = plt.subplot(grid[row, :])
        	row += 1
        	residualplot.plot(self.__x, self.__Res, 'go')
        	residualplot.set_title(u'Plot of Residuals')
        	residualplot.set_xlabel(r'$x$')
        	residualplot.set_ylabel(r'Residuals')
    	if acf:
    		col = 0
    		acfplot = plt.subplot(grid[row, col])
    		col = 1
        	lags, corrs, line, xaxis =\
                    acfplot.acorr(self.__Res, maxlags=None)
        	acfplot.set_xlim((-(len(lags)-1)/200, (len(lags)-1)/2))
        	acfplot.set_title(u'Correlogram of Residuals')
        	acfplot.set_xlabel(u'Lag')
        	acfplot.set_ylabel(u'Autocorrelation')
        if lagplot:
        	lagplot = plt.subplot(grid[row:, col])
        	if col == 1:
        		col = 0
        	row += 1
        	lagplot.plot(self.__Res[1:], self.__Res[:-1], 'g.')
        	lagplot.set_title(u'Lag Plot of Residuals')
        	lagplot.set_xlabel(r'$Y_{i-1}$')
        	lagplot.set_ylabel(r'$Y_i$')
       	if histogramm:
        	hist = plt.subplot(grid[row, col])
        	n, bins, patches = plt.hist(self.__Res, bins=20, normed=1,\
                                                histtype='stepfilled')
        	data = array(self.__Res)
    		mu = data.mean()
    		sigma = data.std()
    		gauss = normpdf(bins,mu,sigma)
    		plt.plot(bins,gauss,'ro-')
        	hist.set_title(u'Histogramm')
        	hist.set_xlabel(r'')
        	hist.set_ylabel(r'')
        grid.tight_layout(fig)
        self.fig = fig
        plt.show(fig)

    def report(self):
    	"""
    	Prints a report about the results of the last fitting procedure
    	"""
        if self.__results == None:
            print 'No results to report!'
            return
        pstring = ''
        for item in self.__pfinalDict:
            pstring += "\n%s = %f +/- %f"\
                %(item, self.__pfinalDict[item], self.__StdDev[item])
        print """
===========================================
Report:
-------------------------------------------
Initial set of parameters:
%s
Number of degrees of freedom (ndf): %d

Results:
Number of function evaluations: %d
      Sum of residuals (Chi^2): %f
 Variance of Chi^2 (Chi^2/ndf): %f
RMS of Chi^2 (sqrt(Chi^2/ndf)): %f

Covariance Matrix:
%s

Final set of parameters: %s
===========================================
""" % (self.__p0, self.__ndf, self.__results['nfev'], self.__results['Chi2'], \
           self.__results['VarRes'], self.__results['RMSChi2'],\
           self.__results['CovMatr'], pstring)

    def bootstrap(self, n=500, plot=False):
        """
        BETA!
	Resamples the residuals and fits the data to the fitted function + resampled residuals. Then calculates the statistics for the fitting parameters.
	n: Number of resamplings
        n_cpu: Number of processes used by this method
        Returns:
            Mean: Dictionary of Mean values found for all fit parameters
            StdDev: Dictionary of Standard Deviations for all fit parameters
            outlist: A list with dictionaries of fit parameters from all fits
	"""

        outlist = [i for i in range(n)]
        print 'Bootstrapping:'
        for i in outlist:
            NewY = self(self.__x) + np.random.permutation(self.__Res)
            ToMinimize = lambda params: NewY - self.__func(self.__x, *params)
            pfinal, covx, infodict, msg, ier =\
                    leastsq(func=ToMinimize, x0=self.__pfinal, full_output=1)
            outlist[i] = dict(zip(self.__pNames, pfinal))
        Results = {}
        for varname in self.__pNames:
            Results[varname] = [outdict[varname] for outdict in outlist]
        # Calculate Mean and Standard Deviation
        Mean = {}
        StdDev = {}
        for varname in Results.keys():
            Mean[varname] = np.array(Results[varname]).mean()
            StdDev[varname] = np.array(Results[varname]).std()
        if plot: # plot fits if requested
            plt.plot(self.__x, self.__y, 'bo')
            plt.title(r'Bootstrapping Fits')
            x = np.linspace(self.__x[0], self.__x[-1], 1000)
            for finalDict in outlist:
                plt.plot(x, self.__func(x, **finalDict))
            plt.show()
        return Mean, StdDev, outlist

# TESTCODE
if __name__ == '__main__':
    from numpy import exp, linspace, array
    from numpy.random import normal
    # Signal generating function
    sig = lambda x, alpha, x0: x**3/(x0**3) * exp(-((x-x0)/alpha)**2) + normal(0,0.1,len(x))
    # Test function
    testfunc = lambda x, alpha, x0: x**3.0/(x0**3.0) * exp(-((x-x0)/alpha)**2)
    # Creating a synthetic dataset
    x = array(linspace(0, 100, 1000))
    y = sig(x=x, alpha=9.67, x0=18.47)
    # Fitting the data with the testfunction by creating an instance of
    # the lmfit class
    testfit = lmfit(testfunc, xdata=x, ydata=y, p0={'x0':20, 'alpha':10}, plot=True)
    Mean, StdDev, outlist = testfit.bootstrap(500, plot=True)
    print Mean, StdDev
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Yang Gao <gaoyang.public@gmail.com>
# vim: ts=4 et

import sys
import requests
import json
import time
import datetime
import yaml
import sqlite3
import logging
from requests_oauthlib import OAuth1Session
from mako.template import Template

TRADEME_BASE_API = 'http://api.trademe.co.nz/v1/Search/%s.json'
CONSUMER_KEY     = ''
CONSUMER_SECRET  = ''
OAUTH_TOKEN_SECRET = ''
OAUTH_TOKEN      = ''

def json_encode(s):
    return json.dumps(s, separators=(',',':'))

def json_decode(s):
    return json.loads(s)

def md5(s):
    import hashlib
    m = hashlib.md5()
    m.update(s)
    return m.hexdigest()

class Trademe(object):
    trademe = requests

    def __init__(self):
        pass

    def _authenticate(self, consumer_key,
                           consumer_secret):
#                           oauth_token,
#                           oauth_secret):
        request_token_url = 'https://secure.trademe.co.nz/Oauth/RequestToken?scope=MyTradeMeRead,MyTradeMeWrite'
        authorization_base_url = 'https://secure.trademe.co.nz/Oauth/Authorize'
        access_token_url = 'https://secure.trademe.co.nz/Oauth/AccessToken'
        kw = dict()
        kw['client_secret']=consumer_secret
        #kw['resource_owner_key']=oauth_token
        #kw['resource_owner_secret']=oauth_secret
        self.trademe = OAuth1Session(consumer_key, **kw)
        self.trademe.fetch_request_token(request_token_url)
        authorization_url = self.trademe.authorization_url(authorization_base_url)
        log.info('Authorization url: %s' % authorization_url)
        redirect_response = raw_input('Paste the full redirect URL here:')
        self.trademe.parse_authorization_response(redirect_response)
        oauthxx = self.trademe.fetch_access_token(access_token_url)
        log.info('oauth token: %s' % authxx)

    def authenticate(self, consumer_key,
                           consumer_secret,
                           oauth_token,
                           oauth_secret):
        kw = dict()
        kw['client_secret'] = consumer_secret
        kw['resource_owner_key'] = oauth_token
        kw['resource_owner_secret'] = oauth_secret
        self.trademe = OAuth1Session(consumer_key, **kw)
        log.info('TradeMe authenticate successfuly.')

    def getListings(self, api_path="General",
                          feedback_func=None, **kw):

        api = TRADEME_BASE_API % api_path
        result = None
        params = dict(kw)
        try:
            resp = self.trademe.get(api, params=params, timeout=30)
            result = resp.json()
        except Exception as e:
            raise

        if not feedback_func is None:
            result = feedback_func(result)
        return result

    def getMyWatchList(self):
        api = 'https://api.trademe.co.nz/v1/MyTradeMe/Watchlist/All.json'
        try:
            resp = self.trademe.get(api)
            result = resp.json()
        except Exception as e:
            raise
        return result

def feedback_searching_result(data):
    def _collect(raw_listing):
        listing = filter(lambda x: x[0] in f_keys,
                              raw_listing.items())
        listing = dict(listing)

        # fix column format
        if listing.has_key('BuyNowPrice'):
            listing['BuyNowPrice'] = '$%s' % listing['BuyNowPrice']
        else:
            listing['BuyNowPrice'] = '-'
        if not listing.has_key('PriceDisplay'):
            listing['PriceDisplay'] = '-'
        if not listing.has_key('PictureHref'):
           listing['PictureHref'] = 'http://www.trademe.co.nz/images/NewSearchCards/LVIcons/hasPhoto_160x120.png'

        listing['ListingUrl'] = listing_url % listing['ListingId']
        which_category = filter(lambda c:
                            c['Category'] == listing['Category'],
                            data['FoundCategories'])
        try:
            listing['CategoryName'] = which_category[0]['Name']
        except IndexError as e:
            listing['CategoryName'] = ''

        return listing

    listing_url = 'http://www.trademe.co.nz/Browse/Listing.aspx?id=%s'
    f_keys = ['ListingId', 'Title', 'Category', 'PictureHref',
              'Region', 'Suburb', 'PriceDisplay', 'StartPrice',
              'BuyNowPrice']
    ret = map(_collect, data['List'])
    return ret

def getConfig(cfile):
    def _merge(data):
        config = dict()
        for item, value in data.items():
            if item.startswith('.'): continue
            config[item] = value.copy()
            if value.has_key('.include'):
                included = dict(data[value['.include']])
                config[item] = dict(included, **config[item])
                del config[item]['.include']
        return config

    config = dict()
    try:
        with open(cfile, 'rb') as f:
            sys, search = yaml.load_all(f.read())
            config['system'] = sys
            config['search'] = _merge(search)
            f.close()
    except IOError as e:
        log.error(str(e))
        sys.exit(2)
    except KeyError as e:
        log.error('Include config(%s) does not exist.' % str(e))
        sys.exit(3)
    except Exception as e:
        raise
    return config

def sendEmail(smtp, user, passwd, me, send_to,
               subject, content, cc_to=None):
    '''
    @smtp SMTP server ip or domain.formart: "smtp.gmail.com:587"
    @user SMTP login user name.
    @passwd SMTP login password.
    @me display this string at "from" when he received mail.
    @to send to who.
    @subject email subject
    @content email content
    '''
    import email
    import mimetypes
    import smtplib
    from email.MIMEMultipart import MIMEMultipart
    from email.MIMEText import MIMEText
    from email.MIMEImage import MIMEImage
    from email.Header import Header
    from email.utils import COMMASPACE,formatdate

    assert(isinstance(send_to, list))
    smtp_host, smtp_port = smtp.split(":")

    toAll = list()
    msg = MIMEText(content.encode('utf-8'), 'html', 'utf-8')
    msg['Subject'] = Header(subject, 'utf-8')
    msg['From'] = me
    msg['To'] = COMMASPACE.join(send_to)
    msg['Date'] = formatdate(localtime=True)

    [toAll.append(i) for i in send_to]

    if cc_to:
        msg['Cc'] = COMMASPACE.join(cc_to)
        [toAll.append(i) for i in cc_to]

    mailServer = smtplib.SMTP(smtp_host, smtp_port)
    mailServer.ehlo()
    mailServer.starttls()
    mailServer.ehlo()
    mailServer.login(user, passwd)
    mailServer.sendmail(user, toAll, msg.as_string())
    mailServer.close()

class ListingModel(object):
    TABLE = 'TradeMe'

    SQL_CREATE_TABLE = '''
CREATE TABLE IF NOT EXISTS %(tb)s(id INTEGER PRIMARY KEY DESC, title, price, buynow, category, url, pic, region, suburb, md5);
CREATE UNIQUE INDEX IF NOT EXISTS TradeMe_md5 ON %(tb)s(md5);
'''

    SQL_SELECT_LISTING = 'SELECT id FROM %(tb)s WHERE md5=%(md5)s;'

    def __init__(self):
        self.db = self._conn('trademe.db')
        c = self.db.cursor()
        c.executescript(self.SQL_CREATE_TABLE % dict( tb = self.TABLE))

    def _conn(self, database):
        conn = sqlite3.connect(database)
        return conn

    def _make_md5(self, listing):
        md5sum =  md5('%s-%s' % ( listing['ListingId'],
                               listing['PriceDisplay']))
        return md5sum

    def save(self, listing):
        assert(isinstance(listing, dict))
        fields = ['id', 'title', 'price', 'buynow',
                  'category', 'url', 'pic', 'region',
                  'suburb', 'md5']

        sql = "INSERT OR REPLACE INTO %s(%s) VALUES(%s);" % (self.TABLE,
                ', '.join(map(lambda x: "%s" % x, fields)),
                ', '.join(map(lambda x: "?", fields)))

        try:
            data = (
                    listing['ListingId'],
                    listing['Title'],
                    listing['PriceDisplay'],
                    listing['BuyNowPrice'],
                    listing['Category'],
                    listing['ListingUrl'],
                    listing['PictureHref'],
                    listing['Region'],
                    listing['Suburb'],
                    self._make_md5(listing),
                    )
        except KeyError as e:
            raise Exception("%s.(data: %s)" % (str(e), listing))

        c = self.db.cursor()
        c.execute(sql, data)
        self.db.commit()

    def is_exist(self, listing):
        sql = "SELECT md5 FROM %s WHERE md5=%s"
        sql = sql % (self.TABLE, self._make_md5(listing))
        c = self.db.cursor()
        ret = c.execute(sql))
        if not ret:
            return False
        return True

def main():
    log.info('Start init trademe.')
    trademe = Trademe()
    #trademe._authenticate(CONSUMER_KEY, CONSUMER_SECRET)
    #                      OAUTH_TOKEN, OAUTH_SECRET)
    #watch_list = trademe.getMyWatchList()
    log.info('Authenticate TradeMe API.')
    trademe.authenticate(CONSUMER_KEY, CONSUMER_SECRET,
                         OAUTH_TOKEN, OAUTH_TOKEN_SECRET)

    for key, value in config['search'].items():
        log.info('Geting %s listing.' % key)
        params = value
        func = feedback_searching_result
        listings = trademe.getListings(feedback_func=func, **params)
        send_row = list()
        for row in listings:
            try:
                obj_listing = ListingModel()
                log.debug('save listing: %s' % row)
                if obj_listing.is_exist(row):
                    send_row.append(row)
                else:
                    obj_listing.save(row)
            except sqlite3.IntegrityError as e:
                if not str(e) == 'column md5 is not unique':
                    raise
            finally:
                # close db
                obj_listing.db.close()

        ## template    = Template(filename='template/listing.htm')
        ## print template.render(listings=listings).encode('utf-8')
        ## sys.exit()

        if send_row:
            template    = Template(filename='template/listing.htm')
            SUBJECT = 'EltonPostman "%s"' % key
            CONTENT     = template.render(listings=send_row)
            sendEmail(SMTP, SMTP_USER, SMTP_PASS,
                       ME, SEND_TO, SUBJECT, CONTENT)
            log.info('Sending email to %s done.' % SEND_TO)
        else:
            log.info('No update Listing in this round.')

if __name__ == '__main__':
    config = getConfig('prod.yaml')
    SMTP        = config['system'].get('SMTP_HOST')
    SMTP_USER   = config['system'].get('SMTP_USER')
    SMTP_PASS   = config['system'].get('SMTP_PASS')
    ME          = config['system'].get('DISPLAY_ME')
    SEND_TO     = config['system'].get('SEND_TO')

    CONSUMER_KEY        = config['system'].get('CONSUMER_KEY')
    CONSUMER_SECRET     = config['system'].get('CONSUMER_SECRET')
    OAUTH_TOKEN         = config['system'].get('OAUTH_TOKEN')
    OAUTH_TOKEN_SECRET  = config['system'].get('OAUTH_TOKEN_SECRET')

    FORMAT = '%(levelname)s: %(message)s'
    logging.basicConfig(format=FORMAT)
    log = logging.getLogger('postman')
    log_level = getattr(logging, config['system'].get('LOG_LEVEL', 'INFO'))
    log.setLevel(log_level)

    try:
        main()
    except Exception as e:
        log.error('%s' % str(e))
        SUBJECT = '[Warning]EltonPostman crashed'
        CONTENT = 'Message: %s' % str(e)
        sendEmail(SMTP, SMTP_USER, SMTP_PASS,
                   ME, SEND_TO, SUBJECT, CONTENT)
        log.debug('Send error email done.')
<commit_msg>fix bug ).<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Yang Gao <gaoyang.public@gmail.com>
# vim: ts=4 et

import sys
import requests
import json
import time
import datetime
import yaml
import sqlite3
import logging
from requests_oauthlib import OAuth1Session
from mako.template import Template

TRADEME_BASE_API = 'http://api.trademe.co.nz/v1/Search/%s.json'
CONSUMER_KEY     = ''
CONSUMER_SECRET  = ''
OAUTH_TOKEN_SECRET = ''
OAUTH_TOKEN      = ''

def json_encode(s):
    return json.dumps(s, separators=(',',':'))

def json_decode(s):
    return json.loads(s)

def md5(s):
    import hashlib
    m = hashlib.md5()
    m.update(s)
    return m.hexdigest()

class Trademe(object):
    trademe = requests

    def __init__(self):
        pass

    def _authenticate(self, consumer_key,
                           consumer_secret):
#                           oauth_token,
#                           oauth_secret):
        request_token_url = 'https://secure.trademe.co.nz/Oauth/RequestToken?scope=MyTradeMeRead,MyTradeMeWrite'
        authorization_base_url = 'https://secure.trademe.co.nz/Oauth/Authorize'
        access_token_url = 'https://secure.trademe.co.nz/Oauth/AccessToken'
        kw = dict()
        kw['client_secret']=consumer_secret
        #kw['resource_owner_key']=oauth_token
        #kw['resource_owner_secret']=oauth_secret
        self.trademe = OAuth1Session(consumer_key, **kw)
        self.trademe.fetch_request_token(request_token_url)
        authorization_url = self.trademe.authorization_url(authorization_base_url)
        log.info('Authorization url: %s' % authorization_url)
        redirect_response = raw_input('Paste the full redirect URL here:')
        self.trademe.parse_authorization_response(redirect_response)
        oauthxx = self.trademe.fetch_access_token(access_token_url)
        log.info('oauth token: %s' % authxx)

    def authenticate(self, consumer_key,
                           consumer_secret,
                           oauth_token,
                           oauth_secret):
        kw = dict()
        kw['client_secret'] = consumer_secret
        kw['resource_owner_key'] = oauth_token
        kw['resource_owner_secret'] = oauth_secret
        self.trademe = OAuth1Session(consumer_key, **kw)
        log.info('TradeMe authenticate successfuly.')

    def getListings(self, api_path="General",
                          feedback_func=None, **kw):

        api = TRADEME_BASE_API % api_path
        result = None
        params = dict(kw)
        try:
            resp = self.trademe.get(api, params=params, timeout=30)
            result = resp.json()
        except Exception as e:
            raise

        if not feedback_func is None:
            result = feedback_func(result)
        return result

    def getMyWatchList(self):
        api = 'https://api.trademe.co.nz/v1/MyTradeMe/Watchlist/All.json'
        try:
            resp = self.trademe.get(api)
            result = resp.json()
        except Exception as e:
            raise
        return result

def feedback_searching_result(data):
    def _collect(raw_listing):
        listing = filter(lambda x: x[0] in f_keys,
                              raw_listing.items())
        listing = dict(listing)

        # fix column format
        if listing.has_key('BuyNowPrice'):
            listing['BuyNowPrice'] = '$%s' % listing['BuyNowPrice']
        else:
            listing['BuyNowPrice'] = '-'
        if not listing.has_key('PriceDisplay'):
            listing['PriceDisplay'] = '-'
        if not listing.has_key('PictureHref'):
           listing['PictureHref'] = 'http://www.trademe.co.nz/images/NewSearchCards/LVIcons/hasPhoto_160x120.png'

        listing['ListingUrl'] = listing_url % listing['ListingId']
        which_category = filter(lambda c:
                            c['Category'] == listing['Category'],
                            data['FoundCategories'])
        try:
            listing['CategoryName'] = which_category[0]['Name']
        except IndexError as e:
            listing['CategoryName'] = ''

        return listing

    listing_url = 'http://www.trademe.co.nz/Browse/Listing.aspx?id=%s'
    f_keys = ['ListingId', 'Title', 'Category', 'PictureHref',
              'Region', 'Suburb', 'PriceDisplay', 'StartPrice',
              'BuyNowPrice']
    ret = map(_collect, data['List'])
    return ret

def getConfig(cfile):
    def _merge(data):
        config = dict()
        for item, value in data.items():
            if item.startswith('.'): continue
            config[item] = value.copy()
            if value.has_key('.include'):
                included = dict(data[value['.include']])
                config[item] = dict(included, **config[item])
                del config[item]['.include']
        return config

    config = dict()
    try:
        with open(cfile, 'rb') as f:
            sys, search = yaml.load_all(f.read())
            config['system'] = sys
            config['search'] = _merge(search)
            f.close()
    except IOError as e:
        log.error(str(e))
        sys.exit(2)
    except KeyError as e:
        log.error('Include config(%s) does not exist.' % str(e))
        sys.exit(3)
    except Exception as e:
        raise
    return config

def sendEmail(smtp, user, passwd, me, send_to,
               subject, content, cc_to=None):
    '''
    @smtp SMTP server ip or domain.formart: "smtp.gmail.com:587"
    @user SMTP login user name.
    @passwd SMTP login password.
    @me display this string at "from" when he received mail.
    @to send to who.
    @subject email subject
    @content email content
    '''
    import email
    import mimetypes
    import smtplib
    from email.MIMEMultipart import MIMEMultipart
    from email.MIMEText import MIMEText
    from email.MIMEImage import MIMEImage
    from email.Header import Header
    from email.utils import COMMASPACE,formatdate

    assert(isinstance(send_to, list))
    smtp_host, smtp_port = smtp.split(":")

    toAll = list()
    msg = MIMEText(content.encode('utf-8'), 'html', 'utf-8')
    msg['Subject'] = Header(subject, 'utf-8')
    msg['From'] = me
    msg['To'] = COMMASPACE.join(send_to)
    msg['Date'] = formatdate(localtime=True)

    [toAll.append(i) for i in send_to]

    if cc_to:
        msg['Cc'] = COMMASPACE.join(cc_to)
        [toAll.append(i) for i in cc_to]

    mailServer = smtplib.SMTP(smtp_host, smtp_port)
    mailServer.ehlo()
    mailServer.starttls()
    mailServer.ehlo()
    mailServer.login(user, passwd)
    mailServer.sendmail(user, toAll, msg.as_string())
    mailServer.close()

class ListingModel(object):
    TABLE = 'TradeMe'

    SQL_CREATE_TABLE = '''
CREATE TABLE IF NOT EXISTS %(tb)s(id INTEGER PRIMARY KEY DESC, title, price, buynow, category, url, pic, region, suburb, md5);
CREATE UNIQUE INDEX IF NOT EXISTS TradeMe_md5 ON %(tb)s(md5);
'''

    SQL_SELECT_LISTING = 'SELECT id FROM %(tb)s WHERE md5=%(md5)s;'

    def __init__(self):
        self.db = self._conn('trademe.db')
        c = self.db.cursor()
        c.executescript(self.SQL_CREATE_TABLE % dict( tb = self.TABLE))

    def _conn(self, database):
        conn = sqlite3.connect(database)
        return conn

    def _make_md5(self, listing):
        md5sum =  md5('%s-%s' % ( listing['ListingId'],
                               listing['PriceDisplay']))
        return md5sum

    def save(self, listing):
        assert(isinstance(listing, dict))
        fields = ['id', 'title', 'price', 'buynow',
                  'category', 'url', 'pic', 'region',
                  'suburb', 'md5']

        sql = "INSERT OR REPLACE INTO %s(%s) VALUES(%s);" % (self.TABLE,
                ', '.join(map(lambda x: "%s" % x, fields)),
                ', '.join(map(lambda x: "?", fields)))

        try:
            data = (
                    listing['ListingId'],
                    listing['Title'],
                    listing['PriceDisplay'],
                    listing['BuyNowPrice'],
                    listing['Category'],
                    listing['ListingUrl'],
                    listing['PictureHref'],
                    listing['Region'],
                    listing['Suburb'],
                    self._make_md5(listing),
                    )
        except KeyError as e:
            raise Exception("%s.(data: %s)" % (str(e), listing))

        c = self.db.cursor()
        c.execute(sql, data)
        self.db.commit()

    def is_exist(self, listing):
        sql = "SELECT md5 FROM %s WHERE md5=%s"
        sql = sql % (self.TABLE, self._make_md5(listing))
        c = self.db.cursor()
        ret = c.execute(sql)
        if not ret:
            return False
        return True

def main():
    log.info('Start init trademe.')
    trademe = Trademe()
    #trademe._authenticate(CONSUMER_KEY, CONSUMER_SECRET)
    #                      OAUTH_TOKEN, OAUTH_SECRET)
    #watch_list = trademe.getMyWatchList()
    log.info('Authenticate TradeMe API.')
    trademe.authenticate(CONSUMER_KEY, CONSUMER_SECRET,
                         OAUTH_TOKEN, OAUTH_TOKEN_SECRET)

    for key, value in config['search'].items():
        log.info('Geting %s listing.' % key)
        params = value
        func = feedback_searching_result
        listings = trademe.getListings(feedback_func=func, **params)
        send_row = list()
        for row in listings:
            try:
                obj_listing = ListingModel()
                log.debug('save listing: %s' % row)
                if obj_listing.is_exist(row):
                    send_row.append(row)
                else:
                    obj_listing.save(row)
            except sqlite3.IntegrityError as e:
                if not str(e) == 'column md5 is not unique':
                    raise
            finally:
                # close db
                obj_listing.db.close()

        ## template    = Template(filename='template/listing.htm')
        ## print template.render(listings=listings).encode('utf-8')
        ## sys.exit()

        if send_row:
            template    = Template(filename='template/listing.htm')
            SUBJECT = 'EltonPostman "%s"' % key
            CONTENT     = template.render(listings=send_row)
            sendEmail(SMTP, SMTP_USER, SMTP_PASS,
                       ME, SEND_TO, SUBJECT, CONTENT)
            log.info('Sending email to %s done.' % SEND_TO)
        else:
            log.info('No update Listing in this round.')

if __name__ == '__main__':
    config = getConfig('prod.yaml')
    SMTP        = config['system'].get('SMTP_HOST')
    SMTP_USER   = config['system'].get('SMTP_USER')
    SMTP_PASS   = config['system'].get('SMTP_PASS')
    ME          = config['system'].get('DISPLAY_ME')
    SEND_TO     = config['system'].get('SEND_TO')

    CONSUMER_KEY        = config['system'].get('CONSUMER_KEY')
    CONSUMER_SECRET     = config['system'].get('CONSUMER_SECRET')
    OAUTH_TOKEN         = config['system'].get('OAUTH_TOKEN')
    OAUTH_TOKEN_SECRET  = config['system'].get('OAUTH_TOKEN_SECRET')

    FORMAT = '%(levelname)s: %(message)s'
    logging.basicConfig(format=FORMAT)
    log = logging.getLogger('postman')
    log_level = getattr(logging, config['system'].get('LOG_LEVEL', 'INFO'))
    log.setLevel(log_level)

    try:
        main()
    except Exception as e:
        log.error('%s' % str(e))
        SUBJECT = '[Warning]EltonPostman crashed'
        CONTENT = 'Message: %s' % str(e)
        sendEmail(SMTP, SMTP_USER, SMTP_PASS,
                   ME, SEND_TO, SUBJECT, CONTENT)
        log.debug('Send error email done.')
<|endoftext|>


<commit_before>"""
Django settings for lna project.

For more information on this file, see
https://docs.djangoproject.com/en/dev/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/dev/ref/settings/
"""
import environ

ROOT_DIR = environ.Path(__file__) - 3  # (lna/config/settings/base.py - 3 = lna/)
APPS_DIR = ROOT_DIR.path('lna')

# Load operating system environment variables and then prepare to use them
env = environ.Env()

# .env file, should load only in development environment
READ_DOT_ENV_FILE = env.bool('DJANGO_READ_DOT_ENV_FILE', default=True)

if READ_DOT_ENV_FILE:
    # Operating System Environment variables have precedence over variables defined in the .env file,
    # that is to say variables from the .env files will only be used if not defined
    # as environment variables.
    env_file = str(ROOT_DIR.path('.env'))
    print('Loading : {}'.format(env_file))
    env.read_env(env_file)
    print('The .env file has been loaded. See base.py for more information')

# APP CONFIGURATION
# ------------------------------------------------------------------------------
DJANGO_APPS = [
    # Default Django apps:
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    # Useful template tags:
    # 'django.contrib.humanize',

    # Admin
    'django.contrib.admin',
]
THIRD_PARTY_APPS = [
    'crispy_forms',  # Form layouts
    'allauth',  # registration
    'allauth.account',  # registration
    'allauth.socialaccount',  # registration
    'el_pagination', # el-pagination
    'channels',  # django channels
    'rest_framework',  #DRF
    'django_celery_beat',  # django-celery-beat extension that stores the schedule in the Django database,
    # and presents a convenient admin interface to manage periodic tasks at runtime.
    'django_celery_results',
    'netfields',
]

# Apps specific for this project go here.
LOCAL_APPS = [
    # custom users app
    'lna.users.apps.UsersConfig',
    # Your stuff: custom apps go here
    'argus',
    'net',
    'lna',
    'api',
]

# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS

# MIDDLEWARE CONFIGURATION
# ------------------------------------------------------------------------------
MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

# MIGRATIONS CONFIGURATION
# ------------------------------------------------------------------------------
MIGRATION_MODULES = {
    'sites': 'lna.contrib.sites.migrations'
}

# DEBUG
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#debug
# DEBUG = env.bool('DJANGO_DEBUG', True)
DEBUG = False
# FIXTURE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-FIXTURE_DIRS
FIXTURE_DIRS = (
    str(APPS_DIR.path('fixtures')),
)

# EMAIL CONFIGURATION
# ------------------------------------------------------------------------------
EMAIL_BACKEND = env('DJANGO_EMAIL_BACKEND', default='django.core.mail.backends.smtp.EmailBackend')

# MANAGER CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#admins
ADMINS = [
    ("""Dehu4ka""", 'dehu4ka@gmail.com'),
]

# See: https://docs.djangoproject.com/en/dev/ref/settings/#managers
MANAGERS = ADMINS

# DATABASE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
DATABASES = {
    'default': env.db('DATABASE_URL', default='postgres:///lna'),
}
DATABASES['default']['ATOMIC_REQUESTS'] = True

# GENERAL CONFIGURATION
# ------------------------------------------------------------------------------
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'Asia/Yekaterinburg'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#language-code
LANGUAGE_CODE = 'en-us'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#site-id
SITE_ID = 1

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-i18n
USE_I18N = True

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-l10n
USE_L10N = True

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-tz
USE_TZ = True

# TEMPLATE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#templates
TEMPLATES = [
    {
        # See: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-TEMPLATES-BACKEND
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        # See: https://docs.djangoproject.com/en/dev/ref/settings/#template-dirs
        'DIRS': [
            str(APPS_DIR.path('templates')),
        ],
        'OPTIONS': {
            # See: https://docs.djangoproject.com/en/dev/ref/settings/#template-debug
            'debug': DEBUG,
            # See: https://docs.djangoproject.com/en/dev/ref/settings/#template-loaders
            # https://docs.djangoproject.com/en/dev/ref/templates/api/#loader-types
            'loaders': [
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader',
            ],
            # See: https://docs.djangoproject.com/en/dev/ref/settings/#template-context-processors
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',  # It was exists | Also For EL-pagination
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
                # Your stuff: custom template context processors go here
            ],
        },
    },
]

# See: http://django-crispy-forms.readthedocs.io/en/latest/install.html#template-packs
CRISPY_TEMPLATE_PACK = 'bootstrap4'

# STATIC FILE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root
STATIC_ROOT = str(ROOT_DIR('staticfiles'))

# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
STATIC_URL = '/static/'

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
STATICFILES_DIRS = [
    str(APPS_DIR.path('static')),
]
# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-finders
STATICFILES_FINDERS = [
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
]

# MEDIA CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
MEDIA_ROOT = str(APPS_DIR('media'))

# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
MEDIA_URL = '/media/'

# URL Configuration
# ------------------------------------------------------------------------------
ROOT_URLCONF = 'config.urls'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
WSGI_APPLICATION = 'config.wsgi.application'

# PASSWORD STORAGE SETTINGS
# ------------------------------------------------------------------------------
# See https://docs.djangoproject.com/en/dev/topics/auth/passwords/#using-argon2-with-django
PASSWORD_HASHERS = [
    'django.contrib.auth.hashers.Argon2PasswordHasher',
    'django.contrib.auth.hashers.PBKDF2PasswordHasher',
    'django.contrib.auth.hashers.PBKDF2SHA1PasswordHasher',
    'django.contrib.auth.hashers.BCryptSHA256PasswordHasher',
    'django.contrib.auth.hashers.BCryptPasswordHasher',
]

# PASSWORD VALIDATION
# https://docs.djangoproject.com/en/dev/ref/settings/#auth-password-validators
# ------------------------------------------------------------------------------

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# AUTHENTICATION CONFIGURATION
# ------------------------------------------------------------------------------
AUTHENTICATION_BACKENDS = [
    'django.contrib.auth.backends.ModelBackend',
    'allauth.account.auth_backends.AuthenticationBackend',
]

# Some really nice defaults
ACCOUNT_AUTHENTICATION_METHOD = 'username'
ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_EMAIL_VERIFICATION = 'mandatory'

ACCOUNT_ALLOW_REGISTRATION = env.bool('DJANGO_ACCOUNT_ALLOW_REGISTRATION', False)
ACCOUNT_ADAPTER = 'lna.users.adapters.AccountAdapter'
SOCIALACCOUNT_ADAPTER = 'lna.users.adapters.SocialAccountAdapter'

# Custom user app defaults
# Select the correct user model
AUTH_USER_MODEL = 'users.User'
LOGIN_REDIRECT_URL = 'users:redirect'
LOGIN_URL = 'account_login'

# SLUGLIFIER
AUTOSLUG_SLUGIFY_FUNCTION = 'slugify.slugify'

########## CELERY
# apps - disabled
# INSTALLED_APPS += ['lna.taskapp.celery.CeleryConfig']
"""
# Deprecated in celery 4.0
CELERY_BROKER_URL = env('CELERY_BROKER_URL', default='django://')
if CELERY_BROKER_URL == 'django://':
    CELERY_RESULT_BACKEND = 'redis://'
else:
    CELERY_RESULT_BACKEND = CELERY_BROKER_URL
"""
# added by hu4
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_RESULT_BACKEND = 'redis://localhost/0'
# BROKER_URL = 'redis://localhost:6379/0'
# BROKER_URL = 'redis://localhost:6379'

########## END CELERY

# Location of root django.contrib.admin URL, use {% url 'admin:index' %}
ADMIN_URL = r'^admin/'

# Your common stuff: Below this line define 3rd party library settings
# ------------------------------------------------------------------------------
ALLOWED_HOSTS = ['10.179.9.55', '10.205.29.29', '127.0.0.1', 'localhost']

EL_PAGINATION_PER_PAGE = 9
ENDLESS_PAGINATION_PAGE_LIST_CALLABLE = 'endless_pagination.utils.get_elastic_page_numbers'

# Django channels
CHANNEL_LAYERS = {
    "default": {
        "BACKEND": "asgi_redis.RedisChannelLayer",
        "CONFIG": {
            "hosts": [("localhost", 6379)],
        },
        "ROUTING": "config.routing.channel_routing",
    },
}

REST_FRAMEWORK = {
    'DEFAULT_PERMISSION_CLASSES': [
        'rest_framework.permissions.IsAuthenticated',
    ],
#    'PAGE_SIZE': 10
}
# for selecting many objects in forms
DATA_UPLOAD_MAX_NUMBER_FIELDS = None
<commit_msg>changing allowed hosts<commit_after>"""
Django settings for lna project.

For more information on this file, see
https://docs.djangoproject.com/en/dev/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/dev/ref/settings/
"""
import environ

ROOT_DIR = environ.Path(__file__) - 3  # (lna/config/settings/base.py - 3 = lna/)
APPS_DIR = ROOT_DIR.path('lna')

# Load operating system environment variables and then prepare to use them
env = environ.Env()

# .env file, should load only in development environment
READ_DOT_ENV_FILE = env.bool('DJANGO_READ_DOT_ENV_FILE', default=True)

if READ_DOT_ENV_FILE:
    # Operating System Environment variables have precedence over variables defined in the .env file,
    # that is to say variables from the .env files will only be used if not defined
    # as environment variables.
    env_file = str(ROOT_DIR.path('.env'))
    print('Loading : {}'.format(env_file))
    env.read_env(env_file)
    print('The .env file has been loaded. See base.py for more information')

# APP CONFIGURATION
# ------------------------------------------------------------------------------
DJANGO_APPS = [
    # Default Django apps:
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    # Useful template tags:
    # 'django.contrib.humanize',

    # Admin
    'django.contrib.admin',
]
THIRD_PARTY_APPS = [
    'crispy_forms',  # Form layouts
    'allauth',  # registration
    'allauth.account',  # registration
    'allauth.socialaccount',  # registration
    'el_pagination', # el-pagination
    'channels',  # django channels
    'rest_framework',  #DRF
    'django_celery_beat',  # django-celery-beat extension that stores the schedule in the Django database,
    # and presents a convenient admin interface to manage periodic tasks at runtime.
    'django_celery_results',
    'netfields',
]

# Apps specific for this project go here.
LOCAL_APPS = [
    # custom users app
    'lna.users.apps.UsersConfig',
    # Your stuff: custom apps go here
    'argus',
    'net',
    'lna',
    'api',
]

# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS

# MIDDLEWARE CONFIGURATION
# ------------------------------------------------------------------------------
MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

# MIGRATIONS CONFIGURATION
# ------------------------------------------------------------------------------
MIGRATION_MODULES = {
    'sites': 'lna.contrib.sites.migrations'
}

# DEBUG
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#debug
# DEBUG = env.bool('DJANGO_DEBUG', True)
DEBUG = False
# FIXTURE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-FIXTURE_DIRS
FIXTURE_DIRS = (
    str(APPS_DIR.path('fixtures')),
)

# EMAIL CONFIGURATION
# ------------------------------------------------------------------------------
EMAIL_BACKEND = env('DJANGO_EMAIL_BACKEND', default='django.core.mail.backends.smtp.EmailBackend')

# MANAGER CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#admins
ADMINS = [
    ("""Dehu4ka""", 'dehu4ka@gmail.com'),
]

# See: https://docs.djangoproject.com/en/dev/ref/settings/#managers
MANAGERS = ADMINS

# DATABASE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
DATABASES = {
    'default': env.db('DATABASE_URL', default='postgres:///lna'),
}
DATABASES['default']['ATOMIC_REQUESTS'] = True

# GENERAL CONFIGURATION
# ------------------------------------------------------------------------------
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'Asia/Yekaterinburg'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#language-code
LANGUAGE_CODE = 'en-us'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#site-id
SITE_ID = 1

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-i18n
USE_I18N = True

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-l10n
USE_L10N = True

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-tz
USE_TZ = True

# TEMPLATE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#templates
TEMPLATES = [
    {
        # See: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-TEMPLATES-BACKEND
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        # See: https://docs.djangoproject.com/en/dev/ref/settings/#template-dirs
        'DIRS': [
            str(APPS_DIR.path('templates')),
        ],
        'OPTIONS': {
            # See: https://docs.djangoproject.com/en/dev/ref/settings/#template-debug
            'debug': DEBUG,
            # See: https://docs.djangoproject.com/en/dev/ref/settings/#template-loaders
            # https://docs.djangoproject.com/en/dev/ref/templates/api/#loader-types
            'loaders': [
                'django.template.loaders.filesystem.Loader',
                'django.template.loaders.app_directories.Loader',
            ],
            # See: https://docs.djangoproject.com/en/dev/ref/settings/#template-context-processors
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',  # It was exists | Also For EL-pagination
                'django.contrib.auth.context_processors.auth',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.messages.context_processors.messages',
                # Your stuff: custom template context processors go here
            ],
        },
    },
]

# See: http://django-crispy-forms.readthedocs.io/en/latest/install.html#template-packs
CRISPY_TEMPLATE_PACK = 'bootstrap4'

# STATIC FILE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root
STATIC_ROOT = str(ROOT_DIR('staticfiles'))

# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
STATIC_URL = '/static/'

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
STATICFILES_DIRS = [
    str(APPS_DIR.path('static')),
]
# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-finders
STATICFILES_FINDERS = [
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
]

# MEDIA CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
MEDIA_ROOT = str(APPS_DIR('media'))

# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
MEDIA_URL = '/media/'

# URL Configuration
# ------------------------------------------------------------------------------
ROOT_URLCONF = 'config.urls'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
WSGI_APPLICATION = 'config.wsgi.application'

# PASSWORD STORAGE SETTINGS
# ------------------------------------------------------------------------------
# See https://docs.djangoproject.com/en/dev/topics/auth/passwords/#using-argon2-with-django
PASSWORD_HASHERS = [
    'django.contrib.auth.hashers.Argon2PasswordHasher',
    'django.contrib.auth.hashers.PBKDF2PasswordHasher',
    'django.contrib.auth.hashers.PBKDF2SHA1PasswordHasher',
    'django.contrib.auth.hashers.BCryptSHA256PasswordHasher',
    'django.contrib.auth.hashers.BCryptPasswordHasher',
]

# PASSWORD VALIDATION
# https://docs.djangoproject.com/en/dev/ref/settings/#auth-password-validators
# ------------------------------------------------------------------------------

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# AUTHENTICATION CONFIGURATION
# ------------------------------------------------------------------------------
AUTHENTICATION_BACKENDS = [
    'django.contrib.auth.backends.ModelBackend',
    'allauth.account.auth_backends.AuthenticationBackend',
]

# Some really nice defaults
ACCOUNT_AUTHENTICATION_METHOD = 'username'
ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_EMAIL_VERIFICATION = 'mandatory'

ACCOUNT_ALLOW_REGISTRATION = env.bool('DJANGO_ACCOUNT_ALLOW_REGISTRATION', False)
ACCOUNT_ADAPTER = 'lna.users.adapters.AccountAdapter'
SOCIALACCOUNT_ADAPTER = 'lna.users.adapters.SocialAccountAdapter'

# Custom user app defaults
# Select the correct user model
AUTH_USER_MODEL = 'users.User'
LOGIN_REDIRECT_URL = 'users:redirect'
LOGIN_URL = 'account_login'

# SLUGLIFIER
AUTOSLUG_SLUGIFY_FUNCTION = 'slugify.slugify'

########## CELERY
# apps - disabled
# INSTALLED_APPS += ['lna.taskapp.celery.CeleryConfig']
"""
# Deprecated in celery 4.0
CELERY_BROKER_URL = env('CELERY_BROKER_URL', default='django://')
if CELERY_BROKER_URL == 'django://':
    CELERY_RESULT_BACKEND = 'redis://'
else:
    CELERY_RESULT_BACKEND = CELERY_BROKER_URL
"""
# added by hu4
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_RESULT_BACKEND = 'redis://localhost/0'
# BROKER_URL = 'redis://localhost:6379/0'
# BROKER_URL = 'redis://localhost:6379'

########## END CELERY

# Location of root django.contrib.admin URL, use {% url 'admin:index' %}
ADMIN_URL = r'^admin/'

# Your common stuff: Below this line define 3rd party library settings
# ------------------------------------------------------------------------------
ALLOWED_HOSTS = ['10.179.9.55', '10.205.29.29', '127.0.0.1', 'localhost']  # +

EL_PAGINATION_PER_PAGE = 9
ENDLESS_PAGINATION_PAGE_LIST_CALLABLE = 'endless_pagination.utils.get_elastic_page_numbers'

# Django channels
CHANNEL_LAYERS = {
    "default": {
        "BACKEND": "asgi_redis.RedisChannelLayer",
        "CONFIG": {
            "hosts": [("localhost", 6379)],
        },
        "ROUTING": "config.routing.channel_routing",
    },
}

REST_FRAMEWORK = {
    'DEFAULT_PERMISSION_CLASSES': [
        'rest_framework.permissions.IsAuthenticated',
    ],
#    'PAGE_SIZE': 10
}
# for selecting many objects in forms
DATA_UPLOAD_MAX_NUMBER_FIELDS = None
<|endoftext|>


<commit_before>import argparse
import configs
import multiprocessing as mp
import numpy as np
import os
from skimage.io import imread
import xgboost as xgb

import data
import featurizer
from configs import HAIR, FACE, BKG
import configs

def train(model_fname, model_type, dtrain, dval=None, dtest=None, cont=None):
    params = getattr(configs, model_type)()
    evallist = []
    if dval: evallist.append((dval, 'val'))
    if dtest: evallist.append((dtest, 'test'))
    evallist.append((dtrain, 'train'))
    bst = xgb.train(params, dtrain, params['num_round'], evallist,
                    xgb_model=cont, early_stopping_rounds=5)
    bst.save_model(model_fname)
    bst.dump_model('dump.raw.txt')

def test():
    pass

def args():
    args = argparse.ArgumentParser()
    args.add_argument('model', help='if --train is specified, then the name of'
                      ' the trained model; else the model name for testing')
    args.add_argument('-c', '--cont', help='model file to resume training')
    args.add_argument('-t', '--train', help='train set file')
    args.add_argument('-v', '--validation', help='validation set file')
    args.add_argument('-e', '--test', help='test set file')
    return args.parse_args()

if __name__ == '__main__':
    parse = args()
    n, e = os.path.splitext(os.path.basename(parse.model))
    mtype = n.split('_')[0]
    if not hasattr(configs, mtype):
        import sys
        print 'Invalid model type. See configs.py'
        sys.exit(0)
    print 'Model type {} & Model name {}'.format(mtype, parse.model)
    dtrain = xgb.DMatrix(parse.train) if parse.train else None
    dval = xgb.DMatrix(parse.validation) if parse.validation else None
    dtest = xgb.DMatrix(parse.test) if parse.test else None

    if dtrain:
        train(parse.model, mtype,
              dtrain, dval=dval, dtest=dtest,
              cont=parse.cont)
    elif dtest or dval:
        test()
<commit_msg>test func<commit_after>import argparse
import configs
import multiprocessing as mp
import numpy as np
import os
from skimage.io import imread
from sklearn.datasets import load_svmlight_file
import xgboost as xgb

import data
import featurizer
from configs import HAIR, FACE, BKG
import configs

def train(model_fname, model_type, dtrain, dval=None, dtest=None, cont=None):
    params = getattr(configs, model_type)()
    evallist = []
    if dval: evallist.append((dval, 'val'))
    if dtest: evallist.append((dtest, 'test'))
    evallist.append((dtrain, 'train'))
    bst = xgb.train(params, dtrain, params['num_round'], evallist,
                    xgb_model=cont, early_stopping_rounds=5)
    bst.save_model(model_fname)
    bst.dump_model('dump.raw.txt')

def test(model_fname, model_type, fset):
    params = getattr(configs, model_type)(val=True)
    bst = xgb.Booster(params=params)
    bst.load_model(model_fname)
    nset = load_svmlight_file(fset)
    dset = xgb.DMatrix(nset)
    preds = bst.predict(dset)
    pass

def get_model_feature_type(name):
    n, e = os.path.splitext(os.path.basename(name))
    mtype, ftype = n.split('_')[:2]
    if not hasattr(configs, mtype):
        print 'Invalid model type in {}. See configs.py'.format(name)
        mtype, ftype = None, None
    if not hasattr(configs, ftype):
        print 'Invalid feature type in {}. See configs.py'.format(name)
        mtype, ftype = None, None
    return mtype, ftype

def args():
    args = argparse.ArgumentParser()
    args.add_argument('model', help='if --train is specified, then the name of'
                      ' the trained model; else the model name for testing')
    args.add_argument('-c', '--cont', help='model file to resume training')
    args.add_argument('-t', '--train', help='train set file')
    args.add_argument('-v', '--validation', help='validation set file')
    args.add_argument('-e', '--test', help='test set file')
    return args.parse_args()

if __name__ == '__main__':
    parse = args()
    mtype, ftype = get_model_feature_type(parse.model)
    if not hasattr(configs, mtype):
        import sys
        print 'Invalid model type. See configs.py'
        sys.exit(0)
    print 'Model type {} & Model name {}'.format(mtype, parse.model)
    dtrain = xgb.DMatrix(parse.train) if parse.train else None
    dval = xgb.DMatrix(parse.validation) if parse.validation else None
    dtest = xgb.DMatrix(parse.test) if parse.test else None

    if dtrain:
        train(parse.model, mtype,
              dtrain, dval=dval, dtest=dtest,
              cont=parse.cont)
    elif dtest or dval:
        test(parse.model, mtype, dval=dval, dtest=dtest)
<|endoftext|>