<commit_before>"""
Seasonal Decomposition by Moving Averages
"""
import numpy as np
from pandas.core.nanops import nanmean as pd_nanmean
from .filters.utils import _maybe_get_pandas_wrapper_freq
from .filters.filtertools import convolution_filter
from statsmodels.tsa.tsatools import freq_to_period
from statsmodels.tools.tools import Bunch

def seasonal_mean(x, freq):
    """
    Return means for each period in x. freq is an int that gives the
    number of periods per cycle. E.g., 12 for monthly. NaNs are ignored
    in the mean.
    """
    return np.array([pd_nanmean(x[i::freq]) for i in range(freq)])

def seasonal_decompose(X, model="additive", filt=None, freq=None):
    """
    Parameters
    ----------
    X : array-like
        Time series
    model : str {"additive", "multiplicative"}
        Type of seasonal component. Abbreviations are accepted.
    filt : array-like
        The filter coefficients for filtering out the seasonal component.
        The default is a symmetric moving average.
    freq : int, optional
        Frequency of the series. Must be used if X is not a pandas
        object with a timeseries index.

    Returns
    -------
    results : obj
        A object with seasonal, trend, and resid attributes.

    Notes
    -----
    This is a naive decomposition. More sophisticated methods should
    be preferred.

    The additive model is Y[t] = T[t] + S[t] + e[t]

    The multiplicative model is Y[t] = T[t] * S[t] * e[t]

    The seasonal component is first removed by applying a convolution
    filter to the data. The average of this smoothed series for each
    period is the returned seasonal component.

    See Also
    --------
    statsmodels.tsa.filters.convolution_filter
    """
    _pandas_wrapper, pfreq = _maybe_get_pandas_wrapper_freq(X)
    X = np.asanyarray(X).squeeze()
    nobs = len(X)

    if not np.all(np.isfinite(X)):
        raise ValueError("This function does not handle missing values")
    if model.startswith('m'):
        if np.any(X <= 0):
            raise ValueError("Multiplicative seasonality is not appropriate "
                             "for zero and negative values")

    if pfreq is not None:
        pfreq = freq_to_period(pfreq)
        if freq and pfreq != freq:
            raise ValueError("Inferred frequency of index and frequency "
                             "don't match. This function does not re-sample")
        else:
            freq = pfreq

    elif freq is None:
        raise ValueError("You must specify a freq or X must be a "
                         "pandas object with a timeseries index")

    if filt is None:
        if freq % 2 == 0:  # split weights at ends
            filt = np.array([.5] + [1] * (freq - 1) + [.5]) / freq
        else:
            filt = np.repeat(1./freq, freq)

    trend = convolution_filter(X, filt)

    # nan pad for conformability - convolve doesn't do it
    if model.startswith('m'):
        detrended = X/trend
    else:
        detrended = X - trend

    period_averages = seasonal_mean(detrended, freq)

    if model.startswith('m'):
        period_averages /= np.mean(period_averages)
    else:
        period_averages -= np.mean(period_averages)

    seasonal = np.tile(period_averages, nobs // freq + 1)[:nobs]

    if model.startswith('m'):
        resid = X / seasonal / trend
    else:
        resid = detrended - seasonal

    results = map(_pandas_wrapper, [seasonal, trend, resid, X])
    return DecomposeResult(seasonal=results[0], trend=results[1],
                           resid=results[2], observed=results[3])

class DecomposeResult(object):
    def __init__(self, **kwargs):
        for key, value in kwargs.iteritems():
            setattr(self, key, value)
            self.nobs = len(self.observed)

    def plot(self):
        from statsmodels.graphics.utils import _import_mpl
        plt = _import_mpl()
        fig, axes = plt.subplots(4, 1, sharex=True)
        if hasattr(self.observed, 'plot'):  # got pandas use it
            self.observed.plot(ax=axes[0], legend=False)
            axes[0].set_ylabel('Observed')
            self.trend.plot(ax=axes[1], legend=False)
            axes[1].set_ylabel('Trend')
            self.seasonal.plot(ax=axes[2], legend=False)
            axes[2].set_ylabel('Seasonal')
            self.resid.plot(ax=axes[3], legend=False)
            axes[3].set_ylabel('Residual')
        else:
            axes[0].plot(self.observed)
            axes[0].set_ylabel('Observed')
            axes[1].plot(self.trend)
            axes[1].set_ylabel('Trend')
            axes[2].plot(self.seasonal)
            axes[2].set_ylabel('Seasonal')
            axes[3].plot(self.resid)
            axes[3].set_ylabel('Residual')
            axes[3].set_xlabel('Time')
            axes[3].set_xlim(0, self.nobs)

        fig.tight_layout()
        return fig

if __name__ == "__main__":
    x = np.array([-50, 175, 149, 214, 247, 237, 225, 329, 729, 809,
                  530, 489, 540, 457, 195, 176, 337, 239, 128, 102,
                  232, 429, 3, 98, 43, -141, -77, -13, 125, 361, -45, 184])
    results = seasonal_decompose(x, freq=4)

    from pandas import DataFrame, DatetimeIndex
    data = DataFrame(x, DatetimeIndex(start='1/1/1951',
                                      periods=len(x),
                                      freq='Q'))

    res = seasonal_decompose(data)

<commit_msg>BUG: Fix indent<commit_after>"""
Seasonal Decomposition by Moving Averages
"""
import numpy as np
from pandas.core.nanops import nanmean as pd_nanmean
from .filters.utils import _maybe_get_pandas_wrapper_freq
from .filters.filtertools import convolution_filter
from statsmodels.tsa.tsatools import freq_to_period
from statsmodels.tools.tools import Bunch

def seasonal_mean(x, freq):
    """
    Return means for each period in x. freq is an int that gives the
    number of periods per cycle. E.g., 12 for monthly. NaNs are ignored
    in the mean.
    """
    return np.array([pd_nanmean(x[i::freq]) for i in range(freq)])

def seasonal_decompose(X, model="additive", filt=None, freq=None):
    """
    Parameters
    ----------
    X : array-like
        Time series
    model : str {"additive", "multiplicative"}
        Type of seasonal component. Abbreviations are accepted.
    filt : array-like
        The filter coefficients for filtering out the seasonal component.
        The default is a symmetric moving average.
    freq : int, optional
        Frequency of the series. Must be used if X is not a pandas
        object with a timeseries index.

    Returns
    -------
    results : obj
        A object with seasonal, trend, and resid attributes.

    Notes
    -----
    This is a naive decomposition. More sophisticated methods should
    be preferred.

    The additive model is Y[t] = T[t] + S[t] + e[t]

    The multiplicative model is Y[t] = T[t] * S[t] * e[t]

    The seasonal component is first removed by applying a convolution
    filter to the data. The average of this smoothed series for each
    period is the returned seasonal component.

    See Also
    --------
    statsmodels.tsa.filters.convolution_filter
    """
    _pandas_wrapper, pfreq = _maybe_get_pandas_wrapper_freq(X)
    X = np.asanyarray(X).squeeze()
    nobs = len(X)

    if not np.all(np.isfinite(X)):
        raise ValueError("This function does not handle missing values")
    if model.startswith('m'):
        if np.any(X <= 0):
            raise ValueError("Multiplicative seasonality is not appropriate "
                             "for zero and negative values")

    if pfreq is not None:
        pfreq = freq_to_period(pfreq)
        if freq and pfreq != freq:
            raise ValueError("Inferred frequency of index and frequency "
                             "don't match. This function does not re-sample")
        else:
            freq = pfreq

    elif freq is None:
        raise ValueError("You must specify a freq or X must be a "
                         "pandas object with a timeseries index")

    if filt is None:
        if freq % 2 == 0:  # split weights at ends
            filt = np.array([.5] + [1] * (freq - 1) + [.5]) / freq
        else:
            filt = np.repeat(1./freq, freq)

    trend = convolution_filter(X, filt)

    # nan pad for conformability - convolve doesn't do it
    if model.startswith('m'):
        detrended = X/trend
    else:
        detrended = X - trend

    period_averages = seasonal_mean(detrended, freq)

    if model.startswith('m'):
        period_averages /= np.mean(period_averages)
    else:
        period_averages -= np.mean(period_averages)

    seasonal = np.tile(period_averages, nobs // freq + 1)[:nobs]

    if model.startswith('m'):
        resid = X / seasonal / trend
    else:
        resid = detrended - seasonal

    results = map(_pandas_wrapper, [seasonal, trend, resid, X])
    return DecomposeResult(seasonal=results[0], trend=results[1],
                           resid=results[2], observed=results[3])

class DecomposeResult(object):
    def __init__(self, **kwargs):
        for key, value in kwargs.iteritems():
            setattr(self, key, value)
        self.nobs = len(self.observed)

    def plot(self):
        from statsmodels.graphics.utils import _import_mpl
        plt = _import_mpl()
        fig, axes = plt.subplots(4, 1, sharex=True)
        if hasattr(self.observed, 'plot'):  # got pandas use it
            self.observed.plot(ax=axes[0], legend=False)
            axes[0].set_ylabel('Observed')
            self.trend.plot(ax=axes[1], legend=False)
            axes[1].set_ylabel('Trend')
            self.seasonal.plot(ax=axes[2], legend=False)
            axes[2].set_ylabel('Seasonal')
            self.resid.plot(ax=axes[3], legend=False)
            axes[3].set_ylabel('Residual')
        else:
            axes[0].plot(self.observed)
            axes[0].set_ylabel('Observed')
            axes[1].plot(self.trend)
            axes[1].set_ylabel('Trend')
            axes[2].plot(self.seasonal)
            axes[2].set_ylabel('Seasonal')
            axes[3].plot(self.resid)
            axes[3].set_ylabel('Residual')
            axes[3].set_xlabel('Time')
            axes[3].set_xlim(0, self.nobs)

        fig.tight_layout()
        return fig

if __name__ == "__main__":
    x = np.array([-50, 175, 149, 214, 247, 237, 225, 329, 729, 809,
                  530, 489, 540, 457, 195, 176, 337, 239, 128, 102,
                  232, 429, 3, 98, 43, -141, -77, -13, 125, 361, -45, 184])
    results = seasonal_decompose(x, freq=4)

    from pandas import DataFrame, DatetimeIndex
    data = DataFrame(x, DatetimeIndex(start='1/1/1951',
                                      periods=len(x),
                                      freq='Q'))

    res = seasonal_decompose(data)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu

import os
import json
import logging
import copy

import requests
from requests.auth import HTTPBasicAuth

from africa_data import countries

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)

GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')

output = 'step1.json'
one_minute = 60
one_hour = one_minute * 60
min_remaining_tostop = 30
reqs = 0
reqs_limit = None
reqs_remaining = None
headers = {}
TOKEN_AUTH = HTTPBasicAuth(GITHUB_TOKEN, "x-oauth-basic")

all_users = []

def add_to(search_term, all_users, country_stub, city=None):

    def users_from(location):
        complete = False
        page = 1
        users = []
        order = 'asc'
        while not complete:
            if page > 10:
                # well, we can't query anymore.
                if order == 'desc':
                    complete = True
                    continue
                order = 'desc'
                page = 1
            req = requests.get(
                'https://api.github.com/legacy/user/search/location:%s' %
                location,
                headers=headers, params={'start_page': page,
                                         'sort': 'joined',
                                         'order': order},
                auth=TOKEN_AUTH)
            page += 1
            try:
                jsusers = json.loads(req.content).get('users')
                if not len(jsusers):
                    complete = True
                    continue
                users += jsusers
            except:
                logger.warning("Failed to parse JSON:")
                logger.warning(req.content)
                complete = True

        return users

    json_users = users_from(search_term)

    if not len(json_users):
        return

    for user in json_users:
        logger.info("FOUND -- %s -- %s" % (user.get('username'),
                                           user.get('location')))
        user.update({'country': country_stub,
                     'city': city})
        all_users.append(user)

for country_code, country in countries.items():
    logger.info("COUNTRY: %s" % country.get('name'))

    country_stub = copy.copy(country)
    country_stub.update({'code': country_code})
    del(country_stub['patterns'])

    for city in country.get('patterns', []):
        logging.info("SEARCHING for city -- %s" % city.get('name'))
        for search_name in city.get('patterns', [city.get('name')]):
            add_to(search_name, all_users, country_stub, city)

    for name in country.get('names', []):
        logging.info("SEARCHING for country -- %s" % name)
        add_to(name, all_users, country_stub, None)

logger.info("Found %d records" % len(all_users))

json.dump(all_users, open(output, 'w'), indent=4)

logger.info("UNIQUE user accounts: %d" %
            len(list(set([u.get('username') for u in all_users]))))
<commit_msg>Update step1_search_by_location.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu

import os
import json
import logging
import copy

import requests
from requests.auth import HTTPBasicAuth

from africaData import countries

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)

GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')

output = 'step1.json'
oneMinute = 60
oneHour = oneMinute * 60
minRemainingToStop = 30
reqs = 0
reqsLimit = None
reqsRemaining = None
headers = {}
TOKEN_AUTH = HTTPBasicAuth(GITHUB_TOKEN, "x-oauth-basic")

allUsers = []

def addTo(searchTerm, allUsers, countryStub, city=None):

    def usersFrom(location):
        complete = False
        page = 1
        users = []
        order = 'asc'
        while not complete:
            if page > 10:
                # well, we can't query anymore.
                if order == 'desc':
                    complete = True
                    continue
                order = 'desc'
                page = 1
            req = requests.get(
                'https://api.github.com/legacy/user/search/location:%s' %
                location,
                headers=headers, params={'start_page': page,
                                         'sort': 'joined',
                                         'order': order},
                auth=TOKEN_AUTH)
            page += 1
            try:
                jsusers = json.loads(req.content).get('users')
                if not len(jsusers):
                    complete = True
                    continue
                users += jsusers
            except:
                logger.warning("Failed to parse JSON:")
                logger.warning(req.content)
                complete = True

        return users

    jsonUsers = usersFrom(searchTerm)

    if not len(jsonUsers):
        return

    for user in jsonUsers:
        logger.info("FOUND -- %s -- %s" % (user.get('username'),
                                           user.get('location')))
        user.update({'country': countryStub,
                     'city': city})
        allUsers.append(user)

for countryCode, country in countries.items():
    logger.info("COUNTRY: %s" % country.get('name'))

    countryStub = copy.copy(country)
    countryStub.update({'code': countryCode})
    del(countryStub['patterns'])

    for city in country.get('patterns', []):
        logging.info("SEARCHING for city -- %s" % city.get('name'))
        for searchName in city.get('patterns', [city.get('name')]):
            addTo(searchName, allUsers, countryStub, city)

    for name in country.get('names', []):
        logging.info("SEARCHING for country -- %s" % name)
        addTo(name, allUsers, countryStub, None)

logger.info("Found %d records" % len(allUsers))

json.dump(allUsers, open(output, 'w'), indent=4)

logger.info("UNIQUE user accounts: %d" %
            len(list(set([u.get('username') for u in allUsers]))))
<|endoftext|>


<commit_before>def fabonacci(n):
	"""
	Return the n'th number of the fabonacci sequence
	"""
	if n == 0:
		return 0
	elif n == 1:
		return 1
	else:
		return fabonacci(n-1) + fabonacci(n-2)

print(fabonacci(14))

def is_palindrome(sequence):
	"""
	Checks if sequence is a palindrome, returns boolean
	"""
	sequence = str(sequence).lower()
	for index, letter in enumerate(sequence):
		if letter != sequence[(index + 1) * -1]:
			return False
	return True

print(is_palindrome('Racecar'))

def my_sqrt(a,x):
	"""
	Newton's method is a method for finding successively better
	approximations (x) to the roots (or zeroes) of a real-valued
	function (a)
	"""
	while True:
		print(x)
		y = (x + a / x) / 2
		if y == x:
			break
		x = y

my_sqrt(4, 2)
<commit_msg>Delete snip.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import sqlite3 as lite
import sys
from datetime import datetime, timedelta
#import codecs
#codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)

def out(unicodeobj):
  print unicodeobj.encode('utf-8')

gctimeformat = '%Y%m%d%H%M%S'

#print sys.getdefaultencoding()
#print sys.getfilesystemencoding()
#print "Encoding is", sys.stdout.encoding
#out ( sys.argv[1].decode(sys.getfilesystemencoding()) )

con = lite.connect ( sys.argv[1].decode(sys.getfilesystemencoding()) )

with con:

  cur = con.cursor()

  # time range
  cur.execute ( "select min(post_date), max(post_date) from transactions" );
  x = cur.fetchone();
  #dt = datetime.strptime ( x[0], gctimeformat );
  dt = datetime.strptime ( '20150101000000', gctimeformat );
  verystart = dt - timedelta(dt.weekday())
  #dt = datetime.strptime ( x[1], gctimeformat );
  dt = datetime.now()
  veryend = dt - timedelta(dt.weekday()) + timedelta(7)
  #print ( verystart, veryend );

  # all expense accounts
  cur.execute ( "select name, parent_guid, a.guid, c.mnemonic from accounts a join commodities c on a.commodity_guid=c.guid where account_type='EXPENSE' and placeholder=0" );
  res = cur.fetchall()
  accs = []
  for x in res:
    name = []
    parent = x[1]
    acc = { 'guid': x[2], 'currency': x[3] }
    cparentacc = con.cursor()
    while parent != None:
      name.append ( x[0] )
      cparentacc.execute ( "select name, parent_guid from accounts where guid=?", (parent,) )
      x = cparentacc.fetchone()
      parent = x[1]
    acc['name'] = [i for i in reversed(name)]
    #out ( ':'.join(acc['name']) )
    accs.append(acc)

  # now accs contains list of expense accounts we are interested in

  # get weekly history for every expense account
  for acc in accs:
    weekst = verystart
    out ( ':'.join(acc['name']) )
    out ( acc['currency'] )
    while weekst < veryend:
      weeklo = weekst
      weekst += timedelta(7)
      cur.execute ( "select sum(cast(quantity_num as numeric(10,2))/100.0) from transactions t join splits s on s.tx_guid=t.guid "
        "where s.account_guid=? and t.post_date>=? and t.post_date<?"
        "and reconcile_state in ('y','c','n')", (acc['guid'],weeklo.strftime(gctimeformat),weekst.strftime(gctimeformat)) )
      x = cur.fetchone()
      if x[0] == None:
        newbalance = 0
      else:
        newbalance = x[0]
      print weeklo, newbalance
      <commit_msg>Account histories to the table<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import sqlite3 as lite
import sys
from datetime import datetime, timedelta
#import codecs
#codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)

def out ( unicodeobj ):
  sys.stdout.write ( unicodeobj.encode('utf-8') )
  sys.stdout.write ( '\t' )

gctimeformat = '%Y%m%d%H%M%S'

#print sys.getdefaultencoding()
#print sys.getfilesystemencoding()
#print "Encoding is", sys.stdout.encoding
#out ( sys.argv[1].decode(sys.getfilesystemencoding()) )

con = lite.connect ( sys.argv[1].decode(sys.getfilesystemencoding()) )

with con:

  cur = con.cursor()

  # time range
  cur.execute ( "select min(post_date), max(post_date) from transactions" );
  x = cur.fetchone();
  #dt = datetime.strptime ( x[0], gctimeformat );
  dt = datetime.strptime ( '20150101000000', gctimeformat );
  verystart = dt - timedelta(dt.weekday())
  #dt = datetime.strptime ( x[1], gctimeformat );
  dt = datetime.now()
  veryend = dt - timedelta(dt.weekday()) + timedelta(7)
  #print ( verystart, veryend );

  # all expense accounts
  cur.execute ( "select name, parent_guid, a.guid, c.mnemonic from accounts a join commodities c on a.commodity_guid=c.guid where account_type='EXPENSE' and placeholder=0" );
  res = cur.fetchall()
  accs = []
  for x in res:
    name = []
    parent = x[1]
    acc = { 'guid': x[2], 'currency': x[3] }
    cparentacc = con.cursor()
    while parent != None:
      name.append ( x[0] )
      cparentacc.execute ( "select name, parent_guid from accounts where guid=?", (parent,) )
      x = cparentacc.fetchone()
      parent = x[1]
    acc['name'] = [i for i in reversed(name)]
    #out ( ':'.join(acc['name']) )
    accs.append(acc)

  # now accs contains list of expense accounts we are interested in

  # print table header
  out ( 'Account' )
  out ( 'Currency' )
  weekst = verystart
  while weekst < veryend:
    out ( weekst.date().isoformat() )
    weekst += timedelta(7)
  print

  # get weekly history for every expense account
  for acc in accs:
    weekst = verystart
    out ( ':'.join(acc['name']) )
    out ( acc['currency'] )
    while weekst < veryend:
      weeklo = weekst
      weekst += timedelta(7)
      cur.execute ( "select sum(cast(quantity_num as numeric(10,2))/100.0) from transactions t join splits s on s.tx_guid=t.guid "
        "where s.account_guid=? and t.post_date>=? and t.post_date<?"
        "and reconcile_state in ('y','c','n')", (acc['guid'],weeklo.strftime(gctimeformat),weekst.strftime(gctimeformat)) )
      x = cur.fetchone()
      if x[0] == None:
        newbalance = 0
      else:
        newbalance = x[0]
      out ( str(newbalance) )
    print
      <|endoftext|>


<commit_before>import numpy as np
import argparse
import glob
import sys

parser = argparse.ArgumentParser(description='Sort the spectra.')
parser.add_argument('folder', metavar='FOLDER', type=str,
                  help='FOLDER with .lopa files to be processed')
parser.add_argument('--bins', metavar='bins', type=str,
                   help='file defining the wawelenghts bins')
parser.add_argument('--merged_file', type=str, default="merged.lopa",
                   help='set the location of the merged file, by default it will be stored in the current directory under the name "merged.lopa"')
#parser.add_argument('-n','--sub-bin-number', metavar='subbins', action='store_const',
#                  help='set the number of subbin each bin will be split during calculations')

args = parser.parse_args()
#set the necessary parameters for parsing

file_list = []
file_list = sorted(glob.glob(args.folder+'*.lopa'))
#print(file_list)
file_list = np.sort(file_list)
#print(file_list)
numberOfItems = len(file_list)
#create a list off all the .lopa files

merged_file = args.merged_file
#create a path to the merged file

def bining(depthList):
    reducedFileList = [str(item) + '.reduced' for item in depthList]
    segmentFileList = [str(item) + '.segment' for item in depthList]
    minMax = np.loadtxt(str(depthList[0]) + '.segment') #get minimum and maximum wawelenghts from the first .segment file
    wawelenghts = [array[0] for array in minMax]
    minimum = np.min(wawelenghts)
    maximum = np.max(wawelenghts)
    print('Minimum: ' + str(minimum))
    print('Maximum: ' + str(maximum))

    binData = np.loadtxt(args.bins)
    if (np.min(binData) < minimum) or (np.max(binData) > maximum):
        print('Bins intervals exceede the available wawelenghts. Please check your bins and make sure they are within the following limits')
        print('Minimum: ' + str(minimum) + '\n')
        print('Maximum: ' + str(maximum) + '\n')
        sys.exit('Check your bins')
        #check if the bins are within range of wawelenghts
    counter = 0
    for currentSegmentFile in segmentFileList: #for every segmentFile
        print('Reducing: ' + str(currentSegmentFile))
        data = [np.array(map(float, line.split())) for line in open(currentSegmentFile)]
        for singleBin in binData: #for each bin
            tempList = np.array([[0,0,0],[0,0,0]]) #list for storing the data which we then write to the file in the end
            encounteredBinYet = False
            for line in data: #for each line in a segmentFile
                outsideOfTheBin = True
                if (line[0] >= singleBin[0]) and (line[0] <= singleBin[1]): #check if the wawelength of the current line is within the bin, append it
                    tempList = np.vstack([tempList, line])
                    outsideOfTheBin = False
                    encounteredBinYet = True
                    if (np.array_equal(line, data[-1])): #if we are on the last line, we must also sort the array otherwise it will go unsorted
                        tempList = sort_array(tempList, 1, 2) #sort by opacities
#==============================================================================
# Subbins
#==============================================================================
                        sub_bins(10, singleBin, tempList, counter, reducedFileList)

                elif ((outsideOfTheBin == True) and (encounteredBinYet == True)):
                    tempList = sort_array(tempList, 1, 2)#sort by opacities
                    sub_bins(10, singleBin, tempList, counter, reducedFileList)
                    break #once we leave the part of the file containing current bin, break and go to the next segment file
        counter = counter + 1
  #iterate over the segment files and extract only the data coresponding to the bins

def sub_bins(numberOfSubbins, singleBin, tempList, counter, reducedFileList):
    subBinSize = (float(singleBin[1]) - float(singleBin[0])) / numberOfSubbins
#    print('Subbin size: ' + str(subBinSize))
    deltaLambda, temp, beginning, end = 0, 0, singleBin[0], 0
    subbinArray = np.array([1,1,1])
    for line in tempList: #templist contains one bin
#        print(line[2])
        deltaLambda += line[2]
        temp += line[1] * line[2] #opacity_i*deltaLambda_i
#        print(deltaLambda)
        if (deltaLambda >= subBinSize):
            end = beginning + deltaLambda
            temp /= deltaLambda
            deltaLambda = 0
            subbinArray = np.vstack([subbinArray,[beginning, end, temp]])
            beginning = end
            end, temp = 0, 0
        elif (np.array_equal(line,tempList[-1])):
            end = beginning + deltaLambda
            temp /= deltaLambda
            deltaLambda = 0
            subbinArray = np.vstack([subbinArray,[beginning, end, temp]])
            subbinArray = np.delete(subbinArray, (0), axis=0)
            f = open(reducedFileList[counter], "a")
            np.savetxt(f, subbinArray, fmt = '%.6e')
            f.close()

def sort_array(array, column, removeHeader):
    if removeHeader > 0:
        for i in range(0, removeHeader):
            array = np.delete(array, (0), axis=0)
    array = array[np.argsort(array[:,column])]
    return array

def write_array(array, fileName, removeHeader = 0, resetTo = 0):
    if removeHeader > 0:
        for i in range(0, removeHeader):
            array = np.delete(array, (0), axis=0)
    f = open(fileName, "a")
    np.savetxt(f, array, fmt = '%.6e')
    f.close()
    if resetTo != 0:
        array = np.array(resetTo)
    return array

def merge_files(filename):
    currentFileIndex = 1.
    depthList = []

    #arrays = [np.array(map(float, line.split())) for line in open(file_list[0])]
    arrays = np.loadtxt(file_list[0])
    for array in arrays:
        if ((array[0] % 1) == 0) and ((array[1] % 1) == 0) :
            depthList.append(int(array[1]))
       #get the list of all depth points from the first .lopa file

    for filename in file_list:
        lastSegment = False #tells us whether we are in the last segment of the current file
        print(filename)
        print(str('%.2f'%(currentFileIndex/float(numberOfItems)*100)) + '%')
        currentFileIndex = currentFileIndex + 1.
        #progress bar

        tempList = np.array([[0,0],[0,0]]) #define the array to which we later append the data, it is already populated, otherwise we cannot append an array with a size of (2,1) to it
        arrays = [np.array(map(float, line.split())) for line in open(filename)] #load the lopa file

        for array in arrays: #walk over the lopa file
            if int(array[1]) > 1: #check whether we are in the header line
                currentFile = str(int(array[1])-1) + '.segment'
                tempList = write_array(tempList, currentFile, 2, [[0,0],[0,0]])

                if array[1] == depthList[-1]: #special case if we are in the last segment
                    lastSegment = True
                    currentFile = str(int(array[1])) + '.segment'

            elif int(array[1]) < 1:
                tempList = np.append(tempList, [array], 0) #if we are not in the header, append current line
                if (lastSegment == True) and (np.array_equal(array, arrays[-1])): #if we are in the last segment AND in the last line, write to file
                     tempList = write_array(tempList, currentFile, 2, [[0,0],[0,0]])

    segmentFileList = [str(item) + '.segment' for item in depthList]
    print(depthList)
    print(segmentFileList)
    for currentFile in segmentFileList:
        print('Delta lambda calculations for: ' + str(currentFile))
        workBuffer = np.loadtxt(currentFile,comments=None)
        sortedWorkBuffer = workBuffer[np.argsort(workBuffer[:,0])] #sort by the first column (wavelenghts)
        #sortedWorkBuffer = workBuffer
        bufferPosition = 0
        deltaLambdaList = np.array([])
        for line in sortedWorkBuffer:
            if bufferPosition <= len(sortedWorkBuffer) - 2:
                deltaLambdaList = np.append(deltaLambdaList, (sortedWorkBuffer[bufferPosition+1][0] - line[0]))
                bufferPosition = bufferPosition + 1
        deltaLambdaList = np.append(deltaLambdaList, 0)
        sortedWorkBuffer = np.c_[sortedWorkBuffer, deltaLambdaList] #append the column with delta lambda values
        np.savetxt(currentFile,sortedWorkBuffer, fmt = '%.6e')

        #sort the files and generate the deltaLambda values for each line
#create a separate file for each depth point as listed in depthList
#iterate over the .lopa files as listed in file_list and fill the .segment files

    print('segmentFileList length: ' + str(len(segmentFileList)))
    return depthList

depthList = merge_files(file_list)
##depthList = [4,55,67]##
bining(depthList)<commit_msg>Remove comments and prints.<commit_after>import numpy as np
import argparse
import glob
import sys

parser = argparse.ArgumentParser(description='Sort the spectra.')
parser.add_argument('folder', metavar='FOLDER', type=str,
                  help='FOLDER with .lopa files to be processed')
parser.add_argument('--bins', metavar='bins', type=str,
                   help='file defining the wawelenghts bins')
parser.add_argument('--merged_file', type=str, default="merged.lopa",
                   help='set the location of the merged file, by default it will be stored in the current directory under the name "merged.lopa"')
#parser.add_argument('-n','--sub-bin-number', metavar='subbins', action='store_const',
#                  help='set the number of subbin each bin will be split during calculations')

args = parser.parse_args()
#set the necessary parameters for parsing

file_list = []
file_list = sorted(glob.glob(args.folder+'*.lopa'))
file_list = np.sort(file_list)
numberOfItems = len(file_list)
#create a list off all the .lopa files

merged_file = args.merged_file
#create a path to the merged file

def bining(depthList):
    reducedFileList = [str(item) + '.reduced' for item in depthList]
    segmentFileList = [str(item) + '.segment' for item in depthList]
    minMax = np.loadtxt(str(depthList[0]) + '.segment') #get minimum and maximum wawelenghts from the first .segment file
    wawelenghts = [array[0] for array in minMax]
    minimum = np.min(wawelenghts)
    maximum = np.max(wawelenghts)
    print('Minimum: ' + str(minimum))
    print('Maximum: ' + str(maximum))

    binData = np.loadtxt(args.bins)
    if (np.min(binData) < minimum) or (np.max(binData) > maximum):
        print('Bins intervals exceede the available wawelenghts. Please check your bins and make sure they are within the following limits')
        print('Minimum: ' + str(minimum) + '\n')
        print('Maximum: ' + str(maximum) + '\n')
        sys.exit('Check your bins')
        #check if the bins are within range of wawelenghts
    counter = 0
    for currentSegmentFile in segmentFileList: #for every segmentFile
        print('Reducing: ' + str(currentSegmentFile))
        data = [np.array(map(float, line.split())) for line in open(currentSegmentFile)]
        for singleBin in binData: #for each bin
            tempList = np.array([[0,0,0],[0,0,0]]) #list for storing the data which we then write to the file in the end
            encounteredBinYet = False
            for line in data: #for each line in a segmentFile
                outsideOfTheBin = True
                if (line[0] >= singleBin[0]) and (line[0] <= singleBin[1]): #check if the wawelength of the current line is within the bin, append it
                    tempList = np.vstack([tempList, line])
                    outsideOfTheBin = False
                    encounteredBinYet = True
                    if (np.array_equal(line, data[-1])): #if we are on the last line, we must also sort the array otherwise it will go unsorted
                        tempList = sort_array(tempList, 1, 2) #sort by opacities
#==============================================================================
# Subbins
#==============================================================================
                        sub_bins(10, singleBin, tempList, counter, reducedFileList)

                elif ((outsideOfTheBin == True) and (encounteredBinYet == True)):
                    tempList = sort_array(tempList, 1, 2)#sort by opacities
                    sub_bins(10, singleBin, tempList, counter, reducedFileList)
                    break #once we leave the part of the file containing current bin, break and go to the next segment file
        counter = counter + 1
  #iterate over the segment files and extract only the data coresponding to the bins

def sub_bins(numberOfSubbins, singleBin, tempList, counter, reducedFileList):
    subBinSize = (float(singleBin[1]) - float(singleBin[0])) / numberOfSubbins
    deltaLambda, temp, beginning, end = 0, 0, singleBin[0], 0
    subbinArray = np.array([1,1,1])
    for line in tempList: #templist contains one bin
        deltaLambda += line[2]
        temp += line[1] * line[2] #opacity_i*deltaLambda_i
        if (deltaLambda >= subBinSize):
            end = beginning + deltaLambda
            temp /= deltaLambda
            deltaLambda = 0
            subbinArray = np.vstack([subbinArray,[beginning, end, temp]])
            beginning = end
            end, temp = 0, 0
        elif (np.array_equal(line,tempList[-1])):
            end = beginning + deltaLambda
            temp /= deltaLambda
            deltaLambda = 0
            subbinArray = np.vstack([subbinArray,[beginning, end, temp]])
            subbinArray = np.delete(subbinArray, (0), axis=0)
            f = open(reducedFileList[counter], "a")
            np.savetxt(f, subbinArray, fmt = '%.6e')
            f.close()

def sort_array(array, column, removeHeader):
    if removeHeader > 0:
        for i in range(0, removeHeader):
            array = np.delete(array, (0), axis=0)
    array = array[np.argsort(array[:,column])]
    return array

def write_array(array, fileName, removeHeader = 0, resetTo = 0):
    if removeHeader > 0:
        for i in range(0, removeHeader):
            array = np.delete(array, (0), axis=0)
    f = open(fileName, "a")
    np.savetxt(f, array, fmt = '%.6e')
    f.close()
    if resetTo != 0:
        array = np.array(resetTo)
    return array

def merge_files(filename):
    currentFileIndex = 1.
    depthList = []

    arrays = np.loadtxt(file_list[0])
    for array in arrays:
        if ((array[0] % 1) == 0) and ((array[1] % 1) == 0) :
            depthList.append(int(array[1]))
    #get the list of all depth points from the first .lopa file

    for filename in file_list:
        lastSegment = False #tells us whether we are in the last segment of the current file
        print(filename)
        print(str('%.2f'%(currentFileIndex/float(numberOfItems)*100)) + '%')
        currentFileIndex = currentFileIndex + 1.
        #progress bar

        tempList = np.array([[0,0],[0,0]]) #define the array to which we later append the data, it is already populated, otherwise we cannot append an array with a size of (2,1) to it
        arrays = [np.array(map(float, line.split())) for line in open(filename)] #load the lopa file

        for array in arrays: #walk over the lopa file
            if int(array[1]) > 1: #check whether we are in the header line
                currentFile = str(int(array[1])-1) + '.segment'
                tempList = write_array(tempList, currentFile, 2, [[0,0],[0,0]])

                if array[1] == depthList[-1]: #special case if we are in the last segment
                    lastSegment = True
                    currentFile = str(int(array[1])) + '.segment'

            elif int(array[1]) < 1:
                tempList = np.append(tempList, [array], 0) #if we are not in the header, append current line
                if (lastSegment == True) and (np.array_equal(array, arrays[-1])): #if we are in the last segment AND in the last line, write to file
                     tempList = write_array(tempList, currentFile, 2, [[0,0],[0,0]])

    segmentFileList = [str(item) + '.segment' for item in depthList]
    for currentFile in segmentFileList:
        print('Delta lambda calculations for: ' + str(currentFile))
        workBuffer = np.loadtxt(currentFile,comments=None)
        sortedWorkBuffer = workBuffer[np.argsort(workBuffer[:,0])] #sort by the first column (wavelenghts)
        #sortedWorkBuffer = workBuffer
        bufferPosition = 0
        deltaLambdaList = np.array([])
        for line in sortedWorkBuffer:
            if bufferPosition <= len(sortedWorkBuffer) - 2:
                deltaLambdaList = np.append(deltaLambdaList, (sortedWorkBuffer[bufferPosition+1][0] - line[0]))
                bufferPosition = bufferPosition + 1
        deltaLambdaList = np.append(deltaLambdaList, 0)
        sortedWorkBuffer = np.c_[sortedWorkBuffer, deltaLambdaList] #append the column with delta lambda values
        np.savetxt(currentFile,sortedWorkBuffer, fmt = '%.6e')

        #sort the files and generate the deltaLambda values for each line
#create a separate file for each depth point as listed in depthList
#iterate over the .lopa files as listed in file_list and fill the .segment files

    print('segmentFileList length: ' + str(len(segmentFileList)))
    return depthList

depthList = merge_files(file_list)
bining(depthList)<|endoftext|>


<commit_before>"""
Copyright 2016 Deepgram

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import logging
from collections import deque
import numpy
from ..utils import get_subclasses, Timer, CriticalSection

logger = logging.getLogger(__name__)

###############################################################################
class Logger:
	""" Base class for loggers.

		Loggers are special callbacks intended to process training statistics
		(such as loss).
	"""

	###########################################################################
	@staticmethod
	def from_specification(spec):
		""" Creates a new evaluation hook from a specification.

			# Arguments

			spec: str or dict. The specification object that was given.

			# Return value

			New EvaluationHook instance.
		"""
		if isinstance(spec, str):
			cls = Logger.DEFAULT_LOGGER
			params = {'path' : spec}
		elif isinstance(spec, dict):
			spec = dict(spec)
			name = spec.pop('name', None)
			if name is None:
				cls = Logger.DEFAULT_LOGGER
			else:
				cls = Logger.get_logger_by_name(name)
			params = spec
		elif spec is None:
			return None
		else:
			raise ValueError('Expected the logger to be a string or '
				'dictionary. We got this instead: {}'.format(spec))

		return cls(**params)

	###########################################################################
	@classmethod
	def get_name(cls):
		""" Returns the name of the logger.

			# Return value

			A lower-case string unique to this logger.
		"""
		return cls.__name__.lower()

	###########################################################################
	@staticmethod
	def get_all_loggers():
		""" Returns an iterator to the names of all logger.
		"""
		for cls in get_subclasses(Logger):
			yield cls

	###########################################################################
	@staticmethod
	def get_logger_by_name(name):
		""" Finds a logger class with the given name.
		"""
		name = name.lower()
		for cls in Logger.get_all_loggers():
			if cls.get_name() == name:
				return cls
		raise ValueError('No such logger with name "{}"'.format(name))

	###########################################################################
	def __init__(self, keep_batch=False, rate=None, **kwargs):
		""" Creates a new logger.

			# Arguments

			keep_batch: bool (default: False). If True, batch-level statistics
				are kept. Otherwise, they are silently discarded. Setting this
				to False can help with logging performance.
			rate: int or None (default: None). If `keep_batch` is True, then
				batch-level statistics will not be written more often than
				once every `rate` seconds (unless an epoch is completed, in
				which case batch statistics are written regardless of the value
				of `rate`, assuming `keep_batch` is True). If `rate` is zero,
				then batch information is written as quickly as possible (every
				batch). If `rate` is None, then, batch information is written
				only every epoch (equivalent to an arbitrarily large value of
				`rate`).
		"""
		if kwargs:
			logger.warning('Unexpected or unsupported arguments to the '
				'logger: %s', ', '.join(str(k) for k in kwargs))

		self.keep_batch = keep_batch
		if rate is not None:
			if not keep_batch:
				logger.warning('Logger rate is only meaningful when '
					'keep_batch is True. Ignoring the rate value.')
				rate = None
			if not isinstance(rate, int):
				raise ValueError('Logger rate must be None or an integer. '
					'Instead, we got: {}'.format(rate))
		self.rate = rate

		self.data = None
		self.batches = 0
		self.epochs = 0
		self.samples = 0
		self.timer = Timer(started=False)

		self._clear()

	###########################################################################
	def _clear(self):
		""" Clears the stored data.
		"""
		self.data = {k : {} for k in ('batch', 'training', 'validation')}

	###########################################################################
	@staticmethod
	def _get_dtype(entry):
		""" Finds an appropriate numpy datatype for the given data value.
		"""
		if isinstance(entry, numpy.ndarray):
			return entry.dtype
		elif isinstance(entry, (list, tuple)):
			if entry:
				return Logger._get_dtype(entry[0])
		elif isinstance(entry, int):
			return numpy.dtype('int32')

		# Floats, empty lists, and "other" types all get mapped to this default.
		return numpy.dtype('float32')

	###########################################################################
	@staticmethod
	def _get_shape(entry):
		""" Determines the shape of a data value.
		"""
		if isinstance(entry, numpy.ndarray):
			return entry.shape
		elif isinstance(entry, (list, tuple)):
			if entry:
				return (len(entry), ) + Logger._get_shape(entry[0])

		return ()

	###########################################################################
	@staticmethod
	def _prepare_like(entry, num_entries):
		""" Allocates a numpy array to hold a certain number of data values.
		"""
		return numpy.empty(
			shape=(num_entries, ) + Logger._get_shape(entry),
			dtype=Logger._get_dtype(entry)
		)

	###########################################################################
	@staticmethod
	def _arrange(data):
		""" Neatly arranges data in a Numpy array.

			# Arguments

			data: deque. Each entry in the deque is a dictionary whose keys name
				the training statistic stored in the respective dictionary
				values.

			# Return value

			Dictionary with the same keys as the entries in `data`, but whose
			values are each an array containing the concatenated values from
			the respective keys in `data`.
		"""
		num_entries = len(data)
		result = None
		for i, entry in enumerate(data):
			if result is None:
				result = {k : Logger._prepare_like(v, num_entries)
					for k, v in entry.items()}
			for k, v in entry.items():
				result[k][i] = v
		return result

	###########################################################################
	def flush(self):
		""" Hook for asking the logger to process log information in its queue.
		"""
		with CriticalSection():
			for key, tags in self.data.items():
				for tag, data in tags.items():
					if not data:
						continue
					data = self._arrange(data)
					self.process(data, key, tag)

			self._clear()
			self.timer.restart()

	###########################################################################
	def _push(self, data_type, tag, data):
		""" Helper function for pushing data into the storage queue.
		"""
		data = dict(data)
		if tag not in self.data[data_type]:
			self.data[data_type][tag] = deque()
		if tag == 'loss':
			if 'total' not in data:
				data['total'] = sum(data.values())
		data['batch'] = self.batches
		self.data[data_type][tag].append(data)

	###########################################################################
	def log_batch(self, batch_size, data, tag=None):
		""" Log training information after a batch.
		"""
		self.samples += batch_size
		self.batches += 1
		if not self.keep_batch:
			return

		self._push('batch', tag, data)

		if not self.timer.started or \
				(self.rate is not None and self.timer.get() > self.rate):
			self.flush()

	###########################################################################
	def log_training(self, data, tag=None):
		""" Log training statistics after an epoch.
		"""
		self.epochs += 1
		self._push('training', tag, data)
		self.flush()

	###########################################################################
	def log_validation(self, data, tag=None):
		""" Log training statistics after a validation run.
		"""
		self._push('validation', tag, data)
		self.flush()

	###########################################################################
	def get_number_of_epochs(self):
		""" Returns the number of epochs this model has historically completed.

			# Return value

			If the number of epochs is known from the logs, it is returned.
			Otherwise, the count is started from zero.
		"""
		return self.epochs

	###########################################################################
	def get_number_of_batches(self):
		""" Returns the number of batches this model has historically
			completed.

			# Return value

			If the number of epochs is known from the logs, it is returned.
			Otherwise, the count is started from zero.
		"""
		return self.batches

	###########################################################################
	def get_number_of_samples(self):
		""" Returns the number of samples this model has historically
			trained on.

			# Return value

			If the number of epochs is known from the logs, it is returned.
			Otherwise, the count is started from zero.
		"""
		return self.samples

	###########################################################################
	def process(self, data, data_type, tag=None):
		""" Processes training statistics.

			This should be implemented in derived classes, and provides the
			primary point for derived classes to receive data. This layer of
			indirection between the `log_X()` functions and this `process()`
			function is an optimization that improves performance by throttling
			the rate at which log data is processed.

			# Arguments

			data: dict. The training statistics dictionary, whose keys name the
				particular statistic, and the values are numpy arrays containing
				the most recent log data.
			data_type: str (one of: "batch", "training", "validation").
				Indicates what stage of training this statistics represent.
			tag: str or None (default: None). A tag to describe the type of data
				provided by `data` (e.g., "loss" or "accuracy").

			# Return value

			None

			# Notes

			- Although the `log_X()` functions accept dictionary values which
			  may be raw floats, `process()` only ever deals with values that
			  are arrays. The base class will collate the information into these
			  arrays so that they can be efficiently parsed by the child
			  classes.
		"""
		raise NotImplementedError

# Need to put this important here to avoid circular dependencies.
from .binary_logger import BinaryLogger	# pylint: disable=wrong-import-position
Logger.DEFAULT_LOGGER = BinaryLogger

### EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF
<commit_msg>Changed default value of keep_batch to True, so all Loggers now record per-batch statistics.<commit_after>"""
Copyright 2016 Deepgram

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import logging
from collections import deque
import numpy
from ..utils import get_subclasses, Timer, CriticalSection

logger = logging.getLogger(__name__)

###############################################################################
class Logger:
	""" Base class for loggers.

		Loggers are special callbacks intended to process training statistics
		(such as loss).
	"""

	###########################################################################
	@staticmethod
	def from_specification(spec):
		""" Creates a new evaluation hook from a specification.

			# Arguments

			spec: str or dict. The specification object that was given.

			# Return value

			New EvaluationHook instance.
		"""
		if isinstance(spec, str):
			cls = Logger.DEFAULT_LOGGER
			params = {'path' : spec}
		elif isinstance(spec, dict):
			spec = dict(spec)
			name = spec.pop('name', None)
			if name is None:
				cls = Logger.DEFAULT_LOGGER
			else:
				cls = Logger.get_logger_by_name(name)
			params = spec
		elif spec is None:
			return None
		else:
			raise ValueError('Expected the logger to be a string or '
				'dictionary. We got this instead: {}'.format(spec))

		return cls(**params)

	###########################################################################
	@classmethod
	def get_name(cls):
		""" Returns the name of the logger.

			# Return value

			A lower-case string unique to this logger.
		"""
		return cls.__name__.lower()

	###########################################################################
	@staticmethod
	def get_all_loggers():
		""" Returns an iterator to the names of all logger.
		"""
		for cls in get_subclasses(Logger):
			yield cls

	###########################################################################
	@staticmethod
	def get_logger_by_name(name):
		""" Finds a logger class with the given name.
		"""
		name = name.lower()
		for cls in Logger.get_all_loggers():
			if cls.get_name() == name:
				return cls
		raise ValueError('No such logger with name "{}"'.format(name))

	###########################################################################
	def __init__(self, keep_batch=True, rate=None, **kwargs):
		""" Creates a new logger.

			# Arguments

			keep_batch: bool (default: True). If True, batch-level statistics
				are kept. Otherwise, they are silently discarded. Setting this
				to False can help with logging performance.
			rate: int or None (default: None). If `keep_batch` is True, then
				batch-level statistics will not be written more often than
				once every `rate` seconds (unless an epoch is completed, in
				which case batch statistics are written regardless of the value
				of `rate`, assuming `keep_batch` is True). If `rate` is zero,
				then batch information is written as quickly as possible (every
				batch). If `rate` is None, then, batch information is written
				only every epoch (equivalent to an arbitrarily large value of
				`rate`).
		"""
		if kwargs:
			logger.warning('Unexpected or unsupported arguments to the '
				'logger: %s', ', '.join(str(k) for k in kwargs))

		self.keep_batch = keep_batch
		if rate is not None:
			if not keep_batch:
				logger.warning('Logger rate is only meaningful when '
					'keep_batch is True. Ignoring the rate value.')
				rate = None
			if not isinstance(rate, int):
				raise ValueError('Logger rate must be None or an integer. '
					'Instead, we got: {}'.format(rate))
		self.rate = rate

		self.data = None
		self.batches = 0
		self.epochs = 0
		self.samples = 0
		self.timer = Timer(started=False)

		self._clear()

	###########################################################################
	def _clear(self):
		""" Clears the stored data.
		"""
		self.data = {k : {} for k in ('batch', 'training', 'validation')}

	###########################################################################
	@staticmethod
	def _get_dtype(entry):
		""" Finds an appropriate numpy datatype for the given data value.
		"""
		if isinstance(entry, numpy.ndarray):
			return entry.dtype
		elif isinstance(entry, (list, tuple)):
			if entry:
				return Logger._get_dtype(entry[0])
		elif isinstance(entry, int):
			return numpy.dtype('int32')

		# Floats, empty lists, and "other" types all get mapped to this default.
		return numpy.dtype('float32')

	###########################################################################
	@staticmethod
	def _get_shape(entry):
		""" Determines the shape of a data value.
		"""
		if isinstance(entry, numpy.ndarray):
			return entry.shape
		elif isinstance(entry, (list, tuple)):
			if entry:
				return (len(entry), ) + Logger._get_shape(entry[0])

		return ()

	###########################################################################
	@staticmethod
	def _prepare_like(entry, num_entries):
		""" Allocates a numpy array to hold a certain number of data values.
		"""
		return numpy.empty(
			shape=(num_entries, ) + Logger._get_shape(entry),
			dtype=Logger._get_dtype(entry)
		)

	###########################################################################
	@staticmethod
	def _arrange(data):
		""" Neatly arranges data in a Numpy array.

			# Arguments

			data: deque. Each entry in the deque is a dictionary whose keys name
				the training statistic stored in the respective dictionary
				values.

			# Return value

			Dictionary with the same keys as the entries in `data`, but whose
			values are each an array containing the concatenated values from
			the respective keys in `data`.
		"""
		num_entries = len(data)
		result = None
		for i, entry in enumerate(data):
			if result is None:
				result = {k : Logger._prepare_like(v, num_entries)
					for k, v in entry.items()}
			for k, v in entry.items():
				result[k][i] = v
		return result

	###########################################################################
	def flush(self):
		""" Hook for asking the logger to process log information in its queue.
		"""
		with CriticalSection():
			for key, tags in self.data.items():
				for tag, data in tags.items():
					if not data:
						continue
					data = self._arrange(data)
					self.process(data, key, tag)

			self._clear()
			self.timer.restart()

	###########################################################################
	def _push(self, data_type, tag, data):
		""" Helper function for pushing data into the storage queue.
		"""
		data = dict(data)
		if tag not in self.data[data_type]:
			self.data[data_type][tag] = deque()
		if tag == 'loss':
			if 'total' not in data:
				data['total'] = sum(data.values())
		data['batch'] = self.batches
		self.data[data_type][tag].append(data)

	###########################################################################
	def log_batch(self, batch_size, data, tag=None):
		""" Log training information after a batch.
		"""
		self.samples += batch_size
		self.batches += 1
		if not self.keep_batch:
			return

		self._push('batch', tag, data)

		if not self.timer.started or \
				(self.rate is not None and self.timer.get() > self.rate):
			self.flush()

	###########################################################################
	def log_training(self, data, tag=None):
		""" Log training statistics after an epoch.
		"""
		self.epochs += 1
		self._push('training', tag, data)
		self.flush()

	###########################################################################
	def log_validation(self, data, tag=None):
		""" Log training statistics after a validation run.
		"""
		self._push('validation', tag, data)
		self.flush()

	###########################################################################
	def get_number_of_epochs(self):
		""" Returns the number of epochs this model has historically completed.

			# Return value

			If the number of epochs is known from the logs, it is returned.
			Otherwise, the count is started from zero.
		"""
		return self.epochs

	###########################################################################
	def get_number_of_batches(self):
		""" Returns the number of batches this model has historically
			completed.

			# Return value

			If the number of epochs is known from the logs, it is returned.
			Otherwise, the count is started from zero.
		"""
		return self.batches

	###########################################################################
	def get_number_of_samples(self):
		""" Returns the number of samples this model has historically
			trained on.

			# Return value

			If the number of epochs is known from the logs, it is returned.
			Otherwise, the count is started from zero.
		"""
		return self.samples

	###########################################################################
	def process(self, data, data_type, tag=None):
		""" Processes training statistics.

			This should be implemented in derived classes, and provides the
			primary point for derived classes to receive data. This layer of
			indirection between the `log_X()` functions and this `process()`
			function is an optimization that improves performance by throttling
			the rate at which log data is processed.

			# Arguments

			data: dict. The training statistics dictionary, whose keys name the
				particular statistic, and the values are numpy arrays containing
				the most recent log data.
			data_type: str (one of: "batch", "training", "validation").
				Indicates what stage of training this statistics represent.
			tag: str or None (default: None). A tag to describe the type of data
				provided by `data` (e.g., "loss" or "accuracy").

			# Return value

			None

			# Notes

			- Although the `log_X()` functions accept dictionary values which
			  may be raw floats, `process()` only ever deals with values that
			  are arrays. The base class will collate the information into these
			  arrays so that they can be efficiently parsed by the child
			  classes.
		"""
		raise NotImplementedError

# Need to put this important here to avoid circular dependencies.
from .binary_logger import BinaryLogger	# pylint: disable=wrong-import-position
Logger.DEFAULT_LOGGER = BinaryLogger

### EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF
<|endoftext|>


<commit_before>"""
You can fit your LikelihoodModel using l1 regularization by changing
    the method argument and adding an argument alpha.  See code for
    details.

The Story
---------
The maximum likelihood (ML) solution works well when the number of data
points is large and the noise is small.  When the ML solution starts
"breaking", the regularized solution should do better.

The l1 Solvers
--------------
The solvers are slower than standard Newton, and sometimes have
    convergence issues Nonetheless, the final solution makes sense and
    is often better than the ML solution.
The standard l1 solver is fmin_slsqp and is included with scipy.  It
    sometimes has trouble verifying convergence when the data size is
    large.
The l1_cvxopt_cp solver is part of CVXOPT and this package needs to be
    installed separately.  It works well even for larger data sizes.
"""
import statsmodels.api as sm
import matplotlib.pyplot as plt
import numpy as np

## Load the data from Spector and Mazzeo (1980)
spector_data = sm.datasets.spector.load()
spector_data.exog = sm.add_constant(spector_data.exog, prepend=True)
N = len(spector_data.endog)
K = spector_data.exog.shape[1]

### Logit Model
logit_mod = sm.Logit(spector_data.endog, spector_data.exog)
## Standard logistic regression
logit_res = logit_mod.fit()
## Regularized regression
# Set the reularization parameter to something reasonable
alpha = 0.05 * N * np.ones(K)
# Use l1, which solves via a built-in (scipy.optimize) solver
logit_l1_res = logit_mod.fit_regularized(method='l1', alpha=alpha, acc=1e-6)
# Use l1_cvxopt_cp, which solves with a CVXOPT solver
logit_l1_cvxopt_res = logit_mod.fit_regularized(
        method='l1_cvxopt_cp', alpha=alpha)
## Print results
print "============ Results for Logit ================="
print "ML results"
print logit_res.summary()
print "l1 results"
print logit_l1_res.summary()
print logit_l1_cvxopt_res.summary()

#### Multinomial Logit Example using American National Election Studies Data
#anes_data = sm.datasets.anes96.load()
#anes_exog = anes_data.exog
#anes_exog[:,0] = np.log(anes_exog[:,0] + .1)
#anes_exog = np.column_stack((anes_exog[:,0],anes_exog[:,2],anes_exog[:,5:8]))
#anes_exog = sm.add_constant(anes_exog, prepend=False)
#mlogit_mod = sm.MNLogit(anes_data.endog, anes_exog)
#mlogit_res = mlogit_mod.fit()
### Set the regularization parameter.
#alpha = 10 * np.ones((mlogit_mod.J - 1, mlogit_mod.K))
## Don't regularize the constant
#alpha[-1,:] = 0
#mlogit_l1_res = mlogit_mod.fit_regularized(
#    method='l1', alpha=alpha, trim_tol=1e-6,
#    start_params=0.0*np.ones(alpha.shape))
#print mlogit_l1_res.params
## CVXOPT isn't up to date
##mlogit_l1_res = mlogit_mod.fit_regularized(
##        method='l1_cvxopt_cp', alpha=alpha, abstol=1e-10, trim_tol=1e-6)
##print mlogit_l1_res.params
### Print results
#print "============ Results for MNLogit ================="
#print "ML results"
#print mlogit_res.summary()
#print "l1 results"
#print mlogit_l1_res.summary()
#
#### Logit example with many params, sweeping alpha
#anes96_data = sm.datasets.anes96.load_pandas()
#X = anes96_data.exog.drop(['vote', 'selfLR'], axis=1)
#Y = anes96_data.exog.vote
### Fit
#N = 100  # number of points to solve at
#K = X.shape[1]
#logit_mod = sm.Logit(Y, X)
#coeff = np.zeros((N, K))  # Holds the coefficients
#alphas = 1 / np.logspace(-2.65, 2, N)
### Sweep alpha and store the coefficients
## QC check doesn't always pass with the default options.
## Use the options QC_verbose=True and disp=True
## to to see what is happening.  It just barely doesn't pass, so I decreased
## acc and increased QC_tol to make it pass
#for n, alpha in enumerate(alphas):
#    logit_res = logit_mod.fit_regularized(
#        method='l1', alpha=alpha, trim_mode='off', QC_tol=0.1, disp=False,
#        QC_verbose=True, acc=1e-15)
#    coeff[n,:] = logit_res.params
### Plot
#plt.figure(1);plt.clf();plt.grid()
#plt.title('Regularization Path');
#plt.xlabel('alpha');
#plt.ylabel('Parameter value');
#for i in xrange(K):
#    plt.plot(alphas, coeff[:,i], label='X'+str(i), lw=3)
#plt.legend(loc='best')
#plt.show()
<commit_msg>checked out short_demo from master<commit_after>"""
You can fit your LikelihoodModel using l1 regularization by changing
    the method argument and adding an argument alpha.  See code for
    details.

The Story
---------
The maximum likelihood (ML) solution works well when the number of data
points is large and the noise is small.  When the ML solution starts
"breaking", the regularized solution should do better.

The l1 Solvers
--------------
The solvers are slower than standard Newton, and sometimes have
    convergence issues Nonetheless, the final solution makes sense and
    is often better than the ML solution.
The standard l1 solver is fmin_slsqp and is included with scipy.  It
    sometimes has trouble verifying convergence when the data size is
    large.
The l1_cvxopt_cp solver is part of CVXOPT and this package needs to be
    installed separately.  It works well even for larger data sizes.
"""
import statsmodels.api as sm
import matplotlib.pyplot as plt
import numpy as np

## Load the data from Spector and Mazzeo (1980)
spector_data = sm.datasets.spector.load()
spector_data.exog = sm.add_constant(spector_data.exog, prepend=True)
N = len(spector_data.endog)
K = spector_data.exog.shape[1]

### Logit Model
logit_mod = sm.Logit(spector_data.endog, spector_data.exog)
## Standard logistic regression
logit_res = logit_mod.fit()
## Regularized regression
# Set the reularization parameter to something reasonable
alpha = 0.05 * N * np.ones(K)
# Use l1, which solves via a built-in (scipy.optimize) solver
logit_l1_res = logit_mod.fit_regularized(method='l1', alpha=alpha, acc=1e-6)
# Use l1_cvxopt_cp, which solves with a CVXOPT solver
logit_l1_cvxopt_res = logit_mod.fit_regularized(
        method='l1_cvxopt_cp', alpha=alpha)
## Print results
print "============ Results for Logit ================="
print "ML results"
print logit_res.summary()
print "l1 results"
print logit_l1_res.summary()
print logit_l1_cvxopt_res.summary()

### Multinomial Logit Example using American National Election Studies Data
anes_data = sm.datasets.anes96.load()
anes_exog = anes_data.exog
anes_exog[:,0] = np.log(anes_exog[:,0] + .1)
anes_exog = np.column_stack((anes_exog[:,0],anes_exog[:,2],anes_exog[:,5:8]))
anes_exog = sm.add_constant(anes_exog, prepend=False)
mlogit_mod = sm.MNLogit(anes_data.endog, anes_exog)
mlogit_res = mlogit_mod.fit()
## Set the regularization parameter.
alpha = 10 * np.ones((mlogit_mod.J - 1, mlogit_mod.K))
# Don't regularize the constant
alpha[-1,:] = 0
mlogit_l1_res = mlogit_mod.fit_regularized(
    method='l1', alpha=alpha, trim_tol=1e-6,
    start_params=0.0*np.ones(alpha.shape))
print mlogit_l1_res.params
# CVXOPT isn't up to date
#mlogit_l1_res = mlogit_mod.fit_regularized(
#        method='l1_cvxopt_cp', alpha=alpha, abstol=1e-10, trim_tol=1e-6)
#print mlogit_l1_res.params
## Print results
print "============ Results for MNLogit ================="
print "ML results"
print mlogit_res.summary()
print "l1 results"
print mlogit_l1_res.summary()
#
#### Logit example with many params, sweeping alpha
#anes96_data = sm.datasets.anes96.load_pandas()
#X = anes96_data.exog.drop(['vote', 'selfLR'], axis=1)
#Y = anes96_data.exog.vote
### Fit
#N = 100  # number of points to solve at
#K = X.shape[1]
#logit_mod = sm.Logit(Y, X)
#coeff = np.zeros((N, K))  # Holds the coefficients
#alphas = 1 / np.logspace(-2.65, 2, N)
### Sweep alpha and store the coefficients
## QC check doesn't always pass with the default options.
## Use the options QC_verbose=True and disp=True
## to to see what is happening.  It just barely doesn't pass, so I decreased
## acc and increased QC_tol to make it pass
#for n, alpha in enumerate(alphas):
#    logit_res = logit_mod.fit_regularized(
#        method='l1', alpha=alpha, trim_mode='off', QC_tol=0.1, disp=False,
#        QC_verbose=True, acc=1e-15)
#    coeff[n,:] = logit_res.params
### Plot
#plt.figure(1);plt.clf();plt.grid()
#plt.title('Regularization Path');
#plt.xlabel('alpha');
#plt.ylabel('Parameter value');
#for i in xrange(K):
#    plt.plot(alphas, coeff[:,i], label='X'+str(i), lw=3)
#plt.legend(loc='best')
#plt.show()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8
#   Python Script
#
#   Copyright © Manoel Vilela
#
#

from __future__ import print_function
import os
import pygame
from labyrinth_generator import generate
from random import randint
from time import time

BLOCKSIZE = 16
WIDTH = 590
HEIGHT = 480

WHITE = (255, 255, 255)
YELLOW = (255, 200, 0)
RED = (255, 0, 0)
GREEN = (0, 255, 0)
BLACK = (0, 0, 0)

def safe_position(level):  # ??
    a, b = len(level) - 2, len(level[0]) - 1
    x, y = (randint(1, a), randint(1, b))
    if level[x][y] != ' ':
        print('Not safe', x, y, '->', level[x][y])
        x, y = safe_position(level)
    return (min(1, x + 1), y)

# Class for the orange dude
class Player(object):

    def __init__(self, x, y):
        self.rect = pygame.Rect(x, y, BLOCKSIZE, BLOCKSIZE)

    def move(self, dx, dy):

        # Move each axis separately.
        # Note that this checks for collisions both times.
        if dx != 0:
            self.move_single_axis(dx, 0)
        if dy != 0:
            self.move_single_axis(0, dy)

    def move_single_axis(self, dx, dy):

        # Move the rect
        self.rect.x += dx
        self.rect.y += dy

        # If you collide with a wall, move out based on velocity
        for wall in walls:
            if self.rect.colliderect(wall.rect):
                if dx > 0:  # Moving right; Hit the left side of the wall
                    self.rect.right = wall.rect.left
                if dx < 0:  # Moving left; Hit the right side of the wall
                    self.rect.left = wall.rect.right
                if dy > 0:  # Moving down; Hit the top side of the wall
                    self.rect.bottom = wall.rect.top
                if dy < 0:  # Moving up; Hit the bottom side of the wall
                    self.rect.top = wall.rect.bottom

# Nice class to hold a wall rect
class Wall(object):
    def __init__(self, pos):
        walls.append(self)
        self.rect = pygame.Rect(pos[0], pos[1], BLOCKSIZE, BLOCKSIZE)

def draw_counter(time):
    text = pygame.font.Font('freesansbold.ttf', BLOCKSIZE)
    text_surf = text.render('Time: {:04.1f}'.format(time), True, WHITE)
    text_rect = text_surf.get_rect()
    text_rect.midright = (WIDTH - 40, HEIGHT - BLOCKSIZE // 2)
    screen.blit(text_surf, text_rect)

def setup():
    global walls, player, clock, end_rect, screen
    # Initialise pygame
    os.environ["SDL_VIDEO_CENTERED"] = "1"
    pygame.init()

    # Set up the display
    pygame.display.set_caption("Get to the red square!")
    screen = pygame.display.set_mode((WIDTH, HEIGHT))

    clock = pygame.time.Clock()
    walls = []  # List to hold the walls
    level = generate()
    x, y = safe_position(level)
    player = Player(x * BLOCKSIZE, y * BLOCKSIZE)  # create the player

    # Holds the level layout in a list of strings.

    # Parse the level string above. W = wall, E = exit
    x = y = 0
    for row in level:
        for col in row:
            if col == "W":
                Wall((x, y))
            if col == "E":
                end_rect = pygame.Rect(x, y, BLOCKSIZE, BLOCKSIZE)
            x += BLOCKSIZE
        y += BLOCKSIZE
        x = 0

    return screen, clock, walls, player, end_rect

def game():
    setup()
    running = True
    start = time()
    while running:

        clock.tick(60)

        for e in pygame.event.get():
            if e.type == pygame.QUIT:
                running = False
            if e.type == pygame.KEYDOWN and e.key == pygame.K_ESCAPE:
                running = False

        # Move the player if an arrow key is pressed
        key = pygame.key.get_pressed()
        if key[pygame.K_LEFT]:
            player.move(-2, 0)
        if key[pygame.K_RIGHT]:
            player.move(2, 0)
        if key[pygame.K_UP]:
            player.move(0, -2)
        if key[pygame.K_DOWN]:
            player.move(0, 2)

        # Just added this to make it slightly fun ;)
        if player.rect.colliderect(end_rect):
            raise SystemExit("You win! @{:2f}s".format(time() - start))
        # Draw the scene
        screen.fill(BLACK)
        for wall in walls:
            pygame.draw.rect(screen, WHITE, wall.rect)
        pygame.draw.rect(screen, RED, end_rect)
        pygame.draw.rect(screen, GREEN, player.rect)
        draw_counter(time() - start)
        pygame.display.flip()

if __name__ == '__main__':
    game()
<commit_msg>Add crazy AI<commit_after>#!/usr/bin/env python
# coding=utf-8
#   Python Script
#
#   Copyright © Manoel Vilela
#
#

from __future__ import print_function
import os
import pygame
from labyrinth_generator import generate
from random import randint
from time import time

BLOCKSIZE = 16
WIDTH = 590
HEIGHT = 480

WHITE = (255, 255, 255)
YELLOW = (255, 200, 0)
RED = (255, 0, 0)
GREEN = (0, 255, 0)
BLACK = (0, 0, 0)

def safe_position(level):  # ??
    a, b = len(level) - 2, len(level[0]) - 1
    x, y = (randint(1, a), randint(1, b))
    if level[x][y] != ' ':
        print('Not safe', x, y, '->', level[x][y])
        x, y = safe_position(level)
    return (min(1, x + 1), y)

# Class for the orange dude
class Player(object):

    def __init__(self, x, y):
        self.rect = pygame.Rect(x, y, BLOCKSIZE, BLOCKSIZE)

    def move(self, dx, dy):

        # Move each axis separately.
        # Note that this checks for collisions both times.
        if dx != 0:
            self.move_single_axis(dx, 0)
        if dy != 0:
            self.move_single_axis(0, dy)

    def move_single_axis(self, dx, dy):

        # Move the rect
        self.rect.x += dx
        self.rect.y += dy

        # If you collide with a wall, move out based on velocity
        for wall in walls:
            if self.rect.colliderect(wall.rect):
                if dx > 0:  # Moving right; Hit the left side of the wall
                    self.rect.right = wall.rect.left
                if dx < 0:  # Moving left; Hit the right side of the wall
                    self.rect.left = wall.rect.right
                if dy > 0:  # Moving down; Hit the top side of the wall
                    self.rect.bottom = wall.rect.top
                if dy < 0:  # Moving up; Hit the bottom side of the wall
                    self.rect.top = wall.rect.bottom

    @property
    def position(self):
        return self.rect.x, self.rect.y

class AI(Player):
    memo = {}
    path = {}

    dx = 2
    dy = 2

    def run(self):
        self.remember()
        self.walk()
        self.forget_it()

    def remember(self):
        self.path[time()] = self.position
        self.memo.setdefault(self.position, 0).__add__(1)

    def walk(self):
        motion = self.decide()
        self.move(*motion)

    def decide(self):
        if self.memo[self.position] > 10:
            self.change()

        if len(self.path) > 2:
            t1, t2 = sorted(self.path, reverse=True)[:2]
            (x1, y1), (x2, y2) = self.path[t1], self.path[t2]
            distance = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
            if distance == 0:
                self.change()

        return self.dx, self.dy

    def forget_it(self):
        if len(self.path) > 3:
            del self.path[min(self.path)]

    def change(self):
        from random import choice
        dice = choice([0, 1])

        if dice:
            self.dx *= -1
        else:
            self.dy *= -1

# Nice class to hold a wall rect
class Wall(object):
    def __init__(self, pos):
        walls.append(self)
        self.rect = pygame.Rect(pos[0], pos[1], BLOCKSIZE, BLOCKSIZE)

def draw_counter(time):
    text = pygame.font.Font('freesansbold.ttf', BLOCKSIZE)
    text_surf = text.render('Time: {:04.1f}'.format(time), True, WHITE)
    text_rect = text_surf.get_rect()
    text_rect.midright = (WIDTH - 40, HEIGHT - BLOCKSIZE // 2)
    screen.blit(text_surf, text_rect)

def setup():
    global walls, player, clock, end_rect, screen, computer
    # Initialise pygame
    os.environ["SDL_VIDEO_CENTERED"] = "1"
    pygame.init()

    # Set up the display
    pygame.display.set_caption("Get to the red square!")
    screen = pygame.display.set_mode((WIDTH, HEIGHT))

    clock = pygame.time.Clock()
    walls = []  # List to hold the walls
    level = generate()
    x, y = safe_position(level)
    player = Player(x * BLOCKSIZE, y * BLOCKSIZE)  # create the player
    x, y = safe_position(level)
    computer = AI(x * BLOCKSIZE, y * BLOCKSIZE)

    # Holds the level layout in a list of strings.

    # Parse the level string above. W = wall, E = exit
    x = y = 0
    for row in level:
        for col in row:
            if col == "W":
                Wall((x, y))
            if col == "E":
                end_rect = pygame.Rect(x, y, BLOCKSIZE, BLOCKSIZE)
            x += BLOCKSIZE
        y += BLOCKSIZE
        x = 0

    return screen, clock, walls, player, end_rect

def game():
    setup()
    running = True
    start = time()
    while running:

        clock.tick(60)

        for e in pygame.event.get():
            if e.type == pygame.QUIT:
                running = False
            if e.type == pygame.KEYDOWN and e.key == pygame.K_ESCAPE:
                running = False

        # Move the player if an arrow key is pressed
        key = pygame.key.get_pressed()
        if key[pygame.K_LEFT]:
            player.move(-2, 0)
        if key[pygame.K_RIGHT]:
            player.move(2, 0)
        if key[pygame.K_UP]:
            player.move(0, -2)
        if key[pygame.K_DOWN]:
            player.move(0, 2)

        # Just added this to make it slightly fun ;)
        if player.rect.colliderect(end_rect):
            raise SystemExit("You win! @{:2f}s".format(time() - start))
        # Draw the scene
        screen.fill(BLACK)
        computer.run()
        for wall in walls:
            pygame.draw.rect(screen, WHITE, wall.rect)
        pygame.draw.rect(screen, RED, end_rect)
        pygame.draw.rect(screen, GREEN, player.rect)
        pygame.draw.rect(screen, YELLOW, computer.rect)
        draw_counter(time() - start)
        pygame.display.flip()

if __name__ == '__main__':
    game()
<|endoftext|>


<commit_before>from gpiozero import Button
from time import sleep
from datetime import datetime
import random
import sqlite3
from signal import pause

print('initializing variables')
like = Button(18)
okay = Button(14)
dislike = Button(15)

questions = ['Did you enjoy your visit today?', 'Would you reccomend us to a friend?', 'Were you satisfied with the service you received today', 'Were you able to find what you were looking for?']

#pull questions from csv file on dropbox, run daily at midnight. From a different script. Q's stored as CSV.

def add_response_to_database():
    """Add response to SQLite 3 database"""

    sqlite_file = 'srvy.db'
    table_name = 'responses'
    date_column = 'date'
    time_column = 'time'
    score_column = 'score'
    question_column = 'question'

    conn = sqlite3.connect(sqlite_file)
    c = conn.cursor()

    current_date = datetime.now().strftime('%Y-%m-%d')
    current_time = datetime.now().strftime('%H:%M:%S')

    try:
        c.execute('''INSERT INTO responses (date, time, score) VALUES (?,?,?,?)''', (current_date, current_time, score, question))
        print ("Successfully added response to database.")
    except Exception as e:
        print(e)

    conn.commit()
    conn.close()

def main():

    qs = random.choice(questions)

    print(qs)

    while True:
        if like.is_pressed:
            score = 2
            question = qs
            sleep(.5)
            add_response_to_database()

        elif okay.is_pressed:
            score = 1
            question = qs
            sleep(.5)
            add_response_to_database()

        elif dislike.is_pressed:
            score = 0
            question = qs
            sleep(.5)
            add_response_to_database()

main()

<commit_msg>Initialize score<commit_after>from gpiozero import Button
from time import sleep
from datetime import datetime
import random
import sqlite3
from signal import pause

print('initializing variables')
like = Button(18)
okay = Button(14)
dislike = Button(15)

questions = ['Did you enjoy your visit today?', 'Would you reccomend us to a friend?', 'Were you satisfied with the service you received today', 'Were you able to find what you were looking for?']

score = 0

#pull questions from csv file on dropbox, run daily at midnight. From a different script. Q's stored as CSV.

def add_response_to_database():
    """Add response to SQLite 3 database"""

    sqlite_file = 'srvy.db'
    table_name = 'responses'
    date_column = 'date'
    time_column = 'time'
    score_column = 'score'
    question_column = 'question'

    conn = sqlite3.connect(sqlite_file)
    c = conn.cursor()

    current_date = datetime.now().strftime('%Y-%m-%d')
    current_time = datetime.now().strftime('%H:%M:%S')

    try:
        c.execute('''INSERT INTO responses (date, time, score) VALUES (?,?,?,?)''', (current_date, current_time, score, question))
        print ("Successfully added response to database.")
    except Exception as e:
        print(e)

    conn.commit()
    conn.close()

def main():

    qs = random.choice(questions)

    print(qs)

    while True:
        if like.is_pressed:
            score = 2
            question = qs
            sleep(.5)
            add_response_to_database()

        elif okay.is_pressed:
            score = 1
            question = qs
            sleep(.5)
            add_response_to_database()

        elif dislike.is_pressed:
            score = 0
            question = qs
            sleep(.5)
            add_response_to_database()

main()

<|endoftext|>


<commit_before>#!/usr/bin/env python3
import os
import sys
import yaml
import mistune
from jinja2 import Environment, FileSystemLoader

ALLOWED = {'.md', '.mkd', '.markdown'}
is_allowed = lambda x: os.path.splitext(x)[1] in ALLOWED
IGNORED = {'README.md'}
is_ignored = lambda x: x in IGNORED
DIR_IGNORED = {'_', '.'} #+ set(config.get('dir_ignored', []))
dir_is_ignored = lambda x: any(os.path.basename(x).startswith(y) for y in DIR_IGNORED)

absjoin = lambda x, y: os.path.abspath(os.path.join(x, y))
ROOT_DIR = os.getcwd()
OUT_DIR = absjoin(ROOT_DIR, '_site')
TEMPLATE_DIR = absjoin(ROOT_DIR, '_layouts')

config = yaml.load(open(absjoin(ROOT_DIR, '_config.yml')).read())
build_dirs_name = config.get('build_dirs', 'blog')
build_dirs = absjoin(ROOT_DIR, build_dirs_name)
default_template = config.get('layout', 'post.html')

jinja_loader = FileSystemLoader(TEMPLATE_DIR)
jinja_env = Environment(loader=jinja_loader)
jinja_env.filters['datetimeformat'] = lambda x, y: x.strftime(y)

def extract(fpath):
    meta, content, first_line, meta_parsed = [], [], True, False
    with open(fpath) as fp:
        for line in fp:
            if line.strip() == '---' and not meta_parsed:
                if not first_line:
                    meta_parsed = True
                first_line = False
            elif not meta_parsed:
                meta.append(line)
            else:
                content.append(line)
        try:
            return yaml.load('\n'.join(meta)), '\n'.join(content)
        except:
            raise SystemExit('File with invalid yaml meta block: ' + fpath)

def build(markdown):
    for root, dirs, files in os.walk(ROOT_DIR):
        if dir_is_ignored(root): continue
        for fname in files:
            fpath = absjoin(root, fname)
            if is_allowed(fname) and not is_ignored(fname):
                meta, content = extract(fpath)
                html = markdown(content)
                template = meta.get('layout', default_template)
                templater = jinja_env.get_template(template)
                info = config.copy()
                info['content'] = html
                info.update(meta)
                with open(absjoin(root, os.path.splitext(fname)[0] + '.html'), 'w') as fp:
                    fp.write(templater.render(info))

def main():
    markdown = mistune.Markdown()
    build(markdown)

if __name__ == '__main__':
    main()
<commit_msg>Back to 64 lines<commit_after>#!/usr/bin/env python3
import os, sys, yaml, mistune
from jinja2 import Environment, FileSystemLoader

ALLOWED = {'.md', '.mkd', '.markdown'}
is_allowed = lambda x: os.path.splitext(x)[1] in ALLOWED
IGNORED = {'README.md'}
is_ignored = lambda x: x in IGNORED
DIR_IGNORED = {'_', '.'} #+ set(config.get('dir_ignored', []))
dir_is_ignored = lambda x: any(os.path.basename(x).startswith(y) for y in DIR_IGNORED)

absjoin = lambda x, y: os.path.abspath(os.path.join(x, y))
ROOT_DIR = os.getcwd()
OUT_DIR = absjoin(ROOT_DIR, '_site')
TEMPLATE_DIR = absjoin(ROOT_DIR, '_layouts')

config = yaml.load(open(absjoin(ROOT_DIR, '_config.yml')).read())
build_dirs_name = config.get('build_dirs', 'blog')
build_dirs = absjoin(ROOT_DIR, build_dirs_name)
default_template = config.get('layout', 'post.html')

jinja_loader = FileSystemLoader(TEMPLATE_DIR)
jinja_env = Environment(loader=jinja_loader)
jinja_env.filters['datetimeformat'] = lambda x, y: x.strftime(y)

def extract(fpath):
    meta, content, first_line, meta_parsed = [], [], True, False
    with open(fpath) as fp:
        for line in fp:
            if line.strip() == '---' and not meta_parsed:
                if not first_line:
                    meta_parsed = True
                first_line = False
            elif not meta_parsed:
                meta.append(line)
            else:
                content.append(line)
        try:
            return yaml.load('\n'.join(meta)), '\n'.join(content)
        except:
            raise SystemExit('File with invalid yaml meta block: ' + fpath)

def build(markdown):
    for root, dirs, files in os.walk(ROOT_DIR):
        if dir_is_ignored(root): continue
        for fname in files:
            fpath = absjoin(root, fname)
            if is_allowed(fname) and not is_ignored(fname):
                meta, content = extract(fpath)
                html = markdown(content)
                template = meta.get('layout', default_template)
                templater = jinja_env.get_template(template)
                info = config.copy()
                info['content'] = html
                info.update(meta)
                with open(absjoin(root, os.path.splitext(fname)[0] + '.html'), 'w') as fp:
                    fp.write(templater.render(info))

def main():
    markdown = mistune.Markdown()
    build(markdown)

if __name__ == '__main__':
    main()
<|endoftext|>