<commit_before>import os
import tornado.httpserver
import tornado.ioloop
import tornado.web
from creds import *
from requests import Request
import requests
import json
import re
import tempfile
import redis
import uuid
import random
import string
import ast
from pydub import AudioSegment

def gettoken(uid):
	red = redis.from_url(redis_url)
	token = red.get(uid+"-access_token")
	refresh = red.get(uid+"-refresh_token")
	if token:
		return token
	elif refresh:
		payload = {"client_id" : Client_ID, "client_secret" : Client_Secret, "refresh_token" : refresh, "grant_type" : "refresh_token", }
		url = "https://api.amazon.com/auth/o2/token"
		r = requests.post(url, data = payload)
		resp = json.loads(r.text)
		red.set(uid+"-access_token", resp['access_token'])
		red.expire(uid+"-access_token", 3600)
		return resp['access_token']
	else:
		return False

def settrigger(user):
	red = redis.from_url(redis_url)
	self.set(user + "-trigger", True)

class BaseHandler(tornado.web.RequestHandler):
    def get_current_user(self):
        return self.get_cookie("user")

class MainHandler(BaseHandler):
	@tornado.web.authenticated
	@tornado.web.asynchronous
	def get(self):
		f = open('static/index.html', 'r')
		resp = f.read()
		f.close()
		self.write(resp)
		self.finish()

class QuestionHandler(BaseHandler):
	# @tornado.web.authenticated
	@tornado.web.asynchronous
	def get(self):
		red = redis.from_url(redis_url)
		uid = self.get_argument("access_token")
		print uid
		res = red.get(uid + "-questions")
		res = res if res != None else ""
		self.write(res)
		self.finish()

	@tornado.web.asynchronous
	def post(self):
		red = redis.from_url(redis_url)
		uid = self.get_argument("access_token")
		print uid
		red.set(uid + "-questions", self.get_argument("questions"))
		print self.get_argument("questions")
		self.write({"success": True})
		self.finish()

class TriggerHandler(BaseHandler):
	@tornado.web.authenticated
	@tornado.web.asynchronous
	def get(self):
		red = redis.from_url(redis_url)
		# print self.get_current_user(), "=>", red.get(self.get_current_user() + "-trigger")
		status = red.get(self.get_current_user() + "-trigger")
		status = True if status == "True" else False
		self.write({"trigger": status})
		red.set(self.get_current_user() + "-trigger", False)
		self.finish()

class StartAuthHandler(tornado.web.RequestHandler):
	@tornado.web.asynchronous
	def get(self):
		sd = json.dumps({
		    "alexa:all": {
		        "productID": Product_ID,
		        "productInstanceAttributes": {
		            "deviceSerialNumber": "1"
		        }
		    }
		})
		url = "https://www.amazon.com/ap/oa"
		path = self.request.protocol + "://" + self.request.host
		callback = path + "/code"
		payload = {"client_id" : Client_ID, "scope" : "alexa:all profile:user_id", "scope_data" : sd, "response_type" : "code", "redirect_uri" : callback }
		req = Request('GET', url, params=payload)
		p = req.prepare()
		self.redirect(p.url)

class CodeAuthHandler(tornado.web.RequestHandler):
	@tornado.web.asynchronous
	def get(self):
		code=self.get_argument("code")
		path = self.request.protocol + "://" + self.request.host
		callback = path+"/code"
		payload = {"client_id" : Client_ID, "client_secret" : Client_Secret, "code" : code, "grant_type" : "authorization_code", "redirect_uri" : callback }
		url = "https://api.amazon.com/auth/o2/token"
		r = requests.post(url, data = payload)
		uid = str(uuid.uuid4())
		red = redis.from_url(redis_url)
		resp = json.loads(r.text)
		print resp
		red.set(uid+"-access_token", resp['access_token'])
		red.expire(uid+"-access_token", 3600)
		red.set(uid+"-refresh_token", resp['refresh_token'])
		self.set_cookie("user", uid)
		self.redirect("/")

class LogoutHandler(BaseHandler):
	@tornado.web.authenticated
	@tornado.web.asynchronous
	def get(self):
		uid = tornado.escape.xhtml_escape(self.current_user)
		red = redis.from_url(redis_url)
		red.delete(uid+"-access_token")
		red.delete(uid+"-refresh_token")
		self.clear_cookie("user")
		self.set_header('Content-Type', 'text/plain')
		self.write("Logged Out, Goodbye")
		self.finish()

class AudioHandler(BaseHandler):
	@tornado.web.authenticated
	@tornado.web.asynchronous
	def post(self):
		uid = tornado.escape.xhtml_escape(self.current_user)
		token = gettoken(uid)
		if (token == False):
			self.set_status(403)
		else:
			rxfile = self.request.files['data'][0]['body']
			tf = tempfile.NamedTemporaryFile(suffix=".wav")
			tf.write(rxfile)
			_input = AudioSegment.from_wav(tf.name)
			tf.close()
			tf = tempfile.NamedTemporaryFile(suffix=".wav")
			output = _input.set_channels(1).set_frame_rate(16000)
			f = output.export(tf.name, format="wav")
			url = 'https://access-alexa-na.amazon.com/v1/avs/speechrecognizer/recognize'
			headers = {'Authorization' : 'Bearer %s' % token}
			d = {
		    	"messageHeader": {
		        	"deviceContext": [
		            	{
		                	"name": "playbackState",
		                	"namespace": "AudioPlayer",
		                	"payload": {
		                    	"streamId": "",
		         			   	"offsetInMilliseconds": "0",
		                    	"playerActivity": "IDLE"
		                	}
		            	}
		        	]
		    	},
		    	"messageBody": {
		        	"profile": "alexa-close-talk",
		        	"locale": "en-us",
		        	"format": "audio/L16; rate=16000; channels=1"
		    	}
			}
			files = [
				('file', ('request', json.dumps(d), 'application/json; charset=UTF-8')),
				('file', ('audio', tf, 'audio/L16; rate=16000; channels=1'))
			]
			r = requests.post(url, headers=headers, files=files)
			tf.close()

			content_type = None

			try:
				content_type = r.headers['Content-Type']
			except Exception as e:
				content_type = r.headers['content-type']

			for v in r.headers['content-type'].split(";"):
				if re.match(r'.*boundary.*', v):
					boundary =  v.split("=")[1]
			data = r.content.split(boundary)
			for d in data:
				if (len(d) >= 1024):
			 	   audio = d.split('\r\n\r\n')[1].rstrip('--')
			self.set_header('Content-Type', 'audio/mpeg')
			self.write(audio)
		self.finish()

class TextHandler(BaseHandler):
	@tornado.web.authenticated
	@tornado.web.asynchronous
	def get(self):
		text = self.get_argument("text", None, True)

		uid = tornado.escape.xhtml_escape(self.current_user)
		token = gettoken(uid)
		if (token == False):
			self.set_status(403)
		else:
			# rxfile = self.request.files['data'][0]['body']
			# tf = tempfile.NamedTemporaryFile(suffix=".wav")
			# tf.write(rxfile)
			# _input = AudioSegment.from_wav(tf.name)
			# tf.close()
			# tf = tempfile.NamedTemporaryFile(suffix=".wav")
			# output = _input.set_channels(1).set_frame_rate(16000)
			# f = output.export(tf.name, format="wav")

			random_str = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(20))
			file_name = "/tmp/{}.wav".format(random_str)
			cmd = 'espeak "{}" --stdout > {}'.format(text, file_name)

			print text
			print file_name
			print cmd

			os.system(cmd)

			_input = AudioSegment.from_wav(file_name)
                        tf = tempfile.NamedTemporaryFile(suffix=".wav")
                        output = _input.set_channels(1).set_frame_rate(16000)
                        f = output.export(tf.name, format="wav")

			#tf = open(file_name)

			url = 'https://access-alexa-na.amazon.com/v1/avs/speechrecognizer/recognize'
			headers = {'Authorization' : 'Bearer %s' % token}
			d = {
		    	"messageHeader": {
		        	"deviceContext": [
		            	{
		                	"name": "playbackState",
		                	"namespace": "AudioPlayer",
		                	"payload": {
		                    	"streamId": "",
		         			   	"offsetInMilliseconds": "0",
		                    	"playerActivity": "IDLE"
		                	}
		            	}
		        	]
		    	},
		    	"messageBody": {
		        	"profile": "alexa-close-talk",
		        	"locale": "en-us",
		        	"format": "audio/L16; rate=16000; channels=1"
		    	}
			}
			files = [
				('file', ('request', json.dumps(d), 'application/json; charset=UTF-8')),
				('file', ('audio', tf, 'audio/L16; rate=16000; channels=1'))
			]
			r = requests.post(url, headers=headers, files=files)
			tf.close()
			print r
			for v in r.headers['content-type'].split(";"):
				if re.match('.*boundary.*', v):
					boundary =  v.split("=")[1]
			data = r.content.split(boundary)
			for d in data:
				if (len(d) >= 1024):
			 	   audio = d.split('\r\n\r\n')[1].rstrip('--')
			self.set_header('Content-Type', 'audio/mpeg')
			self.write(audio)
		self.finish()

def main():
	settings = {
	    "cookie_secret": "parisPOLANDbroadFENCEcornWOULD",
	    "login_url": "/static/welcome.html",
	}
	static_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static')
	application = tornado.web.Application([(r"/", MainHandler),
											(r"/start", StartAuthHandler),
											(r"/code", CodeAuthHandler),
											(r"/logout", LogoutHandler),
											(r"/audio", AudioHandler),
											(r"/trigger", TriggerHandler),
											(r"/questions", QuestionHandler),
											(r"/text", TextHandler),
											(r'/(favicon.ico)', tornado.web.StaticFileHandler,{'path': static_path}),
											(r'/static/(.*)', tornado.web.StaticFileHandler, {'path': static_path}),
											], **settings)
	http_server = tornado.httpserver.HTTPServer(application)
	port = int(os.environ.get("PORT", 8080))
	http_server.listen(port)
	tornado.ioloop.IOLoop.instance().start()

if __name__ == "__main__":
	main()

<commit_msg> stupid bug<commit_after>import os
import tornado.httpserver
import tornado.ioloop
import tornado.web
from creds import *
from requests import Request
import requests
import json
import re
import tempfile
import redis
import uuid
import random
import string
import ast
from pydub import AudioSegment

def gettoken(uid):
	red = redis.from_url(redis_url)
	token = red.get(uid+"-access_token")
	refresh = red.get(uid+"-refresh_token")
	if token:
		return token
	elif refresh:
		payload = {"client_id" : Client_ID, "client_secret" : Client_Secret, "refresh_token" : refresh, "grant_type" : "refresh_token", }
		url = "https://api.amazon.com/auth/o2/token"
		r = requests.post(url, data = payload)
		resp = json.loads(r.text)
		red.set(uid+"-access_token", resp['access_token'])
		red.expire(uid+"-access_token", 3600)
		return resp['access_token']
	else:
		return False

def settrigger(user):
	red = redis.from_url(redis_url)
	self.set(user + "-trigger", True)

class BaseHandler(tornado.web.RequestHandler):
    def get_current_user(self):
        return self.get_cookie("user")

class MainHandler(BaseHandler):
	@tornado.web.authenticated
	@tornado.web.asynchronous
	def get(self):
		f = open('static/index.html', 'r')
		resp = f.read()
		f.close()
		self.write(resp)
		self.finish()

class QuestionHandler(BaseHandler):
	# @tornado.web.authenticated
	@tornado.web.asynchronous
	def get(self):
		red = redis.from_url(redis_url)
		uid = self.get_argument("access_token")
		print uid
		res = red.get(uid + "-questions")
		res = res if res != None else ""
		self.write(res)
		self.finish()

	@tornado.web.asynchronous
	def post(self):
		red = redis.from_url(redis_url)
		uid = self.get_argument("access_token")
		print uid
		red.set(uid + "-questions", self.get_argument("questions"))
		print self.get_argument("questions")
		self.write({"success": True})
		self.finish()

class TriggerHandler(BaseHandler):
	@tornado.web.authenticated
	@tornado.web.asynchronous
	def get(self):
		red = redis.from_url(redis_url)
		# print self.get_current_user(), "=>", red.get(self.get_current_user() + "-trigger")
		status = red.get(self.get_current_user() + "-trigger")
		status = True if status == "True" else False
		self.write({"trigger": status})
		red.set(self.get_current_user() + "-trigger", False)
		self.finish()

class StartAuthHandler(tornado.web.RequestHandler):
	@tornado.web.asynchronous
	def get(self):
		sd = json.dumps({
		    "alexa:all": {
		        "productID": Product_ID,
		        "productInstanceAttributes": {
		            "deviceSerialNumber": "1"
		        }
		    }
		})
		url = "https://www.amazon.com/ap/oa"
		path = self.request.protocol + "://" + self.request.host
		callback = path + "/code"
		payload = {"client_id" : Client_ID, "scope" : "alexa:all profile:user_id", "scope_data" : sd, "response_type" : "code", "redirect_uri" : callback }
		req = Request('GET', url, params=payload)
		p = req.prepare()
		self.redirect(p.url)

class CodeAuthHandler(tornado.web.RequestHandler):
	@tornado.web.asynchronous
	def get(self):
		code=self.get_argument("code")
		path = self.request.protocol + "://" + self.request.host
		callback = path+"/code"
		payload = {"client_id" : Client_ID, "client_secret" : Client_Secret, "code" : code, "grant_type" : "authorization_code", "redirect_uri" : callback }
		url = "https://api.amazon.com/auth/o2/token"
		r = requests.post(url, data = payload)
		uid = str(uuid.uuid4())
		red = redis.from_url(redis_url)
		resp = json.loads(r.text)
		print resp
		red.set(uid+"-access_token", resp['access_token'])
		red.expire(uid+"-access_token", 3600)
		red.set(uid+"-refresh_token", resp['refresh_token'])
		self.set_cookie("user", uid)
		self.redirect("/")

class LogoutHandler(BaseHandler):
	@tornado.web.authenticated
	@tornado.web.asynchronous
	def get(self):
		uid = tornado.escape.xhtml_escape(self.current_user)
		red = redis.from_url(redis_url)
		red.delete(uid+"-access_token")
		red.delete(uid+"-refresh_token")
		self.clear_cookie("user")
		self.set_header('Content-Type', 'text/plain')
		self.write("Logged Out, Goodbye")
		self.finish()

class AudioHandler(BaseHandler):
	@tornado.web.authenticated
	@tornado.web.asynchronous
	def post(self):
		uid = tornado.escape.xhtml_escape(self.current_user)
		token = gettoken(uid)
		if (token == False):
			self.set_status(403)
		else:
			rxfile = self.request.files['data'][0]['body']
			tf = tempfile.NamedTemporaryFile(suffix=".wav")
			tf.write(rxfile)
			_input = AudioSegment.from_wav(tf.name)
			tf.close()
			tf = tempfile.NamedTemporaryFile(suffix=".wav")
			output = _input.set_channels(1).set_frame_rate(16000)
			f = output.export(tf.name, format="wav")
			url = 'https://access-alexa-na.amazon.com/v1/avs/speechrecognizer/recognize'
			headers = {'Authorization' : 'Bearer %s' % token}
			d = {
		    	"messageHeader": {
		        	"deviceContext": [
		            	{
		                	"name": "playbackState",
		                	"namespace": "AudioPlayer",
		                	"payload": {
		                    	"streamId": "",
		         			   	"offsetInMilliseconds": "0",
		                    	"playerActivity": "IDLE"
		                	}
		            	}
		        	]
		    	},
		    	"messageBody": {
		        	"profile": "alexa-close-talk",
		        	"locale": "en-us",
		        	"format": "audio/L16; rate=16000; channels=1"
		    	}
			}
			files = [
				('file', ('request', json.dumps(d), 'application/json; charset=UTF-8')),
				('file', ('audio', tf, 'audio/L16; rate=16000; channels=1'))
			]
			r = requests.post(url, headers=headers, files=files)
			tf.close()

			content_type = None

			try:
				content_type = r.headers['Content-Type']
			except Exception as e:
				content_type = r.headers['content-type']

			boundary = None

			for v in r.headers['content-type'].split(";"):
				if re.match(r'.*boundary.*', v):
					boundary =  v.split("=")[1]
			data = r.content.split(boundary)
			for d in data:
				if (len(d) >= 1024):
			 	   audio = d.split('\r\n\r\n')[1].rstrip('--')
			self.set_header('Content-Type', 'audio/mpeg')
			self.write(audio)
		self.finish()

class TextHandler(BaseHandler):
	@tornado.web.authenticated
	@tornado.web.asynchronous
	def get(self):
		text = self.get_argument("text", None, True)

		uid = tornado.escape.xhtml_escape(self.current_user)
		token = gettoken(uid)
		if (token == False):
			self.set_status(403)
		else:
			# rxfile = self.request.files['data'][0]['body']
			# tf = tempfile.NamedTemporaryFile(suffix=".wav")
			# tf.write(rxfile)
			# _input = AudioSegment.from_wav(tf.name)
			# tf.close()
			# tf = tempfile.NamedTemporaryFile(suffix=".wav")
			# output = _input.set_channels(1).set_frame_rate(16000)
			# f = output.export(tf.name, format="wav")

			random_str = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(20))
			file_name = "/tmp/{}.wav".format(random_str)
			cmd = 'espeak "{}" --stdout > {}'.format(text, file_name)

			print text
			print file_name
			print cmd

			os.system(cmd)

			_input = AudioSegment.from_wav(file_name)
                        tf = tempfile.NamedTemporaryFile(suffix=".wav")
                        output = _input.set_channels(1).set_frame_rate(16000)
                        f = output.export(tf.name, format="wav")

			#tf = open(file_name)

			url = 'https://access-alexa-na.amazon.com/v1/avs/speechrecognizer/recognize'
			headers = {'Authorization' : 'Bearer %s' % token}
			d = {
		    	"messageHeader": {
		        	"deviceContext": [
		            	{
		                	"name": "playbackState",
		                	"namespace": "AudioPlayer",
		                	"payload": {
		                    	"streamId": "",
		         			   	"offsetInMilliseconds": "0",
		                    	"playerActivity": "IDLE"
		                	}
		            	}
		        	]
		    	},
		    	"messageBody": {
		        	"profile": "alexa-close-talk",
		        	"locale": "en-us",
		        	"format": "audio/L16; rate=16000; channels=1"
		    	}
			}
			files = [
				('file', ('request', json.dumps(d), 'application/json; charset=UTF-8')),
				('file', ('audio', tf, 'audio/L16; rate=16000; channels=1'))
			]
			r = requests.post(url, headers=headers, files=files)
			tf.close()
			print r
			for v in r.headers['content-type'].split(";"):
				if re.match('.*boundary.*', v):
					boundary =  v.split("=")[1]
			data = r.content.split(boundary)
			for d in data:
				if (len(d) >= 1024):
			 	   audio = d.split('\r\n\r\n')[1].rstrip('--')
			self.set_header('Content-Type', 'audio/mpeg')
			self.write(audio)
		self.finish()

def main():
	settings = {
	    "cookie_secret": "parisPOLANDbroadFENCEcornWOULD",
	    "login_url": "/static/welcome.html",
	}
	static_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static')
	application = tornado.web.Application([(r"/", MainHandler),
											(r"/start", StartAuthHandler),
											(r"/code", CodeAuthHandler),
											(r"/logout", LogoutHandler),
											(r"/audio", AudioHandler),
											(r"/trigger", TriggerHandler),
											(r"/questions", QuestionHandler),
											(r"/text", TextHandler),
											(r'/(favicon.ico)', tornado.web.StaticFileHandler,{'path': static_path}),
											(r'/static/(.*)', tornado.web.StaticFileHandler, {'path': static_path}),
											], **settings)
	http_server = tornado.httpserver.HTTPServer(application)
	port = int(os.environ.get("PORT", 8080))
	http_server.listen(port)
	tornado.ioloop.IOLoop.instance().start()

if __name__ == "__main__":
	main()

<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

def n_queens():
    """N Queens.

    On an N by N board. Get the number of possible arrangements of the board
    where N queens can be placed on the board without threatening each other,
    i.e. no two queens share the same row, column, or diagonal.
    """
    pass

def main():
    pass

if __name__ == '__main__':
    main()
<commit_msg>Complete alg_n_queens.py<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

def _is_valid(board):
    """Check current queen position is valid."""
    current_row, current_col = len(board) - 1, board[-1]

    # Check any queens can attack the current queen.
    for row, col in enumerate(board[:-1]):
        diff = abs(current_col - col)
        if diff == 0 or diff == current_row - row:
            return False
    return True

def n_queens(n, board=[]):
    """The number of N Queens.

    On an N by N board, get the number of possible arrangements of the board
    where N queens can be placed on the board without attacking each other,
    i.e. no two queens share the same row, column, or diagonal.
    """
    # board is an 1-d array to store the column ids of queens.
    if n == len(board):
        return 1

    count = 0

    for col in range(n):
        # Append current queen's column id.
        board.append(col)

        if _is_valid(board):
            # If current queen's position is valid, add numbers.
            count += n_queens(n, board)

        # Backtrack by poping out current queens.
        board.pop()

    return count

def main():
    # Should be 1, 1, 0, 0, 2, 10, 4, 40, 92, 352.
    for i in range(10):
        print(n_queens(i))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before><commit_msg>Small changes<commit_after><|endoftext|>


<commit_before><commit_msg>computing the ratio for each images and report<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import re
from django.db import transaction
from django.db.models.fields import FieldDoesNotExist
from django.core.exceptions import ValidationError
from django.utils.encoding import force_unicode
from data_importer.core.descriptor import ReadDescriptor
from data_importer.core.exceptions import StopImporter
from data_importer.core.base import objclass2dict
from data_importer.core.base import DATA_IMPORTER_EXCEL_DECODER
from data_importer.core.base import DATA_IMPORTER_DECODER

class BaseImporter(object):
    """
    Base Importer method to create simples importes CSV files.

    set_reader: can be override to create new importers files
    """
    def __new__(cls, **kargs):
        """
        Provide custom methods in subclass Meta
        """
        if hasattr(cls, "Meta"):
            cls.Meta = objclass2dict(cls.Meta)
        return super(BaseImporter, cls).__new__(cls)

    def __init__(self, source=None):
        self._fields = []
        self._error = []
        self._cleaned_data = ()
        self._reader = None
        self._excluded = False
        self._readed = False

        self.start_fields()
        if source:
            self.source = source
            self.set_reader()

    class Meta:
        """
        Importer configurations
        """

    @staticmethod
    def to_unicode(bytestr):
        """
        Receive string bytestr and try to return a utf-8 string.
        """
        if not isinstance(bytestr, str) and not isinstance(bytestr, unicode):
            return bytestr

        try:
            decoded = bytestr.decode(DATA_IMPORTER_EXCEL_DECODER)  # default by excel csv
        except UnicodeEncodeError:
            decoded = force_unicode(bytestr, DATA_IMPORTER_DECODER)

        return decoded

    @property
    def source(self):
        """
        Return source opened
        """
        return self._source

    @source.setter
    def source(self, source):
        """
        Open source to reader
        """
        if isinstance(source, file):
            self._source = source
        elif isinstance(source, str) and os.path.exists(source) and source.endswith('csv'):
            self._source = open(source, 'rb')
        elif isinstance(source, list):
            self._source = source
        elif hasattr(source, 'file_upload'):  # for FileHistory instances
            self._source = source.file_upload
        elif hasattr(source, 'file'):
            self._source = open(source.file.name, 'rb')
        else:
            self._source = source
            # raise ValueError('Invalid Source')

    @property
    def meta(self):
        """
        Is same to use .Meta
        """
        if hasattr(self, 'Meta'):
            return self.Meta

    def start_fields(self):
        """
        Initial function to find fields or headers values
        This values will be used to process clean and save method
        If this method not have fields and have Meta.model this method
        will use model fields to populate content without id
        """
        if self.Meta.model and not hasattr(self, 'fields'):
            all_models_fields = [i.name for i in self.Meta.model._meta.fields if i.name != 'id']
            self.fields = all_models_fields

        self.exclude_fields()

        if self.Meta.descriptor:
            self.load_descriptor()

    def exclude_fields(self):
        """
        Exclude fields from Meta.exclude
        """
        if self.Meta.exclude and not self._excluded:
            self._excluded = True
            for exclude in self.Meta.exclude:
                if exclude in self.fields:
                    self.fields.remove(exclude)

    def load_descriptor(self):
        """
        Set fields from descriptor file
        """
        descriptor = ReadDescriptor(self.Meta.descriptor, self.Meta.descriptor_model)
        self.fields = descriptor.get_fields()
        self.exclude_fields()

    @property
    def errors(self):
        """
        Show errors catch by clean methods
        """
        return self._error

    def is_valid(self):
        """
        Clear content and return False if have errors
        """
        if not self.cleaned_data:
            self.cleaned_data
        return not any(self._error)

    def set_reader(self):
        """
        Method responsable to convert file content into a list with same values that
        have fields

            fields: ['myfield1', 'myfield2']

            response: [['value_myfield1', 'value_myfield2'],
                        ['value2_myfield1', 'value2_myfield2']]
        """
        raise NotImplementedError('No reader implemented')

    def clean_field(self, field_name, value):
        """
        User default django field validators to clean content
        and run custom validates
        """
        if self.Meta.model:
            # default django validate field
            try:
                field = self.Meta.model._meta.get_field(field_name)
                field.clean(value, field)
            except FieldDoesNotExist:
                pass  # do nothing if not find this field in model
            except ValidationError as msg:
                default_msg = msg.messages[0].replace('This field', '')
                new_msg = u'Field (%s) %s' % (field.name, default_msg)
                raise ValidationError(new_msg)

        clean_function = getattr(self, 'clean_%s' % field_name, False)

        if clean_function:
            return clean_function(value)
        return value

    def process_row(self, row, values):
        """
        Read clean functions from importer and return tupla with row number, field and value
        """
        values_encoded = [self.to_unicode(i) for i in values]
        try:
            values = dict(zip(self.fields, values_encoded))
        except TypeError:
            raise TypeError('Invalid Line: %s' % row)

        has_error = False

        if self.Meta.ignore_empty_lines:
            # ignore empty lines
            if not any(values.values()):
                return None

        for k, v in values.items():
            if self.Meta.raise_errors:
                values[k] = self.clean_field(k, v)
            else:
                try:
                    values[k] = self.clean_field(k, v)
                except StopImporter, e:
                    raise StopImporter(self.get_error_message(e, row))
                except Exception, e:
                    self._error.append(self.get_error_message(e, row))
                    has_error = True

        if has_error:
            return None

        # validate full row data
        try:
            values = self.clean_row(values)
        except Exception, e:
            self._error.append(self.get_error_message(e, row))
            return None

        return (row, values)

    def get_error_message(self, error, row=None, error_type=None):
        messages = error

        if not error_type:
            error_type = u"%s" % type(error).__name__

        if hasattr(error, 'message'):
            if error.message:
                messages = u'%s' % error.message
        elif hasattr(error, 'messages'):
            if error.messages:
                messages = u','.join(error.messages)

        messages = re.sub('\'', '', messages)

        if row:
            return row, error_type, messages
        else:
            return error_type, messages

    @property
    def cleaned_data(self):
        """
        Return tupla with data cleaned
        """
        if self._readed:
            return self._cleaned_data

        self._readed = True

        try:
            self.pre_clean()
        except Exception, e:
            self._error.append(self.get_error_message(e, error_type='__pre_clean__'))

        try:
            self.clean()
        except Exception, e:
            self._error.append(self.get_error_message(e, error_type='__clean_all__'))

        # create clean content
        for data in self._read_file():
            if data:
                self._cleaned_data += (data, )

        try:
            self.post_clean()
        except Exception, e:
            self._error.append(self.get_error_message(e, error_type='__post_clean__'))

        return self._cleaned_data

    def pre_clean(self):
        """
        Executed before all clean methods
        Important: pre_clean dont have cleaned_data content
        """

    def post_clean(self):
        """
        Excuted after all clean method
        """

    def clean(self):
        """
        Custom clean method
        """

    def clean_row(self, row_values):
        """
        Custom clean method for full row data
        """
        return row_values

    def pre_commit(self):
        """
        Executed before commit multiple register
        """

    def post_save_all_lines(self):
        """
        End exection
        """

    def _read_file(self):
        """
        Create cleaned_data content
        """

        if hasattr(self._reader, 'read'):
            reader = self._reader.read()
        else:
            reader = self._reader

        for row, values in enumerate(reader, 1):
            if self.Meta.ignore_first_line:
                row -= 1
            if self.Meta.starting_row and row < self.Meta.starting_row:
                pass
            elif row < 1:
                pass
            else:
                yield self.process_row(row, values)

    def save(self, instance=None):
        """
        Save all contents
        DONT override this method
        """
        if not instance:
            instance = self.Meta.model

        if not instance:
            raise AttributeError('Invalid instance model')

        if self.Meta.transaction:
            with transaction.atomic():
                for row, data in self.cleaned_data:
                    record = instance(**data)
                    record.save()

                try:
                    self.pre_commit()
                except Exception, e:
                    self._error.append(self.get_error_message(e, error_type='__pre_commit__'))
                    transaction.rollback()

                try:
                    transaction.commit()
                except Exception, e:
                    self._error.append(self.get_error_message(e, error_type='__trasaction__'))
                    transaction.rollback()

        else:
            for row, data in self.cleaned_data:
                record = instance(**data)
                record.save(force_update=False)

        self.post_save_all_lines()

        return True
<commit_msg>Update error message generator<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import re
from django.db import transaction
from django.db.models.fields import FieldDoesNotExist
from django.core.exceptions import ValidationError
from django.utils.encoding import force_unicode
from data_importer.core.descriptor import ReadDescriptor
from data_importer.core.exceptions import StopImporter
from data_importer.core.base import objclass2dict
from data_importer.core.base import DATA_IMPORTER_EXCEL_DECODER
from data_importer.core.base import DATA_IMPORTER_DECODER

class BaseImporter(object):
    """
    Base Importer method to create simples importes CSV files.

    set_reader: can be override to create new importers files
    """
    def __new__(cls, **kargs):
        """
        Provide custom methods in subclass Meta
        """
        if hasattr(cls, "Meta"):
            cls.Meta = objclass2dict(cls.Meta)
        return super(BaseImporter, cls).__new__(cls)

    def __init__(self, source=None):
        self._fields = []
        self._error = []
        self._cleaned_data = ()
        self._reader = None
        self._excluded = False
        self._readed = False

        self.start_fields()
        if source:
            self.source = source
            self.set_reader()

    class Meta:
        """
        Importer configurations
        """

    @staticmethod
    def to_unicode(bytestr):
        """
        Receive string bytestr and try to return a utf-8 string.
        """
        if not isinstance(bytestr, str) and not isinstance(bytestr, unicode):
            return bytestr

        try:
            decoded = bytestr.decode(DATA_IMPORTER_EXCEL_DECODER)  # default by excel csv
        except UnicodeEncodeError:
            decoded = force_unicode(bytestr, DATA_IMPORTER_DECODER)

        return decoded

    @property
    def source(self):
        """
        Return source opened
        """
        return self._source

    @source.setter
    def source(self, source):
        """
        Open source to reader
        """
        if isinstance(source, file):
            self._source = source
        elif isinstance(source, str) and os.path.exists(source) and source.endswith('csv'):
            self._source = open(source, 'rb')
        elif isinstance(source, list):
            self._source = source
        elif hasattr(source, 'file_upload'):  # for FileHistory instances
            self._source = source.file_upload
        elif hasattr(source, 'file'):
            self._source = open(source.file.name, 'rb')
        else:
            self._source = source
            # raise ValueError('Invalid Source')

    @property
    def meta(self):
        """
        Is same to use .Meta
        """
        if hasattr(self, 'Meta'):
            return self.Meta

    def start_fields(self):
        """
        Initial function to find fields or headers values
        This values will be used to process clean and save method
        If this method not have fields and have Meta.model this method
        will use model fields to populate content without id
        """
        if self.Meta.model and not hasattr(self, 'fields'):
            all_models_fields = [i.name for i in self.Meta.model._meta.fields if i.name != 'id']
            self.fields = all_models_fields

        self.exclude_fields()

        if self.Meta.descriptor:
            self.load_descriptor()

    def exclude_fields(self):
        """
        Exclude fields from Meta.exclude
        """
        if self.Meta.exclude and not self._excluded:
            self._excluded = True
            for exclude in self.Meta.exclude:
                if exclude in self.fields:
                    self.fields.remove(exclude)

    def load_descriptor(self):
        """
        Set fields from descriptor file
        """
        descriptor = ReadDescriptor(self.Meta.descriptor, self.Meta.descriptor_model)
        self.fields = descriptor.get_fields()
        self.exclude_fields()

    @property
    def errors(self):
        """
        Show errors catch by clean methods
        """
        return self._error

    def is_valid(self):
        """
        Clear content and return False if have errors
        """
        if not self.cleaned_data:
            self.cleaned_data
        return not any(self._error)

    def set_reader(self):
        """
        Method responsable to convert file content into a list with same values that
        have fields

            fields: ['myfield1', 'myfield2']

            response: [['value_myfield1', 'value_myfield2'],
                        ['value2_myfield1', 'value2_myfield2']]
        """
        raise NotImplementedError('No reader implemented')

    def clean_field(self, field_name, value):
        """
        User default django field validators to clean content
        and run custom validates
        """
        if self.Meta.model:
            # default django validate field
            try:
                field = self.Meta.model._meta.get_field(field_name)
                field.clean(value, field)
            except FieldDoesNotExist:
                pass  # do nothing if not find this field in model
            except ValidationError as msg:
                default_msg = msg.messages[0].replace('This field', '')
                new_msg = u'Field (%s) %s' % (field.name, default_msg)
                raise ValidationError(new_msg)

        clean_function = getattr(self, 'clean_%s' % field_name, False)

        if clean_function:
            return clean_function(value)
        return value

    def process_row(self, row, values):
        """
        Read clean functions from importer and return tupla with row number, field and value
        """
        values_encoded = [self.to_unicode(i) for i in values]
        try:
            values = dict(zip(self.fields, values_encoded))
        except TypeError:
            raise TypeError('Invalid Line: %s' % row)

        has_error = False

        if self.Meta.ignore_empty_lines:
            # ignore empty lines
            if not any(values.values()):
                return None

        for k, v in values.items():
            if self.Meta.raise_errors:
                values[k] = self.clean_field(k, v)
            else:
                try:
                    values[k] = self.clean_field(k, v)
                except StopImporter, e:
                    raise StopImporter(self.get_error_message(e, row))
                except Exception, e:
                    self._error.append(self.get_error_message(e, row))
                    has_error = True

        if has_error:
            return None

        # validate full row data
        try:
            values = self.clean_row(values)
        except Exception, e:
            self._error.append(self.get_error_message(e, row))
            return None

        return (row, values)

    def get_error_message(self, error, row=None, error_type=None):
        messages = error

        if not error_type:
            error_type = u"%s" % type(error).__name__

        if hasattr(error, 'message'):
            if error.message:
                messages = u'%s' % error.message
        elif hasattr(error, 'messages'):
            if error.messages:
                messages = u','.join(error.messages)

        messages = re.sub('\'', '', messages)
        error_type = re.sub('\'', '', error_type)

        if row:
            return row, error_type, messages
        else:
            return error_type, messages

    @property
    def cleaned_data(self):
        """
        Return tupla with data cleaned
        """
        if self._readed:
            return self._cleaned_data

        self._readed = True

        try:
            self.pre_clean()
        except Exception, e:
            self._error.append(self.get_error_message(e, error_type='__pre_clean__'))

        try:
            self.clean()
        except Exception, e:
            self._error.append(self.get_error_message(e, error_type='__clean_all__'))

        # create clean content
        for data in self._read_file():
            if data:
                self._cleaned_data += (data, )

        try:
            self.post_clean()
        except Exception, e:
            self._error.append(self.get_error_message(e, error_type='__post_clean__'))

        return self._cleaned_data

    def pre_clean(self):
        """
        Executed before all clean methods
        Important: pre_clean dont have cleaned_data content
        """

    def post_clean(self):
        """
        Excuted after all clean method
        """

    def clean(self):
        """
        Custom clean method
        """

    def clean_row(self, row_values):
        """
        Custom clean method for full row data
        """
        return row_values

    def pre_commit(self):
        """
        Executed before commit multiple register
        """

    def post_save_all_lines(self):
        """
        End exection
        """

    def _read_file(self):
        """
        Create cleaned_data content
        """

        if hasattr(self._reader, 'read'):
            reader = self._reader.read()
        else:
            reader = self._reader

        for row, values in enumerate(reader, 1):
            if self.Meta.ignore_first_line:
                row -= 1
            if self.Meta.starting_row and row < self.Meta.starting_row:
                pass
            elif row < 1:
                pass
            else:
                yield self.process_row(row, values)

    def save(self, instance=None):
        """
        Save all contents
        DONT override this method
        """
        if not instance:
            instance = self.Meta.model

        if not instance:
            raise AttributeError('Invalid instance model')

        if self.Meta.transaction:
            with transaction.atomic():
                for row, data in self.cleaned_data:
                    record = instance(**data)
                    record.save()

                try:
                    self.pre_commit()
                except Exception, e:
                    self._error.append(self.get_error_message(e, error_type='__pre_commit__'))
                    transaction.rollback()

                try:
                    transaction.commit()
                except Exception, e:
                    self._error.append(self.get_error_message(e, error_type='__trasaction__'))
                    transaction.rollback()

        else:
            for row, data in self.cleaned_data:
                record = instance(**data)
                record.save(force_update=False)

        self.post_save_all_lines()

        return True
<|endoftext|>


<commit_before>import numpy as N
import copy

class DenseDesignMatrix(object):

    def __init__(self, X, y = None, view_converter = None, rng = None):
        self.X = X
        self.y = y
        self.view_converter = view_converter
        if rng is None:
            rng = N.random.RandomState([17,2,946])
        #
        self.default_rng = copy.copy(rng)
        self.rng = rng
    #

    def reset_RNG(self):
        if 'default_rng' not in dir(self):
            self.default_rng = N.random.RandomState([17,2,946])
        self.rng = copy.copy(self.default_rng)
    #

    def apply_preprocessor(self, preprocessor, can_fit = False):
        preprocessor.apply(self, can_fit)
    #

    def get_topological_view(self, mat = None):
        if self.view_converter is None:
            raise Exception("Tried to call get_topological_view on a dataset that has no view converter")
        #

        if mat is None:
            mat = self.X

        return self.view_converter.design_mat_to_topo_view(mat)
    #

    def get_weights_view(self, mat):
        if self.view_converter is None:
            raise Exception("Tried to call get_weights_view on a dataset that has no view converter")
        #

        return self.view_converter.design_mat_to_weights_view(mat)

    def set_topological_view(self, V):
        assert not N.any(N.isnan(V))
        self.view_converter = DefaultViewConverter(V.shape[1:])
        self.X = self.view_converter.topo_view_to_design_mat(V)
        assert not N.any(N.isnan(self.X))
    #

    def get_design_matrix(self):
        return self.X
    #

    def set_design_matrix(self, X):
        assert not N.any(N.isnan(X))
        self.X = X
    #

    def get_batch_design(self, batch_size):
        idx = self.rng.randint(self.X.shape[0] - batch_size + 1)
        return self.X[idx:idx+batch_size,:]
    #

    def get_batch_topo(self, batch_size):
        return self.view_converter.design_mat_to_topo_view(self.get_batch_design(batch_size))
    #

    def view_shape(self):
        return self.view_converter.view_shape()
#

class DefaultViewConverter:
    def __init__(self, shape):
        self.shape = shape
        self.pixels_per_channel = 1
        for dim in self.shape[:-1]:
            self.pixels_per_channel *= dim
        #
    #

    def view_shape(self):
        return self.shape

    def design_mat_to_topo_view(self, X):
        batch_size = X.shape[0]

        channel_shape = [ batch_size ]
        for dim in self.shape[:-1]:
            channel_shape.append(dim)
        channel_shape.append(1)

        channels = [
                    X[:,i*self.pixels_per_channel:(i+1)*self.pixels_per_channel].reshape(*channel_shape)
                    for i in xrange(self.shape[-1])
                    ]

        rval = N.concatenate(channels,axis=len(self.shape))

        assert len(rval.shape) == len(self.shape) + 1

        return rval
    #

    def design_mat_to_weights_view(self, X):
        return self.design_mat_to_topo_view(X)
    #

    def topo_view_to_design_mat(self, V):
        if N.any( N.asarray(self.shape) != N.asarray(V.shape[1:])):
            raise ValueError('View converter for views of shape batch size followed by '
                            +str(shape)+' given tensor of shape '+str(V.shape))
        #

        batch_size = V.shape[0]

        channels = [
                    V[:,:,:,i].reshape(batch_size,self.pixels_per_channel)
                    for i in xrange(3)
                    ]

        return N.concatenate(channels,axis=1)
    #
#
<commit_msg>added feature where datasets can compress themselves to uint8 for saving<commit_after>import numpy as N
import copy

class DenseDesignMatrix(object):

    def __init__(self, X, y = None, view_converter = None, rng = None):
        self.X = X
        self.y = y
        self.view_converter = view_converter
        if rng is None:
            rng = N.random.RandomState([17,2,946])
        #
        self.default_rng = copy.copy(rng)
        self.rng = rng
        self.compress = False
    #

    def enable_compression(self):
        self.compress = True
    #

    def __getstate__(self):
        rval = copy.copy(self.__dict__)

        if self.compress:
            rval['compress_min'] = rval['X'].min(axis=0)
            rval['X'] -= rval['compress_min']
            rval['compress_max'] = rval['X'].max(axis=0)
            rval['compress_max'][rval['compress_max']==0] = 1
            rval['X'] *= 255. / rval['compress_max']
            rval['X'] = N.cast['uint8'](rval['X'])
        return rval

    def __setstate__(self, d):

        if d['compress']:
            X = d['X']
            mx = d['compress_max']
            mn = d['compress_min']
            del d['compress_max']
            del d['compress_min']
            d['X'] = 0
            self.__dict__.update(d)
            self.X = N.cast['float32'](X)*mx+mn
        else:
            self.__dict__.update(d)

    def reset_RNG(self):
        if 'default_rng' not in dir(self):
            self.default_rng = N.random.RandomState([17,2,946])
        self.rng = copy.copy(self.default_rng)
    #

    def apply_preprocessor(self, preprocessor, can_fit = False):
        preprocessor.apply(self, can_fit)
    #

    def get_topological_view(self, mat = None):
        if self.view_converter is None:
            raise Exception("Tried to call get_topological_view on a dataset that has no view converter")
        #

        if mat is None:
            mat = self.X

        return self.view_converter.design_mat_to_topo_view(mat)
    #

    def get_weights_view(self, mat):
        if self.view_converter is None:
            raise Exception("Tried to call get_weights_view on a dataset that has no view converter")
        #

        return self.view_converter.design_mat_to_weights_view(mat)

    def set_topological_view(self, V):
        assert not N.any(N.isnan(V))
        self.view_converter = DefaultViewConverter(V.shape[1:])
        self.X = self.view_converter.topo_view_to_design_mat(V)
        assert not N.any(N.isnan(self.X))
    #

    def get_design_matrix(self):
        return self.X
    #

    def set_design_matrix(self, X):
        assert not N.any(N.isnan(X))
        self.X = X
    #

    def get_batch_design(self, batch_size):
        idx = self.rng.randint(self.X.shape[0] - batch_size + 1)
        return self.X[idx:idx+batch_size,:]
    #

    def get_batch_topo(self, batch_size):
        return self.view_converter.design_mat_to_topo_view(self.get_batch_design(batch_size))
    #

    def view_shape(self):
        return self.view_converter.view_shape()
#

class DefaultViewConverter:
    def __init__(self, shape):
        self.shape = shape
        self.pixels_per_channel = 1
        for dim in self.shape[:-1]:
            self.pixels_per_channel *= dim
        #
    #

    def view_shape(self):
        return self.shape

    def design_mat_to_topo_view(self, X):
        batch_size = X.shape[0]

        channel_shape = [ batch_size ]
        for dim in self.shape[:-1]:
            channel_shape.append(dim)
        channel_shape.append(1)

        channels = [
                    X[:,i*self.pixels_per_channel:(i+1)*self.pixels_per_channel].reshape(*channel_shape)
                    for i in xrange(self.shape[-1])
                    ]

        rval = N.concatenate(channels,axis=len(self.shape))

        assert len(rval.shape) == len(self.shape) + 1

        return rval
    #

    def design_mat_to_weights_view(self, X):
        return self.design_mat_to_topo_view(X)
    #

    def topo_view_to_design_mat(self, V):
        if N.any( N.asarray(self.shape) != N.asarray(V.shape[1:])):
            raise ValueError('View converter for views of shape batch size followed by '
                            +str(shape)+' given tensor of shape '+str(V.shape))
        #

        batch_size = V.shape[0]

        channels = [
                    V[:,:,:,i].reshape(batch_size,self.pixels_per_channel)
                    for i in xrange(3)
                    ]

        return N.concatenate(channels,axis=1)
    #
#
<|endoftext|>


<commit_before>import operator
import re
from collections import Counter, defaultdict
from datetime import datetime

from hamper.interfaces import ChatCommandPlugin, Command
from hamper.utils import ude, uen

from sqlalchemy import Column, DateTime, Integer, String
from sqlalchemy.ext.declarative import declarative_base

SQLAlchemyBase = declarative_base()

class Karma(ChatCommandPlugin):
    '''Give, take, and scoreboard Internet Points'''

    """
    Hamper will look for lines that end in ++ or -- and modify that user's
    karma value accordingly

    NOTE: The user is just a string, this really could be anything...like
    potatoes or the infamous cookie clicker....
    """

    name = 'karma'

    priority = -2

    short_desc = ("karma - Give positive or negative karma. Where you see"
                  " !karma, !score will work as well")
    long_desc = ("username++ - Give karma\n"
                 "username-- - Take karma\n"
                 "!karma --top - Show the top 5 karma earners\n"
                 "!karma --bottom - Show the bottom 5 karma earners\n"
                 "!karma --giver or --taker - Show who's given the most"
                 " positive or negative karma\n"
                 "!karma --when-positive or --when-negative "
                 " - Show when people are the most positive or negative\n"
                 "!karma username - Show the user's karma count\n")

    gotta_catch_em_all = r"""# 3 or statement
                             (

                             # Starting with a (, look for anything within
                             # parens that end with 2 or more + or -
                             (?=\()[^\)]+\)(\+\++|--+) |

                             # Looking from the start of the line until 2 or
                             # more - or + are found. No whitespace in this
                             # grouping
                             ^[^\s]+(\+\++|--+) |

                             # Finally group any non-whitespace groupings
                             # that end with 2 or more + or -
                             [^\s]+?(\+\++|--+)((?=\s)|(?=$))
                            )
                          """

    regstr = re.compile(gotta_catch_em_all, re.X)

    def setup(self, loader):
        super(Karma, self).setup(loader)
        self.db = loader.db
        SQLAlchemyBase.metadata.create_all(self.db.engine)

    def message(self, bot, comm):
        """
        Check for strings ending with 2 or more '-' or '+'
        """
        super(Karma, self).message(bot, comm)

        # No directed karma giving or taking
        if not comm['directed'] and not comm['pm']:
            msg = comm['message'].strip().lower()
            # use the magic above
            words = self.regstr.findall(msg)
            # Do things to people
            karmas = self.modify_karma(words)
            # Notify the users they can't modify their own karma
            if comm['user'] in karmas.keys():
                if karmas[comm['user']] <= 0:
                    bot.reply(comm, "Don't be so hard on yourself.")
                else:
                    bot.reply(comm, "Tisk, tisk, no up'ing your own karma.")
            # Commit karma changes to the db
            self.update_db(comm["user"], karmas)

    def modify_karma(self, words):
        """
        Given a regex object, look through the groups and modify karma
        as necessary
        """

        # 'user': karma
        k = defaultdict(int)

        if words:
            # For loop through all of the group members
            for word_tuple in words:
                word = word_tuple[0]
                ending = word[-1]
                # This will either end with a - or +, if it's a - subract 1
                # kara, if it ends with a +, add 1 karma
                change = -1 if ending == '-' else 1
                # Now strip the ++ or -- from the end
                if '-' in ending:
                    word = word.rstrip('-')
                elif '+' in ending:
                    word = word.rstrip('+')
                # Check if surrounded by parens, if so, remove them
                if word.startswith('(') and word.endswith(')'):
                    word = word[1:-1]
                # Finally strip whitespace
                word = word.strip()
                # Add the user to the dict
                if word:
                    k[word] += change
        return k

    def update_db(self, giver, receiverkarma):
        """
        Record a the giver of karma, the receiver of karma, and the karma
        amount. Typically the count will be 1, but it can be any positive or
        negative integer.
        """

        for receiver in receiverkarma:
            if receiver != giver:
                urow = KarmaTable(ude(giver), ude(receiver),
                                  receiverkarma[receiver])
                self.db.session.add(urow)
        self.db.session.commit()

    class KarmaList(Command):
        """
        Return the highest or lowest 5 receivers of karma
        """

        regex = r'^(?:score|karma) --(top|bottom)$'

        LIMIT = 5

        def command(self, bot, comm, groups):
            # We'll need all the rows
            kts = bot.factory.loader.db.session.query(KarmaTable).all()
            # From all the rows, tally the karma for each receiver
            receivers = defaultdict(int)
            for row in kts:
                receivers[row.receiver] += row.kcount
            rec_count = len(receivers.keys())
            rec_sorted = sorted(receivers.iteritems(),
                                key=operator.itemgetter(1))

            # We should limit the list of users to at most self.LIMIT
            limit = self.LIMIT if rec_count >= self.LIMIT else rec_count
            if limit:
                if groups[0] == 'top':
                    snippet = rec_sorted[-limit:]
                elif groups[0] == 'bottom':
                    snippet = rec_sorted[0:limit]
                else:
                    bot.reply(
                        comm, r'Something went wrong with karma\'s regex'
                    )
                    return

                for rec in snippet:
                    bot.reply(
                        comm, '%s\x0f: %d' % (uen(rec[0]), rec[1]),
                        encode=False
                    )
            else:
                bot.reply(comm, r'No one has any karma yet :-(')

    class UserKarma(Command):
        """
        Retrieve karma for a given user
        """

        # !karma <username>
        regex = r'^(?:score|karma)\s+([^-].*)$'

        def command(self, bot, comm, groups):
            # Play nice when the user isn't in the db
            kt = bot.factory.loader.db.session.query(KarmaTable)
            thing = ude(groups[0].strip().lower())
            rec_list = kt.filter(KarmaTable.receiver == thing).all()

            if rec_list:
                total = 0
                for r in rec_list:
                    total += r.kcount
                bot.reply(
                    comm, '%s has %d points' % (uen(thing), total),
                    encode=False
                )
            else:
                bot.reply(
                    comm, 'No karma for %s ' % uen(thing), encode=False
                )

    class KarmaGiver(Command):
        """
        Identifies the person who gives the most karma
        """

        regex = r'^(?:score|karma) --(giver|taker)$'

        def command(self, bot, comm, groups):
            kt = bot.factory.loader.db.session.query(KarmaTable)
            counter = Counter()

            if groups[0] == 'giver':
                positive_karma = kt.filter(KarmaTable.kcount > 0)
                for row in positive_karma:
                    counter[row.giver] += row.kcount

                m = counter.most_common(1)
                most = m[0] if m else None
                if most:
                    bot.reply(
                        comm,
                        '%s has given the most karma (%d)' %
                        (uen(most[0]), most[1])
                    )
                else:
                    bot.reply(
                        comm,
                        'No positive karma has been given yet :-('
                    )
            elif groups[0] == 'taker':
                negative_karma = kt.filter(KarmaTable.kcount < 0)
                for row in negative_karma:
                    counter[row.giver] += row.kcount

                m = counter.most_common()
                most = m[-1] if m else None
                if most:
                    bot.reply(
                        comm,
                        '%s has given the most negative karma (%d)' %
                        (uen(most[0]), most[1])
                    )
                else:
                    bot.reply(
                        comm,
                        'No negative karma has been given yet'
                    )

    class MostActive(Command):
        """
        Least/Most active hours of karma giving/taking
        """

        regex = r'^(?:score|karma)\s+--when-(positive|negative)'

        def command(self, bot, comm, groups):
            kt = bot.factory.loader.db.session.query(KarmaTable)
            counter = Counter()

            if groups[0] == "positive":
                karma = kt.filter(KarmaTable.kcount > 0)
            elif groups[0] == "negative":
                karma = kt.filter(KarmaTable.kcount < 0)

            for row in karma:
                hour = row.datetime.hour
                counter[hour] += row.kcount

            common_hour = (counter.most_common(1)[0][0]
                           if counter.most_common(1) else None)
            title_case = groups[0][0].upper() + groups[0][1:]

            if common_hour:
                bot.reply(
                    comm,
                    '%s karma is usually given during the %d:00 hour (UTC)' %
                    (title_case, common_hour)
                )
            else:
                bot.reply(
                    comm,
                    '%s karma has been given yet' % title_case
                )

class KarmaTable(SQLAlchemyBase):
    """
    Keep track of users karma in a persistant manner
    """

    __tablename__ = 'karma'

    # Calling the primary key user, though, really, this can be any string
    id = Column(Integer, primary_key=True)
    giver = Column(String)
    receiver = Column(String)
    kcount = Column(Integer)
    datetime = Column(DateTime, default=datetime.utcnow())

    def __init__(self, giver, receiver, kcount):
        self.giver = giver
        self.receiver = receiver
        self.kcount = kcount

karma = Karma()
<commit_msg>karma: use the database to organize more effeicent<commit_after>import operator
import re
from collections import Counter, defaultdict
from datetime import datetime

from hamper.interfaces import ChatCommandPlugin, Command
from hamper.utils import ude, uen

from sqlalchemy import Column, DateTime, Integer, String, func
from sqlalchemy.ext.declarative import declarative_base

SQLAlchemyBase = declarative_base()

class Karma(ChatCommandPlugin):
    '''Give, take, and scoreboard Internet Points'''

    """
    Hamper will look for lines that end in ++ or -- and modify that user's
    karma value accordingly

    NOTE: The user is just a string, this really could be anything...like
    potatoes or the infamous cookie clicker....
    """

    name = 'karma'

    priority = -2

    short_desc = ("karma - Give positive or negative karma. Where you see"
                  " !karma, !score will work as well")
    long_desc = ("username++ - Give karma\n"
                 "username-- - Take karma\n"
                 "!karma --top - Show the top 5 karma earners\n"
                 "!karma --bottom - Show the bottom 5 karma earners\n"
                 "!karma --giver or --taker - Show who's given the most"
                 " positive or negative karma\n"
                 "!karma --when-positive or --when-negative "
                 " - Show when people are the most positive or negative\n"
                 "!karma username - Show the user's karma count\n")

    gotta_catch_em_all = r"""# 3 or statement
                             (

                             # Starting with a (, look for anything within
                             # parens that end with 2 or more + or -
                             (?=\()[^\)]+\)(\+\++|--+) |

                             # Looking from the start of the line until 2 or
                             # more - or + are found. No whitespace in this
                             # grouping
                             ^[^\s]+(\+\++|--+) |

                             # Finally group any non-whitespace groupings
                             # that end with 2 or more + or -
                             [^\s]+?(\+\++|--+)((?=\s)|(?=$))
                            )
                          """

    regstr = re.compile(gotta_catch_em_all, re.X)

    def setup(self, loader):
        super(Karma, self).setup(loader)
        self.db = loader.db
        SQLAlchemyBase.metadata.create_all(self.db.engine)

    def message(self, bot, comm):
        """
        Check for strings ending with 2 or more '-' or '+'
        """
        super(Karma, self).message(bot, comm)

        # No directed karma giving or taking
        if not comm['directed'] and not comm['pm']:
            msg = comm['message'].strip().lower()
            # use the magic above
            words = self.regstr.findall(msg)
            # Do things to people
            karmas = self.modify_karma(words)
            # Notify the users they can't modify their own karma
            if comm['user'] in karmas.keys():
                if karmas[comm['user']] <= 0:
                    bot.reply(comm, "Don't be so hard on yourself.")
                else:
                    bot.reply(comm, "Tisk, tisk, no up'ing your own karma.")
            # Commit karma changes to the db
            self.update_db(comm["user"], karmas)

    def modify_karma(self, words):
        """
        Given a regex object, look through the groups and modify karma
        as necessary
        """

        # 'user': karma
        k = defaultdict(int)

        if words:
            # For loop through all of the group members
            for word_tuple in words:
                word = word_tuple[0]
                ending = word[-1]
                # This will either end with a - or +, if it's a - subract 1
                # kara, if it ends with a +, add 1 karma
                change = -1 if ending == '-' else 1
                # Now strip the ++ or -- from the end
                if '-' in ending:
                    word = word.rstrip('-')
                elif '+' in ending:
                    word = word.rstrip('+')
                # Check if surrounded by parens, if so, remove them
                if word.startswith('(') and word.endswith(')'):
                    word = word[1:-1]
                # Finally strip whitespace
                word = word.strip()
                # Add the user to the dict
                if word:
                    k[word] += change
        return k

    def update_db(self, giver, receiverkarma):
        """
        Record a the giver of karma, the receiver of karma, and the karma
        amount. Typically the count will be 1, but it can be any positive or
        negative integer.
        """

        for receiver in receiverkarma:
            if receiver != giver:
                urow = KarmaTable(ude(giver), ude(receiver),
                                  receiverkarma[receiver])
                self.db.session.add(urow)
        self.db.session.commit()

    class KarmaList(Command):
        """
        Return the highest or lowest 5 receivers of karma
        """

        regex = r'^(?:score|karma) --(top|bottom)$'

        LIST_MAX = 5

        def command(self, bot, comm, groups):
            # We'll need all the rows
            kts = bot.factory.loader.db.session\
                                    .query(KarmaTable.receiver,
                                           func.sum(KarmaTable.kcount))\
                                    .group_by(KarmaTable.receiver)
            if kts.count():
                # We should limit the list of users to at most self.LIST_MAX
                if groups[0] == 'top':
                    snippet = kts.order_by(KarmaTable.kcount.desc())\
                                 .limit(self.LIST_MAX).all()
                elif groups[0] == 'bottom':
                    snippet = kts.order_by(KarmaTable.kcount)\
                                 .limit(self.LIST_MAX).all()
                else:
                    bot.reply(
                        comm, r'Something went wrong with karma\'s regex'
                    )
                    return

                for rec in snippet:
                    bot.reply(
                        comm, '%s\x0f: %d' % (uen(rec[0]), rec[1]),
                        encode=False
                    )
            else:
                bot.reply(comm, r'No one has any karma yet :-(')

    class UserKarma(Command):
        """
        Retrieve karma for a given user
        """

        # !karma <username>
        regex = r'^(?:score|karma)\s+([^-].*)$'

        def command(self, bot, comm, groups):
            # Play nice when the user isn't in the db
            kt = bot.factory.loader.db.session.query(KarmaTable)
            thing = ude(groups[0].strip().lower())
            rec_list = kt.filter(KarmaTable.receiver == thing).all()

            if rec_list:
                total = 0
                for r in rec_list:
                    total += r.kcount
                bot.reply(
                    comm, '%s has %d points' % (uen(thing), total),
                    encode=False
                )
            else:
                bot.reply(
                    comm, 'No karma for %s ' % uen(thing), encode=False
                )

    class KarmaGiver(Command):
        """
        Identifies the person who gives the most karma
        """

        regex = r'^(?:score|karma) --(giver|taker)$'

        def command(self, bot, comm, groups):
            kt = bot.factory.loader.db.session.query(KarmaTable)
            counter = Counter()

            if groups[0] == 'giver':
                positive_karma = kt.filter(KarmaTable.kcount > 0)
                for row in positive_karma:
                    counter[row.giver] += row.kcount

                m = counter.most_common(1)
                most = m[0] if m else None
                if most:
                    bot.reply(
                        comm,
                        '%s has given the most karma (%d)' %
                        (uen(most[0]), most[1])
                    )
                else:
                    bot.reply(
                        comm,
                        'No positive karma has been given yet :-('
                    )
            elif groups[0] == 'taker':
                negative_karma = kt.filter(KarmaTable.kcount < 0)
                for row in negative_karma:
                    counter[row.giver] += row.kcount

                m = counter.most_common()
                most = m[-1] if m else None
                if most:
                    bot.reply(
                        comm,
                        '%s has given the most negative karma (%d)' %
                        (uen(most[0]), most[1])
                    )
                else:
                    bot.reply(
                        comm,
                        'No negative karma has been given yet'
                    )

    class MostActive(Command):
        """
        Least/Most active hours of karma giving/taking
        """

        regex = r'^(?:score|karma)\s+--when-(positive|negative)'

        def command(self, bot, comm, groups):
            kt = bot.factory.loader.db.session.query(KarmaTable)
            counter = Counter()

            if groups[0] == "positive":
                karma = kt.filter(KarmaTable.kcount > 0)
            elif groups[0] == "negative":
                karma = kt.filter(KarmaTable.kcount < 0)

            for row in karma:
                hour = row.datetime.hour
                counter[hour] += row.kcount

            common_hour = (counter.most_common(1)[0][0]
                           if counter.most_common(1) else None)
            title_case = groups[0][0].upper() + groups[0][1:]

            if common_hour:
                bot.reply(
                    comm,
                    '%s karma is usually given during the %d:00 hour (UTC)' %
                    (title_case, common_hour)
                )
            else:
                bot.reply(
                    comm,
                    '%s karma has been given yet' % title_case
                )

class KarmaTable(SQLAlchemyBase):
    """
    Keep track of users karma in a persistant manner
    """

    __tablename__ = 'karma'

    # Calling the primary key user, though, really, this can be any string
    id = Column(Integer, primary_key=True)
    giver = Column(String)
    receiver = Column(String)
    kcount = Column(Integer)
    datetime = Column(DateTime, default=datetime.utcnow())

    def __init__(self, giver, receiver, kcount):
        self.giver = giver
        self.receiver = receiver
        self.kcount = kcount

karma = Karma()
<|endoftext|>


<commit_before>from datetime import datetime
import random

from sqlalchemy import Integer, String, Date, Column
from sqlalchemy.ext.declarative import declarative_base

from hamper.interfaces import Command, ChatCommandPlugin

SQLAlchemyBase = declarative_base()

class Quotes(ChatCommandPlugin):
    '''Remember quotes, and recall on demand.'''

    name = 'quotes'
    priority = 0

    def setup(self, loader):
        super(Quotes, self).setup(loader)
        SQLAlchemyBase.metadata.create_all(loader.db.engine)

    class DeliverQuote(Command):
        """Deliver a quote."""
        regex = r'^quotes?$'

        name = 'quote'
        short_desc = 'quote - Show, add, or count quotes.'
        long_desc = ('quote - Show a quote\n'
                     'quote --add QUOTE - Add a quote\n'
                     'quote --count - Count quotes\n')

        def command(self, bot, comm, groups):
            index = random.randrange(0, bot.db.query(Quote).count() + 1)
            quote = bot.factory.loader.db.session.query(Quote)[index]
            # Lame twisted irc doesn't support unicode.
            bot.reply(comm, str(quote.text))
            return True

    class AddQuote(Command):
        """Add a quote."""
        regex = r'^quotes? --add (.*)$'

        def command(self, bot, comm, groups):
            text = groups[0]
            quote = Quote(text, comm['user'])
            bot.factory.loader.db.session.add(quote)
            bot.reply(comm, 'Succesfully added quote.')

    class CountQuotes(Command):
        """Count how many quotes the bot knows."""
        regex = r'^quotes? --count$'

        def command(self, bot, comm, groups):
            count = bot.factory.loader.db.session.query(Quote).count()
            bot.reply(comm, 'I know {0} quotes.'.format(count))

class Quote(SQLAlchemyBase):
    '''The object that will get persisted by the database.'''

    __tablename__ = 'quotes'

    id = Column(Integer, primary_key=True)
    text = Column(String)
    adder = Column(String)
    added = Column(Date)

    def __init__(self, text, adder, added=None):
        if not added:
            added = datetime.now()

        self.text = text
        self.adder = adder
        self.added = added

quotes = Quotes()
<commit_msg>Fix typo.<commit_after>from datetime import datetime
import random

from sqlalchemy import Integer, String, Date, Column
from sqlalchemy.ext.declarative import declarative_base

from hamper.interfaces import Command, ChatCommandPlugin

SQLAlchemyBase = declarative_base()

class Quotes(ChatCommandPlugin):
    '''Remember quotes, and recall on demand.'''

    name = 'quotes'
    priority = 0

    def setup(self, loader):
        super(Quotes, self).setup(loader)
        SQLAlchemyBase.metadata.create_all(loader.db.engine)

    class DeliverQuote(Command):
        """Deliver a quote."""
        regex = r'^quotes?$'

        name = 'quote'
        short_desc = 'quote - Show, add, or count quotes.'
        long_desc = ('quote - Show a quote\n'
                     'quote --add QUOTE - Add a quote\n'
                     'quote --count - Count quotes\n')

        def command(self, bot, comm, groups):
            index = random.randrange(0, bot.db.query(Quote).count() + 1)
            quote = bot.factory.loader.db.session.query(Quote)[index]
            # Lame twisted irc doesn't support unicode.
            bot.reply(comm, str(quote.text))
            return True

    class AddQuote(Command):
        """Add a quote."""
        regex = r'^quotes? --add (.*)$'

        def command(self, bot, comm, groups):
            text = groups[0]
            quote = Quote(text, comm['user'])
            bot.factory.loader.db.session.add(quote)
            bot.reply(comm, 'Successfully added quote.')

    class CountQuotes(Command):
        """Count how many quotes the bot knows."""
        regex = r'^quotes? --count$'

        def command(self, bot, comm, groups):
            count = bot.factory.loader.db.session.query(Quote).count()
            bot.reply(comm, 'I know {0} quotes.'.format(count))

class Quote(SQLAlchemyBase):
    '''The object that will get persisted by the database.'''

    __tablename__ = 'quotes'

    id = Column(Integer, primary_key=True)
    text = Column(String)
    adder = Column(String)
    added = Column(Date)

    def __init__(self, text, adder, added=None):
        if not added:
            added = datetime.now()

        self.text = text
        self.adder = adder
        self.added = added

quotes = Quotes()
<|endoftext|>


<commit_before>"""Kraken - objects.components.component_output_port module.

Classes:
ComponentOutputPort -- Component output port representation.

"""

from kraken.core.objects.scene_item import SceneItem

class ComponentOutputPort(SceneItem):
    """Component Output Object."""

    def __init__(self, name, parent, dataType):
        super(ComponentOutputPort, self).__init__(name, parent=parent)
        self._dataType = None
        self._connections = []
        self._target = None

        self.setDataType(dataType)

    # =================
    # DataType Methods
    # =================
    def setDataType(self, dataType):
        """Sets the data type for this input.

        Args:
            dataType (str): type of input source.

        Returns:
            bool: True if successful.

        """

        self._dataType = dataType

        return True

    def getDataType(self):
        """Returns the data type for this input.

        Returns:
            str: Data type of this input.

        """

        return self._dataType

    # ====================
    # Connections Methods
    # ====================
    def isConnected(self):
        """Checks if there is a connection.

        Returns:
            bool: Whether there are any output connections.

        """

        return len(self._connections) > 0

    def getConnection(self, index):
        """Gets the connection of this input.

        Args:
            index (int): Index of connection to return.

        Returns:
            object: Connection object or None if not set.

        """

        return self._connections[index]

    def _addConnection(self, connectionObj):
        """Adds a connection to the output.

        Args:
            connectionObj (object): Object to set as a connection.

        Returns:
            bool: True if successful.

        """

        if connectionObj.getDataType() != self.getDataType() and connectionObj.getDataType() != self.getDataType()[:-2]:
            raise Exception("Data Type mismatch! Cannot connect '" +
                connectionObj.getDataType() + "' to '" + self.getDataType())

        if connectionObj in self._connections:
            raise Exception("'connectionObj' is already in the connections.")

        self._connections.append(connectionObj)

        return True

    def _removeConnection(self, connectionObj):
        """Removes a connection.

        Args:
            connectionObj (object): Object to remove the connection for.

        Returns:
            bool: True if successful.

        """

        if connectionObj not in self._connections:
            raise Exception("'connectionObj' is not in the connections list.")

        self._connections.remove(connectionObj)

        return True

    # ===============
    # Target Methods
    # ===============
    def setTarget(self, target):
        """Sets the taret for this input.

        Args:
            target (object): Kraken object that is the target of this input.

        Returns:
            bool: True if successful.

        """

        self._target = target

    def getTarget(self):
        """Returns the target of the input.

        Returns:
            Object, the target of the input.

        """

        return self._target
<commit_msg>Added method to get number of output port connections.<commit_after>"""Kraken - objects.components.component_output_port module.

Classes:
ComponentOutputPort -- Component output port representation.

"""

from kraken.core.objects.scene_item import SceneItem

class ComponentOutputPort(SceneItem):
    """Component Output Object."""

    def __init__(self, name, parent, dataType):
        super(ComponentOutputPort, self).__init__(name, parent=parent)
        self._dataType = None
        self._connections = []
        self._target = None

        self.setDataType(dataType)

    # =================
    # DataType Methods
    # =================
    def setDataType(self, dataType):
        """Sets the data type for this input.

        Args:
            dataType (str): type of input source.

        Returns:
            bool: True if successful.

        """

        self._dataType = dataType

        return True

    def getDataType(self):
        """Returns the data type for this input.

        Returns:
            str: Data type of this input.

        """

        return self._dataType

    # ====================
    # Connections Methods
    # ====================
    def isConnected(self):
        """Checks if there is a connection.

        Returns:
            bool: Whether there are any output connections.

        """

        return len(self._connections) > 0

    def getConnection(self, index):
        """Gets the connection of this input.

        Args:
            index (int): Index of connection to return.

        Returns:
            object: Connection object or None if not set.

        """

        return self._connections[index]

    def getNumConnections(self):
        """Gets the number of connections for this output port.

        Returns:
            int: Number of connections.

        """

        return len(self._connections)

    def _addConnection(self, connectionObj):
        """Adds a connection to the output.

        Args:
            connectionObj (object): Object to set as a connection.

        Returns:
            bool: True if successful.

        """

        if connectionObj.getDataType() != self.getDataType() and connectionObj.getDataType() != self.getDataType()[:-2]:
            raise Exception("Data Type mismatch! Cannot connect '" +
                connectionObj.getDataType() + "' to '" + self.getDataType())

        if connectionObj in self._connections:
            raise Exception("'connectionObj' is already in the connections.")

        self._connections.append(connectionObj)

        return True

    def _removeConnection(self, connectionObj):
        """Removes a connection.

        Args:
            connectionObj (object): Object to remove the connection for.

        Returns:
            bool: True if successful.

        """

        if connectionObj not in self._connections:
            raise Exception("'connectionObj' is not in the connections list.")

        self._connections.remove(connectionObj)

        return True

    # ===============
    # Target Methods
    # ===============
    def setTarget(self, target):
        """Sets the taret for this input.

        Args:
            target (object): Kraken object that is the target of this input.

        Returns:
            bool: True if successful.

        """

        self._target = target

    def getTarget(self):
        """Returns the target of the input.

        Returns:
            Object, the target of the input.

        """

        return self._target
<|endoftext|>


<commit_before># Split a dataset based on an attribute and an attribute value
def test_split(index, value, dataset):
	left, right = list(), list()
	for row in dataset:
		if row[index] < value:
			left.append(row)
		else:
			right.append(row)
	return left, right

# Calculate the Gini index for a split dataset
def gini_index(groups, class_values):
	gini = 0.0
	for class_value in class_values:
		for group in groups:
			size = len(group)
			if size == 0:
				continue
			proportion = [row[-1] for row in group].count(class_value) / float(size)
			gini += (proportion * (1.0 - proportion))
	return gini

# Select the best split point for a dataset
def get_split(dataset):
	class_values = list(set(row[-1] for row in dataset))
	b_index, b_value, b_score, b_groups = 999, 999, 999, None
	for index in range(len(dataset[0])-1):
		for row in dataset:
			groups = test_split(index, row[index], dataset)
			gini = gini_index(groups, class_values)
			if gini < b_score:
				b_index, b_value, b_score, b_groups = index, row[index], gini, groups
	return {'index':b_index, 'value':b_value, 'groups':b_groups}

# Create a terminal node value
def to_terminal(group):
	outcomes = [row[-1] for row in group]
	return max(set(outcomes), key=outcomes.count)

# Create child splits for a node or make terminal
def split(node, max_depth, min_size, depth):
	left, right = node['groups']
	del(node['groups'])
	# check for a no split
	if not left or not right:
		node['left'] = node['right'] = to_terminal(left + right)
		return
	# check for max depth
	if depth >= max_depth:
		node['left'], node['right'] = to_terminal(left), to_terminal(right)
		return
	# process left child
	if len(left) <= min_size:
		node['left'] = to_terminal(left)
	else:
		node['left'] = get_split(left)
		split(node['left'], max_depth, min_size, depth+1)
	# process right child
	if len(right) <= min_size:
		node['right'] = to_terminal(right)
	else:
		node['right'] = get_split(right)
		split(node['right'], max_depth, min_size, depth+1)

# Build a decision tree
def build_tree(train, max_depth, min_size):
	root = get_split(dataset)
	split(root, max_depth, min_size, 1)
	return root

# Print a decision tree
def print_tree(node, depth=0):
	if isinstance(node, dict):
		print('%s[X%d < %.3f]' % ((depth*' ', (node['index']+1), node['value'])))
		print_tree(node['left'], depth+1)
		print_tree(node['right'], depth+1)
	else:
		print('%s[%s]' % ((depth*' ', node)))

def predict(node, row):
	if row[node['index']] < node['value']:
		if isinstance(node['left'], dict):
			return predict(node['left'], row)
		else:
			return node['left']
	else:
		if isinstance(node['right'], dict):
			return predict(node['right'], row)
		else:
			return node['right']

def test(testData, tree):
    right = 0
    for row in testData:
        answer = row[-1]
        test = predict(tree, row)
        if answer == test:
            right += 1
    accuracy = right/float(len(testData))
    return accuracy

def read_data():
    '''with open('parsed_admission.csv', 'rb') as f:
        reader = csv.reader(f)
        data = list(reader)
    data = data[1:]'''
    data = [[2.771244718,1.784783929,0],
	[1.728571309,1.169761413,0],
	[3.678319846,2.81281357,0],
	[3.961043357,2.61995032,0],
	[2.999208922,2.209014212,0],
	[7.497545867,3.162953546,1],
	[9.00220326,3.339047188,1],
	[7.444542326,0.476683375,1],
	[10.12493903,3.234550982,1],
	[6.642287351,3.319983761,1]]
    for i in range(len(data)):
        data[i] = data[i][1:]
        for j in range(len(data[i])):
            data[i][j] = float(data[i][j])

    test = data[0:200]
    train = data[200:500]
    return test, train

'''

'''

testdata, traindata = read_data()
tree = build_tree(traindata, 5, 1)
print(test(testdata, tree))
<commit_msg>source<commit_after>#http://machinelearningmastery.com/implement-decision-tree-algorithm-scratch-python/

# Split a dataset based on an attribute and an attribute value
def test_split(index, value, dataset):
	left, right = list(), list()
	for row in dataset:
		if row[index] < value:
			left.append(row)
		else:
			right.append(row)
	return left, right

# Calculate the Gini index for a split dataset
def gini_index(groups, class_values):
	gini = 0.0
	for class_value in class_values:
		for group in groups:
			size = len(group)
			if size == 0:
				continue
			proportion = [row[-1] for row in group].count(class_value) / float(size)
			gini += (proportion * (1.0 - proportion))
	return gini

# Select the best split point for a dataset
def get_split(dataset):
	class_values = list(set(row[-1] for row in dataset))
	b_index, b_value, b_score, b_groups = 999, 999, 999, None
	for index in range(len(dataset[0])-1):
		for row in dataset:
			groups = test_split(index, row[index], dataset)
			gini = gini_index(groups, class_values)
			if gini < b_score:
				b_index, b_value, b_score, b_groups = index, row[index], gini, groups
	return {'index':b_index, 'value':b_value, 'groups':b_groups}

# Create a terminal node value
def to_terminal(group):
	outcomes = [row[-1] for row in group]
	return max(set(outcomes), key=outcomes.count)

# Create child splits for a node or make terminal
def split(node, max_depth, min_size, depth):
	left, right = node['groups']
	del(node['groups'])
	# check for a no split
	if not left or not right:
		node['left'] = node['right'] = to_terminal(left + right)
		return
	# check for max depth
	if depth >= max_depth:
		node['left'], node['right'] = to_terminal(left), to_terminal(right)
		return
	# process left child
	if len(left) <= min_size:
		node['left'] = to_terminal(left)
	else:
		node['left'] = get_split(left)
		split(node['left'], max_depth, min_size, depth+1)
	# process right child
	if len(right) <= min_size:
		node['right'] = to_terminal(right)
	else:
		node['right'] = get_split(right)
		split(node['right'], max_depth, min_size, depth+1)

# Build a decision tree
def build_tree(train, max_depth, min_size):
	root = get_split(dataset)
	split(root, max_depth, min_size, 1)
	return root

# Print a decision tree
def print_tree(node, depth=0):
	if isinstance(node, dict):
		print('%s[X%d < %.3f]' % ((depth*' ', (node['index']+1), node['value'])))
		print_tree(node['left'], depth+1)
		print_tree(node['right'], depth+1)
	else:
		print('%s[%s]' % ((depth*' ', node)))

def predict(node, row):
	if row[node['index']] < node['value']:
		if isinstance(node['left'], dict):
			return predict(node['left'], row)
		else:
			return node['left']
	else:
		if isinstance(node['right'], dict):
			return predict(node['right'], row)
		else:
			return node['right']

def test(testData, tree):
    right = 0
    for row in testData:
        answer = row[-1]
        test = predict(tree, row)
        if answer == test:
            right += 1
    accuracy = right/float(len(testData))
    return accuracy

def read_data():
    '''with open('parsed_admission.csv', 'rb') as f:
        reader = csv.reader(f)
        data = list(reader)
    data = data[1:]'''
    data = [[2.771244718,1.784783929,0],
	[1.728571309,1.169761413,0],
	[3.678319846,2.81281357,0],
	[3.961043357,2.61995032,0],
	[2.999208922,2.209014212,0],
	[7.497545867,3.162953546,1],
	[9.00220326,3.339047188,1],
	[7.444542326,0.476683375,1],
	[10.12493903,3.234550982,1],
	[6.642287351,3.319983761,1]]
    for i in range(len(data)):
        data[i] = data[i][1:]
        for j in range(len(data[i])):
            data[i][j] = float(data[i][j])

    test = data[0:200]
    train = data[200:500]
    return test, train

'''

'''

testdata, traindata = read_data()
tree = build_tree(traindata, 5, 1)
print(test(testdata, tree))
<|endoftext|>