<commit_before>"""
This file was written by Jose Antonio Martin H. for the RL-Glue project.
you are allowed to use it (and see it) fully but subject to the next conditions

1. to not cause damage to any person
2. to not use it to earn money except when you give me the 50%
3. to use it to produce a state of the art RL agent, if not, think a lot and then come back to write a super agent.

This code is a 'parser' for the RL-Glue 3.0 TaskSpec.
It does not make any duplication of information, that is, what you get is always a view of the original string.
This is not the classic state-machine or automata approach to parsing languages so in particular you will se that
the parser is robust to a big set of taskpec string malformations still getting the right information.

Last modifed 22-1-2009 by Jose Antonio Martin H.
Added enforced parsing error catching.
"""

import sys
try:
    import psyco
    psyco.full()
except ImportError:
    pass

class TaskSpecParser:
    """
       RL-Glue TaskSpec Sparser V3
    """
    w = ["VERSION","PROBLEMTYPE","DISCOUNTFACTOR", "OBSERVATIONS","ACTIONS","REWARDS","EXTRA"]
    v = ["INTS","DOUBLES","CHARCOUNT"]
    expected_version = "RL-Glue-3.0"
    valid            = True
    last_error       = ""

    def __init__(self,ts):
        self.ts = ts
        if self.expected_version != self.getVersion():
            print "Warning: TaskSpec Version is not "+self.expected_version+" but "+self.getVersion()
            self.valid = False

    def getVersion(self):
        a = len(self.w[0])+1
        return self.ts[a:self.ts.find(" ",a)]

    def Validate(self):
        if not self.valid:
            print "Warning: TaskSpec String is invalid: "+self.last_error
            return False
        return True

    def getValue(self,i,ts,w):
        try:
            a = ts.index(w[i]) + len(w[i]) + 1
        except: #ValueError:
            #raise AttributeError("Malformed TaskSpec String: could not find the "+w[i]+" keyword")
            self.last_error = "could not find the "+w[i]+" keyword"
            print "Warning: Malformed TaskSpec String: " +self.last_error
            self.valid = False
            return ""
        b=None
        if (i+1)<len(w):
            try:
                b = ts.index(w[i+1])-1
            except: #ValueError:
                #raise AttributeError("Malformed TaskSpec String: could not find the "+w[i+1]+" keyword")
                self.last_error = "could not find the "+w[i+1]+" keyword"
                print "Warning: Malformed TaskSpec String: " +self.last_error
                self.valid = False
                return ""

        return ts[a:b]

    def getProblemType(self):
        if not self.Validate():
            return ""
        return self.getValue(1,self.ts,self.w)

    def getDiscountFactor(self):
        if not self.Validate():
            return ""
        return self.getValue(2,self.ts,self.w)

    def CompleteVars(self,str_in):
        if not self.Validate():
            return ""
        """ forces the vars to have ints doubles and charcount
        """
        if self.v[0] not in str_in:
            str_in = self.v[0]+" (0 0 0) " + str_in
        if self.v[2] not in str_in:
            str_in+=self.v[2]+" 0 "
        if self.v[1] not in str_in:
            i = str_in.find(self.v[2])
            str_in= str_in[0:i]+self.v[1]+" (0 0 0) "+str_in[i:]
        return str_in

    def getObservations(self):
        if not self.Validate():
            return ""
        str_o = self.getValue(3,self.ts,self.w)
        return self.CompleteVars(str_o)

    def getActions(self):
        if not self.Validate():
            return ""
        str_a = self.getValue(4,self.ts,self.w)
        return self.CompleteVars(str_a)

    def getReward(self):
        if not self.Validate():
            return ""
        return self.getValue(5,self.ts,self.w)

    def getExtra(self):
        if not self.Validate():
            return ""
        return self.getValue(6,self.ts,self.w)

    def getRange(self,str_input):
        if not self.Validate():
            return ""

        try:
            str_input = str_input.replace("UNSPEC","'UNSPEC'")
            str_input = str_input.replace("NEGINF","'NEGINF'")
            str_input = str_input.replace("POSINF","'POSINF'")
            str_input = str_input.replace(" ",",")
            r = eval(str_input)
            if len(r)==2:
                return [list(r)]

            out = r[0]*([[r[1],r[2]]])
            return out

        except:
            self.last_error = "error ocurred while parsing a Range in "+str_input
            print "Warning: Malformed TaskSpec String: " +self.last_error
            print sys.exc_info()
            self.valid = False
            return ""

    def getRewardRange(self):
        if not self.Validate():
            return ""
        str_reward = self.getReward()
        return self.getRange(str_reward)

    def getVarInfoRange(self,i,ts,w):
        self.Validate()
        a = ts.index(w[i])
        b = ts.index(w[i+1])+1
        return ts[a:b]

    def GetVarValue(self,i,str_o):
        if not self.Validate():
            return ""
        str_r = self.getValue(i,str_o,self.v)
        str_r = str_r.replace(") (",")#(")
        str_r = str_r.replace(")(",")#(")  # Ok I can parse it but this (there is no space between consecutiive ranges) should be checked since this means that the taskspec is malformed.
        parts = str_r.split("#")
        obs=[]
        for p in parts:
            obs.extend(self.getRange(p))
        return obs

    def getIntObservations(self):
        if not self.Validate():
            return ""
        return self.GetVarValue(0,self.getObservations())

    def getDoubleObservations(self):
        if not self.Validate():
            return ""
        return self.GetVarValue(1,self.getObservations())

    def getCharCountObservations(self):
        if not self.Validate():
            return ""
        str_o = self.getObservations()
        return int(self.getValue(2,str_o,self.v))

    def getIntActions(self):
        if not self.Validate():
            return ""
        return self.GetVarValue(0,self.getActions())

    def getDoubleActions(self):
        if not self.Validate():
            return ""
        return self.GetVarValue(1,self.getActions())

    def getCharCountActions(self):
        if not self.Validate():
            return ""
        str_a = self.getActions()
        return int(self.getValue(2,str_a,self.v))

def test():
    # you can cut the taskspec by the main words with new line
    ts= """VERSION RL-Glue-3.0 PROBLEMTYPE episodic DISCOUNTFACTOR 1 OBSERVATIONS INTS (3 0 1) DOUBLES (2 -1.2 0.5)(-.07 .07) CHARCOUNT 1024
         ACTIONS INTS (2 0 4) CHARCOUNT 1024 REWARDS (5.0 UNSPEC) EXTRA some other stuff goes here"""
    print ts
    print
    print

    TaskSpec = TaskSpecParser(ts)
    if TaskSpec.valid:
        print "======================================================================================================="
        print "Version: ["+TaskSpec.getVersion()+"]"
        print "ProblemType: ["+TaskSpec.getProblemType()+"]"
        print "DiscountFactor: ["+TaskSpec.getDiscountFactor()+"]"
        print "======================================================================================================="
        print "\t \t \t \t Observations"
        print "======================================================================================================="
        #print "Observations: ["+TaskSpec.getObservations()+"]"
        print "Integers:",TaskSpec.getIntObservations()
        print "Doubles: ",TaskSpec.getDoubleObservations()
        print "Chars:   ",TaskSpec.getCharCountObservations()
        print "======================================================================================================="
        print "\t \t \t \t Actions"
        print "======================================================================================================"
        #print "Actions: ["+TaskSpec.getActions()+"]"
        print "Integers:",TaskSpec.getIntActions()
        print "Doubles: ",TaskSpec.getDoubleActions()
        print "Chars:   ",TaskSpec.getCharCountActions()
        print "======================================================================================================="
        #print "Reward :["+TaskSpec.getReward()+"]")
        print "Reward Range:",TaskSpec.getRewardRange()
        print "Extra: ["+TaskSpec.getExtra()+"]"
        print "remeber that by using len() you get the cardinality of lists!"
        print "Thus:"
        print "len(",TaskSpec.getDoubleObservations(),") ==> ",len(TaskSpec.getDoubleObservations())," Double Observations"

if __name__=="__main__":
    test()

<commit_msg>More improvements in error checking, at least now it parses all the taskspecs in the test directory.<commit_after>"""
This file was written by Jose Antonio Martin H. for the RL-Glue project.
you are allowed to use it (and see it) fully but subject to the next conditions

1. to not cause damage to any person
2. to not use it to earn money except when you give me the 50%
3. to use it to produce a state of the art RL agent, if not, think a lot and then come back to write a super agent.

This code is a 'parser' for the RL-Glue 3.0 TaskSpec.
It does not make any duplication of information, that is, what you get is always a view of the original string.
This is not the classic state-machine or automata approach to parsing languages so in particular you will se that
the parser is robust to a big set of taskpec string malformations still getting the right information. blablabla

Last modifed 22-1-2009 by Jose Antonio Martin H.
Added enforced parsing error catching.
"""

import sys
try:
    import psyco
    psyco.full()
except ImportError:
    pass

class TaskSpecParser:
    """
       RL-Glue TaskSpec Sparser V3
    """
    w = ["VERSION","PROBLEMTYPE","DISCOUNTFACTOR", "OBSERVATIONS","ACTIONS","REWARDS","EXTRA"]
    v = ["INTS","DOUBLES","CHARCOUNT"]
    expected_version = "RL-Glue-3.0"
    valid            = True
    last_error       = ""

    def __init__(self,ts):
        self.ts = ts
        if self.expected_version != self.getVersion():
            print "Warning: TaskSpec Version is not "+self.expected_version+" but "+self.getVersion()
            self.valid = False

    def getVersion(self):
        a = len(self.w[0])+1
        return self.ts[a:self.ts.find(" ",a)]

    def Validate(self):
        if not self.valid:
            print "Warning: TaskSpec String is invalid: "+self.last_error
            return False
        return True

    def getValue(self,i,ts,w):
        try:
            a = ts.index(w[i]) + len(w[i]) + 1
        except: #ValueError:
            #raise AttributeError("Malformed TaskSpec String: could not find the "+w[i]+" keyword")
            self.last_error = "could not find the "+w[i]+" keyword"
            print "Warning: Malformed TaskSpec String: " +self.last_error
            self.valid = False
            return ""
        b=None
        if (i+1)<len(w):
            try:
                b = ts.index(w[i+1])-1
            except: #ValueError:
                #raise AttributeError("Malformed TaskSpec String: could not find the "+w[i+1]+" keyword")
                self.last_error = "could not find the "+w[i+1]+" keyword"
                print "Warning: Malformed TaskSpec String: " +self.last_error
                self.valid = False
                return ""

        return ts[a:b].strip()

    def getProblemType(self):
        if not self.Validate():
            return ""
        return self.getValue(1,self.ts,self.w)

    def getDiscountFactor(self):
        if not self.Validate():
            return ""
        return self.getValue(2,self.ts,self.w)

    def CompleteVars(self,str_in):
        if not self.Validate():
            return ""
        """ forces the vars to have ints doubles and charcount
        """
        if self.v[0] not in str_in:
            str_in = self.v[0]+" (0 0 0) " + str_in
        if self.v[2] not in str_in:
            str_in= str_in.rstrip()+" "+self.v[2]+" 0 "
        if self.v[1] not in str_in:
            i = str_in.find(self.v[2])
            str_in= str_in[0:i]+self.v[1]+" (0 0 0) "+str_in[i:]

        return str_in

    def getObservations(self):
        if not self.Validate():
            return ""
        str_o = self.getValue(3,self.ts,self.w)
        return self.CompleteVars(str_o)

    def getActions(self):
        if not self.Validate():
            return ""
        str_a = self.getValue(4,self.ts,self.w)
        return self.CompleteVars(str_a)

    def getReward(self):
        if not self.Validate():
            return ""
        return self.getValue(5,self.ts,self.w)

    def getExtra(self):
        if not self.Validate():
            return ""
        return self.getValue(6,self.ts,self.w)

    def getRange(self,str_input):
        if not self.Validate():
            return ""
        try:
            str_input = str_input.replace("UNSPEC","'UNSPEC'")
            str_input = str_input.replace("NEGINF","'NEGINF'")
            str_input = str_input.replace("POSINF","'POSINF'")
            str_input = str_input.replace(" ",",")
            r = eval(str_input)
            if len(r)==2:
                return [list(r)]

            out = r[0]*([[r[1],r[2]]])
            return out

        except:
            self.last_error = "error ocurred while parsing a Range in "+str_input
            print "Warning: Malformed TaskSpec String: " +self.last_error
            print sys.exc_info()
            self.valid = False
            return ""

    def getRewardRange(self):
        if not self.Validate():
            return ""
        str_reward = self.getReward()
        return self.getRange(str_reward)

    def getVarInfoRange(self,i,ts,w):
        self.Validate()
        a = ts.index(w[i])
        b = ts.index(w[i+1])+1
        return ts[a:b]

    def GetVarValue(self,i,str_o):
        if not self.Validate():
            return ""
        str_r = self.getValue(i,str_o,self.v)
        str_r = str_r.replace(") (",")#(")
        # Ok I can parse it but this (there is no space or there is an extra space in ranges)
        # should be checked since this means that the taskspec is malformed
        str_r = str_r.replace("( ","(")
        str_r = str_r.replace(" )",")")
        str_r = str_r.replace(")(",")#(")

        parts = str_r.split("#")
        obs=[]
        for p in parts:
            obs.extend(self.getRange(p))
        return obs

    def getIntObservations(self):
        if not self.Validate():
            return ""
        return self.GetVarValue(0,self.getObservations())

    def getDoubleObservations(self):
        if not self.Validate():
            return ""
        return self.GetVarValue(1,self.getObservations())

    def getCharCountObservations(self):
        if not self.Validate():
            return ""
        str_o = self.getObservations()
        return int(self.getValue(2,str_o,self.v))

    def getIntActions(self):
        if not self.Validate():
            return ""
        return self.GetVarValue(0,self.getActions())

    def getDoubleActions(self):
        if not self.Validate():
            return ""
        return self.GetVarValue(1,self.getActions())

    def getCharCountActions(self):
        if not self.Validate():
            return ""
        str_a = self.getActions()
        return int(self.getValue(2,str_a,self.v))

def test():
    # you can cut the taskspec by the main words with new line
    ts ="""VERSION RL-Glue-3.0 PROBLEMTYPE episodic DISCOUNTFACTOR .7 OBSERVATIONS INTS (NEGINF 1) ( 2 -5 POSINF ) DOUBLES (2 -1.2 0.5 )(-.07 .07) (UNSPEC 3.3) (0 100.5) CHARCOUNT 32
         ACTIONS INTS (5 0 4) DOUBLES (-.5 2) (2 7.8 9) (NEGINF UNSPEC) REWARDS (-5.0 5.0) EXTRA some other stuff goes here"""

    print ts
    print
    print

    TaskSpec = TaskSpecParser(ts)
    if TaskSpec.valid:
        print "======================================================================================================="
        print "Version: ["+TaskSpec.getVersion()+"]"
        print "ProblemType: ["+TaskSpec.getProblemType()+"]"
        print "DiscountFactor: ["+TaskSpec.getDiscountFactor()+"]"
        print "======================================================================================================="
        print "\t \t \t \t Observations"
        print "======================================================================================================="
        print "Observations: ["+TaskSpec.getObservations()+"]"
        print "Integers:",TaskSpec.getIntObservations()
        print "Doubles: ",TaskSpec.getDoubleObservations()
        print "Chars:   ",TaskSpec.getCharCountObservations()
        print "======================================================================================================="
        print "\t \t \t \t Actions"
        print "======================================================================================================"
        print "Actions: ["+TaskSpec.getActions()+"]"
        print "Integers:",TaskSpec.getIntActions()
        print "Doubles: ",TaskSpec.getDoubleActions()
        print "Chars:   ",TaskSpec.getCharCountActions()
        print "======================================================================================================="
        print "Reward :["+TaskSpec.getReward()+"]"
        print "Reward Range:",TaskSpec.getRewardRange()
        print "Extra: ["+TaskSpec.getExtra()+"]"
        print "remeber that by using len() you get the cardinality of lists!"
        print "Thus:"
        print "len(",TaskSpec.getDoubleObservations(),") ==> ",len(TaskSpec.getDoubleObservations())," Double Observations"

if __name__=="__main__":
    test()

<|endoftext|>


<commit_before>#!/usr/bin/python

""" Program to receive HTTP request from server with JSON body which will
contain the data for the song to be played/paused/stopped. The program will
use three threads.

Thread 1 - This thread will be used for playback.
Thread 2 - This thread will be used to keep the connection with the Server open
and when it timesout, the connection will be reopened and communicate with the playback thread.
Thread 3 - This thread will send Thread 1 commands for the song to be played/paused etc and also
communicate with the server.

The threads will use a queue which will be a basic FIFO implementation holding object types
which represent the request for the thread to execute. When the queue dequeues the thread will
execute that command and keep dequeueing and executing till the queue it empty in which case the
thread will wait on the queue.

@ Author - Anant Goel
@ date - April 2014
@ purpose - CS 252 Lab 6.

"""

#
# NOTES:
#
# Author: Jason P. Rahman
#
# Added code to handle:
#	The case when a song file is already present
#	Record the next song ID as a state machine
#	Record the currently playing song ID
#	Added a sanity check for the "Stopped" code
#		More places similar checks can be made
#	Added a few other notes
#
#	Next Steps:
#		Update stopped code to send Ready/Send song request/wait for previous song request
#		Continue adding more sanity checked
#		Continue tracking _current_song _next_song state variables
#

#Importing the thread function
from threading import Thread
import threading
import httplib
import json
import socket
import sys
import Queue
import pygame
import os
import fnmatch

import logging

logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s')

timeout = 50
_Rlock = threading.RLock()
flag_serv_func = 0
SONG_END = pygame.USEREVENT + 1

server_playback_queue = Queue.Queue(0)
playback_connection_queue = Queue.Queue(0)
connection_playback_queue = Queue.Queue(0)
server_communication_ID_queue = Queue.Queue(0)

# Define "Constants"
UNKNOWN = -1
PLAYING = 1
STOPPED = 2
DOWNLOADING = 3
READY = 4

client_id = UNKNOWN

# Initialize state to default values
current_song = UNKNOWN
current_song_state = UNKNOWN
next_song = UNKNOWN
next_song_state = UNKNOWN

server_url = "klamath.dnsdynamic.com"
server_port = "5050"

#
# query_vars is a dictionary of query variables
#
def create_url(request, query_vars = {}):
	global server_url

	url = server_url + ":" + server_port + "/speaker/" + request

	# Add query vars iff we have query vars to add
	if len(query_vars) > 0:
		url = url + "?"
		count = 0
		for key in query_vars:
			if count != 0:
				url = url + "&"
			url = url + str(key) + "=" + str(query_vars[key])

	return url

#
# Send a request over the given socket
#
def send_request(http_socket, request, query_vars = {}, body_params = {}, method = "GET"):
	headers = {"Content-Type": "application/json"}

	url = create_url(request, query_vars)

	logging.info("Sending request to: " + url)

	if body_params.keys() != 0:
		body = json.dumps(body_params, encoding = "ASCII")
	else:
		body = ""

	http_socket.request(method, url, body, headers)
	return http_socket.getresponse()

#
# Main function for the server thread
#
def serv_func():
	global client_id
	global server_url

	global server_url
	global server_port

	_serv_sock = httplib.HTTPConnection(server_url, server_port, timeout = timeout)
	socket.setdefaulttimeout(timeout)

	if client_id == UNKNOWN:

		logging.info("Authenticating")
		_serv_resp_json = send_request(_serv_sock, "authenticate", {}, {"pin":1234}, "POST")

		if _serv_resp_json.status != 200:
			sys.exit()

		logging.info("Authenticated response" + str(_serv_resp_json.status) + " " + _serv_resp_json.reason)

		_serv_resp_message = _serv_resp_json.read()
		_serv_response = json.loads(_serv_resp_message)

		client_id = _serv_response['id']
		logging.debug("Client ID is " + str(client_id))
		server_communication_ID_queue.put(client_id)

	try:
		while True:
			logging.info("Sending request_update")
			logging.info("Client ID in try is ="+str(client_id))

			_upcoming_song_resp = send_request(_serv_sock, "request_update", {"clientID": client_id})

			logging.info("Request Update Response")
			logging.info(str(_upcoming_song_resp.status) + " " + str(_upcoming_song_resp.reason) + " " + str(_upcoming_song_resp.getheaders()))
			#Push _upcoming_song_resp.read() the Queue it could also be the playcommand

			if _upcoming_song_resp.status == 200:

				rresp = _upcoming_song_resp.read()
				logging.debug("Message from server: "+str(rresp))
				server_playback_queue.put(rresp)
			else:
				logging.warning("Exiting")
				sys.exit()

	except socket.timeout:
		logging.debug("Timeout for connection")

		# NOTE: This will lead to stack overflow
		serv_func()

	_serv_sock.close()
	# DEAUTHENTICATE
	params1 = json.dumps({"id":str(client_id)},encoding = "ASCII")
	headers = {"Content-Type":"application/json"}
	_serv_sock.request("POST","klamath.dnsdynamic.com:5050/speaker/deauthenticate",params1,headers)

#
# Main function for playback control thread
#
def playback_func():
	global next_song
	global current_song
	global next_song_state
	global current_song_state

	pygame.mixer.init() #might have to make global if going to recursively call

	_message = {"id":"","status":"","position":""}

	while True:

		try:
			update_body = server_playback_queue.get(True, 0.1)

			_response = json.loads(update_body)
			server_playback_queue.task_done()

			logging.info("Response in playback_func is " + str(_response))

			update_type = _response['update_type']

			logging.debug("Update type is = " + update_type)

			if update_type == "playbackcommand":
				values = _response['values']
				song_id = values['id']
				command = values['command']

				if command == 'Play':

					logging.info("Play command recieved")

					if song_id == current_song:

						logging.error('Play command for current song, ignoring')

					elif current_song_state != PLAYING and song_id == next_song and next_song_state == READY:

						logging.info('Playback command received for next song')

						# Set next song to UNKNOWN since the next song
						# because the current song and we don't know
						# the new next song
						next_song = UNKNOWN
						next_song_state = UNKNOWN

						# Update current song
						current_song = next_song
						current_song_state = PLAYING

						# Start playback via PyGame
						pygame.mixer.music.set_endevent(SONG_END)
						#pygame.mixer.music.load(str(song_id))
						#pygame.mixer.music.play()

						#while pygame.mixer.music.get_busy():
						#	pygame.time.Clock().tick(10)

						logging.debug('Started playback through PyGame')
						logging.debug('Sending Playing status message to communcation thread')

						_message['id'] = str()
						_message['status'] = 'Playing'
						_message['position'] = str(0)

						logging.info("The message in Play is " + str(_message))

						playback_connection_queue.put(_message)

					elif song_id == next_song and next_song_state == UNKNOWN:

						logging.info('Play command for unavailable song, requesting from server')

						# We need to get the next song from the server
						_message['id'] = str(song_id)
						_message['status'] = 'need_song'
						_message['position'] = str(0);

						playback_connection_queue.put(_message)

				if command == 'Stop':

					# Sanity check for command
					if song_id == current_song:

						# Check the current state of the song
						if current_song_state == PLAYING:
							pygame.mixer.music.stop()

							_message['id'] = str(song_id)
							_message['status'] = 'Stopped'
							_message['position'] = str(pygame.mixer.music.get_pos())

							playback_connection_queue.put(_message)
							# TODO Look at _next_song
							# and send requestSong/ready or wait until
							# previous ready call succeeds
						else:
							pass # TODO decide how to handle this

					else:
						# TODO Error, how to handle this
						logging.error("Stop command from server for wrong song")

			if update_type == "upcoming_song":

				logging.info("Processing upcoming_song update")

				values = _response['values']
				song_id = values['id']

				logging.debug("Setting next_song to " + str(song_id))

				# Set next song
				next_song = song_id
				next_song_state = UNKNOWN

				found_song = 0

				for file in os.listdir('.'):
					if fnmatch.fnmatch(file,str(song_id)):
						found_song = 1
					else:
						pass

				if found_song == 0: # We need the file, so request first

					logging.debug('Could not find song, asking next')

					_message['id'] = str(song_id)
					_message['status'] = 'need_song'
					_message['position'] = str(0);

					playback_connection_queue.put(_message)
				else:
					logging.debug('Song ready for playback')
					next_song_state = READY

					if current_song_state != PLAYING:
						# We aren't playing anything right now
						# So send ready immediately since we have the file

						_message['id'] = str(song_id)
						_message['status'] = 'Ready'
						_message['position'] = '0'
						playback_connection_queue.put(_message)

		except Queue.Empty:

			# If the queue is empty, extract an event from PyGame
			if current_song_state == PLAYING:
				event = pygame.event.poll()
				if event.type == SONG_END:
					current_song_state = STOPPED

					_message['status'] = 'Stopped'
					_message['position'] = str(pygame.mixer.music.get_pos())
					playback_connection_queue.put(_message)

#
# Main function for server communication thread
#
def communicate_func():
	while True:
		logging.debug("Entered WHILE")
		global client_id

		global server_url
		global server_port

		global next_song
		global current_song
		global next_song_state
		global current_song_state

		# Pop the specific request from the Queue, depending on that do the following
		_request_set = playback_connection_queue.get()
		playback_connection_queue.task_done()

		logging.debug("communicate_func received request: \n"+str(_request_set))
		_comm_sock = httplib.HTTPConnection(server_url, server_port, timeout = timeout)

		if client_id == UNKNOWN:
			client_id = server_communication_ID_queue.get() # Getting the clientID from the queue
			server_communication_ID_queue.task_done()

		logging.info("client_id in communicate_func is " + str(client_id))

		if _request_set['status']=='need_song':

			logging.debug("IN NEED SONG with song_id = " + str(song_id))

			# Request song data from server
			song_id = _request_set['id']
			_song_data_resp = send_request(_comm_sock, "request_song", {"clientID": client_id, "songID": song_id})

			logging.debug("Song Data Response: " + str(_song_data_resp.status) + " " + str(_song_data_resp.reason))

			if _song_data_resp.status == 200:
				logging.debug("Song song data from server, saving to file")

				_song_data= _song_data_resp.read()

				output_file = open(str(song_id),'w')
				output_file.write(_song_data)
				output_file.close()

				if current_song_state != PLAYING:
					# Send ready message to the server

					_request_set['status'] = 'Ready'
					logging.debug("The Message in playing state is "+str(_request_set))
					send_request(_comm_sock, "status_update", {"clientID": client_id}, _request_set, "POST")

				else:

					# Update status in the background
					# When PyGame finishes playing, it will see the status
					next_song_state = READY

			else:

				logging.error("Failed to get song data from server")
				# TODO How to handle this error??

		# NEED TO IMPLEMENT READY, playback position for READY(?)

		else:
			if _request_set['status']== 'Playing':
				logging.debug("IN PLAYING")

			if _request_set['status']== 'Stopped':
				logging.debug("IN STOPPED")

			if _request_set['status']== 'Ready':
				logging.debug("IN READY")

			logging.debug("Sending " + str(_request_set['status']) + " status update")
			response = send_request(_comm_sock, "status_update", {"clientID": client_id}, _request_set, "POST")

if __name__ == "__main__":
	thread1 = Thread(target = playback_func, args =() )
	thread2 = Thread(target = serv_func, args=() )
	thread3 = Thread(target = communicate_func, args =() )
	thread1.start()
	thread2.start()
	thread3.start()
	thread1.join()
	thread2.join()
	thread3.join()
<commit_msg>More refactoring, etc<commit_after>#!/usr/bin/python

""" Program to receive HTTP request from server with JSON body which will
contain the data for the song to be played/paused/stopped. The program will
use three threads.

Thread 1 - This thread will be used for playback.
Thread 2 - This thread will be used to keep the connection with the Server open
and when it timesout, the connection will be reopened and communicate with the playback thread.
Thread 3 - This thread will send Thread 1 commands for the song to be played/paused etc and also
communicate with the server.

The threads will use a queue which will be a basic FIFO implementation holding object types
which represent the request for the thread to execute. When the queue dequeues the thread will
execute that command and keep dequeueing and executing till the queue it empty in which case the
thread will wait on the queue.

@ Author - Anant Goel
@ date - April 2014
@ purpose - CS 252 Lab 6.

"""

#
# NOTES:
#
# Author: Jason P. Rahman
#
# Added code to handle:
#	The case when a song file is already present
#	Record the next song ID as a state machine
#	Record the currently playing song ID
#	Added a sanity check for the "Stopped" code
#		More places similar checks can be made
#	Added a few other notes
#
#	Next Steps:
#		Update stopped code to send Ready/Send song request/wait for previous song request
#		Continue adding more sanity checked
#		Continue tracking _current_song _next_song state variables
#

#Importing the thread function
from threading import Thread
import threading
import httplib
import json
import socket
import sys
import Queue
import pygame
import os
import fnmatch

import logging

logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s')

timeout = 50
_Rlock = threading.RLock()
flag_update_func = 0
SONG_END = pygame.USEREVENT + 1

server_playback_queue = Queue.Queue(0)
playback_connection_queue = Queue.Queue(0)
connection_playback_queue = Queue.Queue(0)
server_communication_ID_queue = Queue.Queue(0)

# Define "Constants"
UNKNOWN = -1
PLAYING = 1
STOPPED = 2
DOWNLOADING = 3
READY = 4

client_id = UNKNOWN

# Initialize state to default values
current_song = UNKNOWN
current_song_state = UNKNOWN
next_song = UNKNOWN
next_song_state = UNKNOWN

server_url = "klamath.dnsdynamic.com"
server_port = "5050"

#
# query_vars is a dictionary of query variables
#
def create_url(request, query_vars = {}):
	global server_url

	url = server_url + ":" + server_port + "/speaker/" + request

	# Add query vars iff we have query vars to add
	if len(query_vars) > 0:
		url = url + "?"
		count = 0
		for key in query_vars:
			if count != 0:
				url = url + "&"
			url = url + str(key) + "=" + str(query_vars[key])

	return url

#
# Send a request over the given socket
#
def send_request(http_socket, request, query_vars = {}, body_params = {}, method = "GET"):
	headers = {"Content-Type": "application/json"}

	url = create_url(request, query_vars)

	logging.info("Sending request to: " + url)

	if body_params.keys() != 0:
		body = json.dumps(body_params, encoding = "ASCII")
	else:
		body = ""

	http_socket.request(method, url, body, headers)
	return http_socket.getresponse()

#
# Main function for the server thread
#
def update_func():
	global client_id
	global server_url

	global server_url
	global server_port

	http_connection = httplib.HTTPConnection(server_url, server_port, timeout = timeout)
	socket.setdefaulttimeout(timeout)

	if client_id == UNKNOWN:

		logging.info("Authenticating")
		server_response = send_request(http_connection, "authenticate", {}, {"pin":1234}, "POST")

		if server_response.status != 200:
			sys.exit()

		logging.info("Authenticated response " + str(server_response.status) + " " + server_response.reason)

		server_response = json.loads(server_response.read())

		client_id = server_response['id']
		logging.debug("Authenticated with client ID " + str(client_id))
		server_communication_ID_queue.put(client_id)

	try:
		while True:
			logging.info("Sending request_update"))

			update_response = send_request(http_connection, "request_update", {"clientID": client_id})

			logging.info("Request Update Response")
			logging.info(str(update_response.status) + " " + str(update_response.reason) + " " + str(update_response.getheaders()))
			#Push update_response.read() the Queue it could also be the playcommand

			if update_response.status == 200:

				rresp = update_response.read()
				logging.debug("Message from server: "+str(rresp))
				server_playback_queue.put(rresp)
			else:
				logging.warning("Exiting")
				sys.exit()

	except socket.timeout:
		logging.debug("Timeout for connection")

		# NOTE: This will lead to stack overflow
		update_func()

	http_connection.close()
	# DEAUTHENTICATE
	params1 = json.dumps({"id":str(client_id)},encoding = "ASCII")
	headers = {"Content-Type":"application/json"}
	http_connection.request("POST","klamath.dnsdynamic.com:5050/speaker/deauthenticate",params1,headers)

#
# Main function for playback control thread
#
def playback_func():
	global next_song
	global current_song
	global next_song_state
	global current_song_state

	pygame.mixer.init() #might have to make global if going to recursively call

	_message = {"id":"","status":"","position":""}

	while True:

		try:

			# Get the update from the queue, but use timeout
			# to multiplex this with the Pygame event loop
			update_body = server_playback_queue.get(True, 0.1)

			response = json.loads(update_body)
			server_playback_queue.task_done()

			update_type = response['update_type']

			logging.debug("Update type is = " + update_type)

			if update_type == "playbackcommand":
				values = response['values']
				song_id = values['id']
				command = values['command']

				if command == 'Play':

					logging.info("Play command recieved")

					if song_id == current_song:

						logging.error('Play command for current song, ignoring')

					elif current_song_state != PLAYING and song_id == next_song and next_song_state == READY:

						logging.info('Playback command received for next song')

						# Set next song to UNKNOWN since the next song
						# because the current song and we don't know
						# the new next song
						next_song = UNKNOWN
						next_song_state = UNKNOWN

						# Update current song
						current_song = next_song
						current_song_state = PLAYING

						# Start playback via PyGame
						pygame.mixer.music.set_endevent(SONG_END)
						pygame.mixer.music.load(str(song_id))
						pygame.mixer.music.play()

						#while pygame.mixer.music.get_busy():
						#	pygame.time.Clock().tick(10)

						logging.debug('Started playback through PyGame')
						logging.debug('Sending Playing status message to communcation thread')

						_message['id'] = str(current_song)
						_message['status'] = 'Playing'
						_message['position'] = str(0)

						logging.info("The message in Play is " + str(_message))

						playback_connection_queue.put(_message)

					elif song_id == next_song and next_song_state == UNKNOWN:

						logging.info('Play command for unavailable song, requesting from server')

						# We need to get the next song from the server
						_message['id'] = str(song_id)
						_message['status'] = 'need_song'
						_message['position'] = str(0);

						playback_connection_queue.put(_message)

				if command == 'Stop':

					# Sanity check for command
					if song_id == current_song:

						# Check the current state of the song
						if current_song_state == PLAYING:
							pygame.mixer.music.stop()

							_message['id'] = str(song_id)
							_message['status'] = 'Stopped'
							_message['position'] = str(pygame.mixer.music.get_pos())

							playback_connection_queue.put(_message)
							# TODO Look at _next_song
							# and send requestSong/ready or wait until
							# previous ready call succeeds
						else:
							pass # TODO decide how to handle this

					else:
						# TODO Error, how to handle this
						logging.error("Stop command from server for wrong song")

			if update_type == "upcoming_song":

				logging.info("Processing upcoming_song update")

				values = response['values']
				song_id = values['id']

				logging.debug("Setting next_song to " + str(song_id))

				# Set next song
				next_song = song_id
				next_song_state = UNKNOWN

				found_song = 0

				for file in os.listdir('.'):
					if fnmatch.fnmatch(file,str(song_id)):
						found_song = 1
					else:
						pass

				if found_song == 0: # We need the file, so request first

					logging.debug('Could not find song, asking next')

					_message['id'] = str(song_id)
					_message['status'] = 'need_song'
					_message['position'] = str(0);

					playback_connection_queue.put(_message)
				else:
					logging.debug('Song ready for playback')
					next_song_state = READY

					if current_song_state != PLAYING:
						# We aren't playing anything right now
						# So send ready immediately since we have the file

						_message['id'] = str(song_id)
						_message['status'] = 'Ready'
						_message['position'] = '0'
						playback_connection_queue.put(_message)

		except Queue.Empty:

			# If the queue is empty, extract an event from PyGame
			if current_song_state == PLAYING:
				event = pygame.event.poll()
				if event.type == SONG_END:
					current_song_state = STOPPED

					_message['id'] = current_song
					_message['status'] = 'Stopped'
					_message['position'] = str(pygame.mixer.music.get_pos())
					playback_connection_queue.put(_message)

#
# Main function for server communication thread
#
def communicate_func():
	while True:

		global client_id

		global server_url
		global server_port

		global next_song
		global current_song
		global next_song_state
		global current_song_state

		# Pop the specific request from the Queue, depending on that do the following
		playback_message = playback_connection_queue.get()
		playback_connection_queue.task_done()

		logging.info("Message in communication thread: " + str(playback_message))

		logging.debug("communicate_func received request: " + str(playback_message))
		_comm_sock = httplib.HTTPConnection(server_url, server_port, timeout = timeout)

		if client_id == UNKNOWN:

			client_id = server_communication_ID_queue.get() # Getting the clientID from the queue
			server_communication_ID_queue.task_done()

			logging.info("client_id in communicate_func is " + str(client_id))

		if playback_message['status']=='need_song':

			logging.debug("IN NEED SONG with song_id = " + str(song_id))

			# Request song data from server
			song_id = playback_message['id']
			song_data_response = send_request(_comm_sock, "request_song", {"clientID": client_id, "songID": song_id})

			logging.debug("Song Data Response: " + str(song_data_response.status) + " " + str(song_data_response.reason))

			if song_data_response.status == 200:
				logging.debug("Song song data from server, saving to file")

				_song_data= song_data_response.read()

				output_file = open(str(song_id),'w')
				output_file.write(_song_data)
				output_file.close()

				if current_song_state != PLAYING:
					# Send ready message to the server

					playback_message['status'] = 'Ready'
					logging.debug("The Message in playing state is "+str(playback_message))
					send_request(_comm_sock, "status_update", {"clientID": client_id}, playback_message, "POST")

				else:

					# Update status in the background
					# When PyGame finishes playing, it will see the status
					# And begin send ready
					next_song_state = READY

			else:

				logging.error("Failed to get song data from server")
				# TODO How to handle this error??

		# NEED TO IMPLEMENT READY, playback position for READY(?)

		else:

			logging.debug("Sending " + str(playback_message['status']) + " status update")
			response = send_request(_comm_sock, "status_update", {"clientID": client_id}, playback_message, "POST")

			# TODO Check response status from the server (Need 200)

			# If we stopped, we need to check if we can start the next song
			if playback_message['status'] == 'Stopped':

				# Tell the server we are ready to play the next song
				playback_message['id'] = next_song
				playback_message['status'] = 'Ready'
				playback_message['position'] = '0'
				response = send_request(_comm_sock, "status_update", {"clientID": client_id}, playback_message, "POST")
				# TODO Check response status

if __name__ == "__main__":
	thread1 = Thread(target = playback_func, args =() )
	thread2 = Thread(target = update_func, args=() )
	thread3 = Thread(target = communicate_func, args =() )
	thread1.start()
	thread2.start()
	thread3.start()
	thread1.join()
	thread2.join()
	thread3.join()
<|endoftext|>


<commit_before>from information_gain_ratio import *
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.cross_validation import train_test_split, cross_val_score
from process_loaded_data import *
from sklearn.metrics import classification_report
from imblearn.under_sampling import RandomUnderSampler
from imblearn.over_sampling import RandomOverSampler, SMOTE
from sklearn.cluster import DBSCAN
import numpy as np
import dill as pickle
import pandas as pd
# from evaltestcvbs import EvalTestCVBS

def evaluate_model(model, X_train, y_train):
    '''
    INPUT
         - model: this is a classification model from sklearn
         - X_train: 2d array of the features
         - y_train: 1d array of the target
    OUTPUT
         - information about the model's accuracy using 10
         fold cross validation
         - model: the fit model
    Returns the model
    '''
    print(np.mean(cross_val_score(model, X_train, y_train,
                                  cv=10, n_jobs=-1, verbose=10)))
    model.fit(X_train, y_train)
    return model

def balance_classes(sm, X, y):
    '''
    INPUT
         - sm: imblearn oversampling/undersampling method
         - X: 2d array of features
         - y: 1d array of targets
    OUTPUT
         - X (balanced feature set)
         - y (balanced feature set)
    Returns X and y after being fit with the resampling method
    '''
    X, y = sm.fit_sample(X, y)
    return X, y

def cluster_based_oversampling_with_smote(metric, X_train, y_train, sm):
    '''
    INPUT
         - metric: this is the kind of distance formula DBSCAN will use
         - X_train: 2d array of features
         - y_train: 1d array of targets
         - sm: imblearn oversampling/undersampling method
    OUTPUT
         - X_train (stratified and balanced feature set)
         - y_train (stratified and balanced feature set)
    Clusters X_train using dbscan with jaccard similarity, then oversamples
    or undersamples
    '''
    db = DBSCAN(metric=metric)
    db.fit(X_train)
    X_list = []
    y_list = []
    for cluster in np.unique(db.labels_):
        X_samp = X_train[db.labels_ == cluster]
        y_samp = y_train[db.labels_ == cluster]
        X_samp, y_samp = balance_classes(sm, X_train, y_train)
        X_list.append(X_samp)
        y_list.append(y_samp)
    return np.vstack(X_list), np.hstack(y_list)

def view_classification_report(model, X_test, y_test):
    '''
    INPUT
         - model: an sklearn classifier model that has already been fit
         - X_test: 2d array of the features
         - y_test: 1d array of the target
    OUTPUT
         - information on the classifiaction performance of the model
    Returns none
    '''
    y_pred = model.predict(X_test)
    print(classification_report(y_test, y_pred))

def check_classifier_with_different_test_subsamples(model, X_test, y_test):
    '''
    INPUT
         - model: classifier model fit with sklearn
         - X_test: 2d array of the features
         - y_test: 1d array of the target
    OUTPUT
         - classification report for different classifier output splits
    returns nothing
    '''
    stepsets = np.arange(start=0.05, stop=1, step=.05)
    pass

def write_model_to_pkl(model, model_name):
    '''
    INPUT
         - model_name: str, this is the name of the model
         - model: the sklearn classification model that will be saved
    OUTPUT
         - saves the model to a pkl file
    Returns None
    '''
    with open('models/{}_model.pkl'.format(model_name), 'w+') as f:
        pickle.dump(model, f)

def view_feature_importances(df, model):
    '''
    INPUT
         - df: dataframe which has the original data
         - model: this is the sklearn classification model that has
         already been fit (work with tree based models)
    OUTPUT
         - prints the feature importances in descending order
    Returns nothing
    '''
    columns = df.columns
    features = model.feature_importances_
    featimps = []
    for column, feature in zip(columns, features):
        featimps.append([column, feature])
    print(pd.DataFrame(featimps, columns=['Features',
                       'Importances']).sort_values(by='Importances',
                                                   ascending=False))

if __name__ == "__main__":
    df = pd.read_csv('data/training_df.csv')
    df.drop('Unnamed: 0', axis=1, inplace=True)
    user_id_array = df.pop('id')
    y = df.pop('label')
    X = df.values
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2)
    X_train_b, y_train_b = balance_classes(RandomUnderSampler(),
                                           X_train, y_train)
    # model = RandomForestClassifier(n_jobs=-1)
    model = MultinomialNB()
    model = evaluate_model(model, X_train_b, y_train_b)
    print("\nthis is the model performance on the training data\n")
    view_classification_report(model, X_train_b, y_train_b)
    print("this is the model performance on the test data\n")
    view_classification_report(model, X_test, y_test)
    print("this is the model performance on different split ratios\n")
    # etcb = EvalTestCVBS(model, .05, .7, .05, 10)
    # percent_range, avg_precision_0, avg_recall_0, avg_precision_1, \
    #     avg_recall_1 = etcb.evaluate_data(X_test, y_test)
    # etcb.plot_performance()
    # print("\nthese are the model feature importances\n")
    # view_feature_importances(df, model)
    # write_model_to_pkl(model, 'vanilla_random_forest')
<commit_msg>got the imported evaltestcvbs to work<commit_after>from information_gain_ratio import *
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.cross_validation import train_test_split, cross_val_score
from process_loaded_data import *
from sklearn.metrics import classification_report
from imblearn.under_sampling import RandomUnderSampler
from imblearn.over_sampling import RandomOverSampler, SMOTE
from sklearn.cluster import DBSCAN
import numpy as np
import dill as pickle
import pandas as pd
# from evaltestcvbs import EvalTestCVBS
from testsetevaluation import Eval

def evaluate_model(model, X_train, y_train):
    '''
    INPUT
         - model: this is a classification model from sklearn
         - X_train: 2d array of the features
         - y_train: 1d array of the target
    OUTPUT
         - information about the model's accuracy using 10
         fold cross validation
         - model: the fit model
    Returns the model
    '''
    print(np.mean(cross_val_score(model, X_train, y_train,
                                  cv=10, n_jobs=-1, verbose=10)))
    model.fit(X_train, y_train)
    return model

def balance_classes(sm, X, y):
    '''
    INPUT
         - sm: imblearn oversampling/undersampling method
         - X: 2d array of features
         - y: 1d array of targets
    OUTPUT
         - X (balanced feature set)
         - y (balanced feature set)
    Returns X and y after being fit with the resampling method
    '''
    X, y = sm.fit_sample(X, y)
    return X, y

def cluster_based_oversampling_with_smote(metric, X_train, y_train, sm):
    '''
    INPUT
         - metric: this is the kind of distance formula DBSCAN will use
         - X_train: 2d array of features
         - y_train: 1d array of targets
         - sm: imblearn oversampling/undersampling method
    OUTPUT
         - X_train (stratified and balanced feature set)
         - y_train (stratified and balanced feature set)
    Clusters X_train using dbscan with jaccard similarity, then oversamples
    or undersamples
    '''
    db = DBSCAN(metric=metric)
    db.fit(X_train)
    X_list = []
    y_list = []
    for cluster in np.unique(db.labels_):
        X_samp = X_train[db.labels_ == cluster]
        y_samp = y_train[db.labels_ == cluster]
        X_samp, y_samp = balance_classes(sm, X_train, y_train)
        X_list.append(X_samp)
        y_list.append(y_samp)
    return np.vstack(X_list), np.hstack(y_list)

def view_classification_report(model, X_test, y_test):
    '''
    INPUT
         - model: an sklearn classifier model that has already been fit
         - X_test: 2d array of the features
         - y_test: 1d array of the target
    OUTPUT
         - information on the classifiaction performance of the model
    Returns none
    '''
    y_pred = model.predict(X_test)
    print(classification_report(y_test, y_pred))

def check_classifier_with_different_test_subsamples(model, X_test, y_test):
    '''
    INPUT
         - model: classifier model fit with sklearn
         - X_test: 2d array of the features
         - y_test: 1d array of the target
    OUTPUT
         - classification report for different classifier output splits
    returns nothing
    '''
    stepsets = np.arange(start=0.05, stop=1, step=.05)
    pass

def write_model_to_pkl(model, model_name):
    '''
    INPUT
         - model_name: str, this is the name of the model
         - model: the sklearn classification model that will be saved
    OUTPUT
         - saves the model to a pkl file
    Returns None
    '''
    with open('models/{}_model.pkl'.format(model_name), 'w+') as f:
        pickle.dump(model, f)

def view_feature_importances(df, model):
    '''
    INPUT
         - df: dataframe which has the original data
         - model: this is the sklearn classification model that has
         already been fit (work with tree based models)
    OUTPUT
         - prints the feature importances in descending order
    Returns nothing
    '''
    columns = df.columns
    features = model.feature_importances_
    featimps = []
    for column, feature in zip(columns, features):
        featimps.append([column, feature])
    print(pd.DataFrame(featimps, columns=['Features',
                       'Importances']).sort_values(by='Importances',
                                                   ascending=False))

if __name__ == "__main__":
    df = pd.read_csv('data/training_df.csv')
    df.drop('Unnamed: 0', axis=1, inplace=True)
    user_id_array = df.pop('id')
    y = df.pop('label')
    X = df.values
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2)
    X_train_b, y_train_b = balance_classes(RandomUnderSampler(),
                                           X_train, y_train)
    model = RandomForestClassifier(n_jobs=-1)
    # model = MultinomialNB()
    model = evaluate_model(model, X_train_b, y_train_b)
    print("\nthis is the model performance on the training data\n")
    view_classification_report(model, X_train_b, y_train_b)
    print("this is the model performance on the test data\n")
    view_classification_report(model, X_test, y_test)
    print("this is the model performance on different split ratios\n")
    etcb = Eval(model, .05, .7, .05, 10)
    percent_range, avg_precision_0, avg_recall_0, avg_precision_1, \
        avg_recall_1 = etcb.evaluate_data(X_test, y_test)
    etcb.plot_performance()
    # print("\nthese are the model feature importances\n")
    # view_feature_importances(df, model)
    # write_model_to_pkl(model, 'vanilla_random_forest')
<|endoftext|>


<commit_before>import regex
import unicodedata

from .language_info import get_language_info
from .transliterate import transliterate

MARK_RE = regex.compile(r'[\p{Mn}\N{ARABIC TATWEEL}]', regex.V1)

DIGIT_RE = regex.compile('\d')
MULTI_DIGIT_RE = regex.compile('\d[\d.,]+')

def preprocess_text(text, language):
    """
    This function applies pre-processing steps that convert forms of words
    considered equivalent into one standardized form.

    As one straightforward step, it case-folds the text. For the purposes of
    wordfreq and related tools, a capitalized word shouldn't have a different
    frequency from its lowercase version.

    The steps that are applied in order, only some of which apply to each
    language, are:

    - NFC or NFKC normalization, as needed for the language
    - Transliteration of multi-script languages
    - Abjad mark removal
    - Case folding
    - Fixing of diacritics

    We'll describe these steps out of order, to start with the more obvious
    steps.

    Case folding
    ------------

    The most common effect of this function is that it case-folds alphabetic
    text to lowercase:

    >>> preprocess_text('Word', 'en')
    'word'

    This is proper Unicode-aware case-folding, so it eliminates distinctions
    in lowercase letters that would not appear in uppercase. This accounts for
    the German ß and the Greek final sigma:

    >>> preprocess_text('groß', 'de')
    'gross'
    >>> preprocess_text('λέξις', 'el')
    'λέξισ'

    In Turkish (and Azerbaijani), case-folding is different, because the
    uppercase and lowercase I come in two variants, one with a dot and one
    without. They are matched in a way that preserves the number of dots, which
    the usual pair of "I" and "i" do not.

    >>> preprocess_text('HAKKINDA İSTANBUL', 'tr')
    'hakkında istanbul'

    Fixing of diacritics
    --------------------

    While we're talking about Turkish: the Turkish alphabet contains letters
    with cedillas attached to the bottom. In the case of "ş" and "ţ", these
    letters are very similar to two Romanian letters, "ș" and "ț", which have
    separate _commas_ below them.

    (Did you know that a cedilla is not the same as a comma under a letter? I
    didn't until I started dealing with text normalization. My keyboard layout
    even inputs a letter with a cedilla when you hit Compose+comma.)

    Because these letters look so similar, and because some fonts only include
    one pair of letters and not the other, there are many cases where the
    letters are confused with each other. Our preprocessing normalizes these
    Turkish and Romanian letters to the letters each language prefers.

    >>> preprocess_text('kișinin', 'tr')   # comma to cedilla
    'kişinin'
    >>> preprocess_text('ACELAŞI', 'ro')   # cedilla to comma
    'același'

    Unicode normalization
    ---------------------

    Unicode text is NFC normalized in most languages, removing trivial
    distinctions between strings that should be considered equivalent in all
    cases:

    >>> word = preprocess_text('natu\N{COMBINING DIAERESIS}rlich', 'de')
    >>> word
    'natürlich'
    >>> '\N{LATIN SMALL LETTER U WITH DIAERESIS}' in word
    True

    NFC normalization is sufficient (and NFKC normalization is a bit too strong)
    for many languages that are written in cased, alphabetic scripts.
    Languages in other scripts tend to need stronger normalization to properly
    compare text. So we use NFC normalization when the language's script is
    Latin, Greek, or Cyrillic, and we use NFKC normalization for all other
    languages.

    Here's an example in Japanese, where preprocessing changes the width (and
    the case) of a Latin letter that's used as part of a word:

    >>> preprocess_text('Ｕターン', 'ja')
    'uターン'

    In Korean, NFKC normalization is important because it aligns two different
    ways of encoding text -- as individual letters that are grouped together
    into square characters, or as the entire syllables that those characters
    represent:

    >>> word = '\u1102\u1161\u11c0\u1106\u1161\u11af'
    >>> word
    '낱말'
    >>> len(word)
    6
    >>> word = preprocess_text(word, 'ko')
    >>> word
    '낱말'
    >>> len(word)
    2

    Abjad mark removal
    ------------------

    There are many abjad languages, such as Arabic, Hebrew, Persian, and Urdu,
    where words can be marked with vowel points but rarely are. In languages
    that use abjad scripts, we remove all modifiers that are classified by
    Unicode as "marks". We also remove an Arabic character called the tatweel,
    which is used to visually lengthen a word.

    >>> preprocess_text("كَلِمَة", 'ar')
    'كلمة'
    >>> preprocess_text("الحمــــــد", 'ar')
    'الحمد'

    Transliteration of multi-script languages
    -----------------------------------------

    Some languages are written in multiple scripts, and require special care.
    These languages include Chinese, Serbian, and Azerbaijani.

    In Serbian, there is a well-established mapping from Cyrillic letters to
    Latin letters. We apply this mapping so that Serbian is always represented
    in Latin letters.

    >>> preprocess_text('схваташ', 'sr')
    'shvataš'

    The transliteration is more complete than it needs to be to cover just
    Serbian, so that -- for example -- borrowings from Russian can be
    transliterated, instead of coming out in a mixed script.

    >>> preprocess_text('культуры', 'sr')
    "kul'tury"

    Azerbaijani (Azeri) has a similar transliteration step to Serbian,
    and then the Latin-alphabet text is handled similarly to Turkish.

    >>> preprocess_text('бағырты', 'az')
    'bağırtı'

    We don't transliterate Traditional to Simplified Chinese in this step.
    There are some steps where we unify them internally: see chinese.py
    for more information.
    """
    # NFC or NFKC normalization, as needed for the language
    info = get_language_info(language)
    text = unicodedata.normalize(info['normal_form'], text)

    # Transliteration of multi-script languages
    if info['transliteration'] is not None:
        text = transliterate(info['transliteration'], text)

    # Abjad mark removal
    if info['remove_marks']:
        text = remove_marks(text)

    # Case folding
    if info['dotless_i']:
        text = casefold_with_i_dots(text)
    else:
        text = text.casefold()

    # Fixing of diacritics
    if info['diacritics_under'] == 'commas':
        text = cedillas_to_commas(text)
    elif info['diacritics_under'] == 'cedillas':
        text = commas_to_cedillas(text)

    return text

def remove_marks(text):
    """
    Remove decorations from words in abjad scripts:

    - Combining marks of class Mn, which tend to represent non-essential
      vowel markings.
    - Tatweels, horizontal segments that are used to extend or justify an
      Arabic word.
    """
    return MARK_RE.sub('', text)

def casefold_with_i_dots(text):
    """
    Convert capital I's and capital dotted İ's to lowercase in the way
    that's appropriate for Turkish and related languages, then case-fold
    the rest of the letters.
    """
    text = unicodedata.normalize('NFC', text).replace('İ', 'i').replace('I', 'ı')
    return text.casefold()

def commas_to_cedillas(text):
    """
    Convert s and t with commas (ș and ț) to cedillas (ş and ţ), which is
    preferred in Turkish.

    Only the lowercase versions are replaced, because this assumes the
    text has already been case-folded.
    """
    return text.replace(
        '\N{LATIN SMALL LETTER S WITH COMMA BELOW}',
        '\N{LATIN SMALL LETTER S WITH CEDILLA}'
    ).replace(
        '\N{LATIN SMALL LETTER T WITH COMMA BELOW}',
        '\N{LATIN SMALL LETTER T WITH CEDILLA}'
    )

def cedillas_to_commas(text):
    """
    Convert s and t with cedillas (ş and ţ) to commas (ș and ț), which is
    preferred in Romanian.

    Only the lowercase versions are replaced, because this assumes the
    text has already been case-folded.
    """
    return text.replace(
        '\N{LATIN SMALL LETTER S WITH CEDILLA}',
        '\N{LATIN SMALL LETTER S WITH COMMA BELOW}'
    ).replace(
        '\N{LATIN SMALL LETTER T WITH CEDILLA}',
        '\N{LATIN SMALL LETTER T WITH COMMA BELOW}'
    )

def _sub_zeroes(match):
    """
    Given a regex match, return what it matched with digits replaced by
    zeroes.
    """
    return DIGIT_RE.sub('0', match.group(0))

def smash_numbers(text):
    """
    Replace sequences of multiple digits with zeroes, so we don't need to
    distinguish the frequencies of thousands of numbers.
    """
    return MULTI_DIGIT_RE.sub(_sub_zeroes, text)
<commit_msg>fix a deprecation warning by using raw strings<commit_after>import regex
import unicodedata

from .language_info import get_language_info
from .transliterate import transliterate

MARK_RE = regex.compile(r'[\p{Mn}\N{ARABIC TATWEEL}]', regex.V1)

DIGIT_RE = regex.compile(r'\d')
MULTI_DIGIT_RE = regex.compile(r'\d[\d.,]+')

def preprocess_text(text, language):
    """
    This function applies pre-processing steps that convert forms of words
    considered equivalent into one standardized form.

    As one straightforward step, it case-folds the text. For the purposes of
    wordfreq and related tools, a capitalized word shouldn't have a different
    frequency from its lowercase version.

    The steps that are applied in order, only some of which apply to each
    language, are:

    - NFC or NFKC normalization, as needed for the language
    - Transliteration of multi-script languages
    - Abjad mark removal
    - Case folding
    - Fixing of diacritics

    We'll describe these steps out of order, to start with the more obvious
    steps.

    Case folding
    ------------

    The most common effect of this function is that it case-folds alphabetic
    text to lowercase:

    >>> preprocess_text('Word', 'en')
    'word'

    This is proper Unicode-aware case-folding, so it eliminates distinctions
    in lowercase letters that would not appear in uppercase. This accounts for
    the German ß and the Greek final sigma:

    >>> preprocess_text('groß', 'de')
    'gross'
    >>> preprocess_text('λέξις', 'el')
    'λέξισ'

    In Turkish (and Azerbaijani), case-folding is different, because the
    uppercase and lowercase I come in two variants, one with a dot and one
    without. They are matched in a way that preserves the number of dots, which
    the usual pair of "I" and "i" do not.

    >>> preprocess_text('HAKKINDA İSTANBUL', 'tr')
    'hakkında istanbul'

    Fixing of diacritics
    --------------------

    While we're talking about Turkish: the Turkish alphabet contains letters
    with cedillas attached to the bottom. In the case of "ş" and "ţ", these
    letters are very similar to two Romanian letters, "ș" and "ț", which have
    separate _commas_ below them.

    (Did you know that a cedilla is not the same as a comma under a letter? I
    didn't until I started dealing with text normalization. My keyboard layout
    even inputs a letter with a cedilla when you hit Compose+comma.)

    Because these letters look so similar, and because some fonts only include
    one pair of letters and not the other, there are many cases where the
    letters are confused with each other. Our preprocessing normalizes these
    Turkish and Romanian letters to the letters each language prefers.

    >>> preprocess_text('kișinin', 'tr')   # comma to cedilla
    'kişinin'
    >>> preprocess_text('ACELAŞI', 'ro')   # cedilla to comma
    'același'

    Unicode normalization
    ---------------------

    Unicode text is NFC normalized in most languages, removing trivial
    distinctions between strings that should be considered equivalent in all
    cases:

    >>> word = preprocess_text('natu\N{COMBINING DIAERESIS}rlich', 'de')
    >>> word
    'natürlich'
    >>> '\N{LATIN SMALL LETTER U WITH DIAERESIS}' in word
    True

    NFC normalization is sufficient (and NFKC normalization is a bit too strong)
    for many languages that are written in cased, alphabetic scripts.
    Languages in other scripts tend to need stronger normalization to properly
    compare text. So we use NFC normalization when the language's script is
    Latin, Greek, or Cyrillic, and we use NFKC normalization for all other
    languages.

    Here's an example in Japanese, where preprocessing changes the width (and
    the case) of a Latin letter that's used as part of a word:

    >>> preprocess_text('Ｕターン', 'ja')
    'uターン'

    In Korean, NFKC normalization is important because it aligns two different
    ways of encoding text -- as individual letters that are grouped together
    into square characters, or as the entire syllables that those characters
    represent:

    >>> word = '\u1102\u1161\u11c0\u1106\u1161\u11af'
    >>> word
    '낱말'
    >>> len(word)
    6
    >>> word = preprocess_text(word, 'ko')
    >>> word
    '낱말'
    >>> len(word)
    2

    Abjad mark removal
    ------------------

    There are many abjad languages, such as Arabic, Hebrew, Persian, and Urdu,
    where words can be marked with vowel points but rarely are. In languages
    that use abjad scripts, we remove all modifiers that are classified by
    Unicode as "marks". We also remove an Arabic character called the tatweel,
    which is used to visually lengthen a word.

    >>> preprocess_text("كَلِمَة", 'ar')
    'كلمة'
    >>> preprocess_text("الحمــــــد", 'ar')
    'الحمد'

    Transliteration of multi-script languages
    -----------------------------------------

    Some languages are written in multiple scripts, and require special care.
    These languages include Chinese, Serbian, and Azerbaijani.

    In Serbian, there is a well-established mapping from Cyrillic letters to
    Latin letters. We apply this mapping so that Serbian is always represented
    in Latin letters.

    >>> preprocess_text('схваташ', 'sr')
    'shvataš'

    The transliteration is more complete than it needs to be to cover just
    Serbian, so that -- for example -- borrowings from Russian can be
    transliterated, instead of coming out in a mixed script.

    >>> preprocess_text('культуры', 'sr')
    "kul'tury"

    Azerbaijani (Azeri) has a similar transliteration step to Serbian,
    and then the Latin-alphabet text is handled similarly to Turkish.

    >>> preprocess_text('бағырты', 'az')
    'bağırtı'

    We don't transliterate Traditional to Simplified Chinese in this step.
    There are some steps where we unify them internally: see chinese.py
    for more information.
    """
    # NFC or NFKC normalization, as needed for the language
    info = get_language_info(language)
    text = unicodedata.normalize(info['normal_form'], text)

    # Transliteration of multi-script languages
    if info['transliteration'] is not None:
        text = transliterate(info['transliteration'], text)

    # Abjad mark removal
    if info['remove_marks']:
        text = remove_marks(text)

    # Case folding
    if info['dotless_i']:
        text = casefold_with_i_dots(text)
    else:
        text = text.casefold()

    # Fixing of diacritics
    if info['diacritics_under'] == 'commas':
        text = cedillas_to_commas(text)
    elif info['diacritics_under'] == 'cedillas':
        text = commas_to_cedillas(text)

    return text

def remove_marks(text):
    """
    Remove decorations from words in abjad scripts:

    - Combining marks of class Mn, which tend to represent non-essential
      vowel markings.
    - Tatweels, horizontal segments that are used to extend or justify an
      Arabic word.
    """
    return MARK_RE.sub('', text)

def casefold_with_i_dots(text):
    """
    Convert capital I's and capital dotted İ's to lowercase in the way
    that's appropriate for Turkish and related languages, then case-fold
    the rest of the letters.
    """
    text = unicodedata.normalize('NFC', text).replace('İ', 'i').replace('I', 'ı')
    return text.casefold()

def commas_to_cedillas(text):
    """
    Convert s and t with commas (ș and ț) to cedillas (ş and ţ), which is
    preferred in Turkish.

    Only the lowercase versions are replaced, because this assumes the
    text has already been case-folded.
    """
    return text.replace(
        '\N{LATIN SMALL LETTER S WITH COMMA BELOW}',
        '\N{LATIN SMALL LETTER S WITH CEDILLA}'
    ).replace(
        '\N{LATIN SMALL LETTER T WITH COMMA BELOW}',
        '\N{LATIN SMALL LETTER T WITH CEDILLA}'
    )

def cedillas_to_commas(text):
    """
    Convert s and t with cedillas (ş and ţ) to commas (ș and ț), which is
    preferred in Romanian.

    Only the lowercase versions are replaced, because this assumes the
    text has already been case-folded.
    """
    return text.replace(
        '\N{LATIN SMALL LETTER S WITH CEDILLA}',
        '\N{LATIN SMALL LETTER S WITH COMMA BELOW}'
    ).replace(
        '\N{LATIN SMALL LETTER T WITH CEDILLA}',
        '\N{LATIN SMALL LETTER T WITH COMMA BELOW}'
    )

def _sub_zeroes(match):
    """
    Given a regex match, return what it matched with digits replaced by
    zeroes.
    """
    return DIGIT_RE.sub('0', match.group(0))

def smash_numbers(text):
    """
    Replace sequences of multiple digits with zeroes, so we don't need to
    distinguish the frequencies of thousands of numbers.
    """
    return MULTI_DIGIT_RE.sub(_sub_zeroes, text)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

##############################################################################
#    Copyright (C) 2012 SICLIC http://siclic.fr
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>
#
#############################################################################
from osv import fields, osv
import datetime as dt
from datetime import datetime, date, timedelta
from dateutil import *
from dateutil.tz import *
#Core abstract model to add SICLIC custom features, such as actions rights calculation (to be used in SICLIC custom GUI)
class OpenbaseCore(osv.Model):
    _auto = True
    _register = False # not visible in ORM registry, meant to be python-inherited only
    _transient = False # True in a TransientModel

    _actions_to_eval = {}
    _fields_names_to_eval = {}
    _actions = {}
    _fields_names = {}

    #keywords to compute filter domain
    DATE_KEYWORDS = ['FIRSTDAYWEEK', 'LASTDAYWEEK',  'FIRSTDAYMONTH',  'LASTDAYMONTH', 'OVERMONTH', 'OUTDATED']
    DATE_FMT = "%Y-%m-%d"
    DATE_TIME_FMT = "%Y-%m-%d %H:%M:%S"

    def _get_actions(self, cr, uid, ids, myFields ,arg, context=None):
        #default value: empty string for each id
        ret = {}.fromkeys(ids,'')
        groups_code = []
        groups_code = [group.code for group in self.pool.get("res.users").browse(cr, uid, uid, context=context).groups_id if group.code]

        for record in self.browse(cr, uid, ids, context=context):
            ret.update({record.id:[key for key,func in self._actions_to_eval[self._name].items() if func(self,cr,uid,record,groups_code)]})
        return ret

    _columns_to_add = {
        'actions':fields.function(_get_actions, method=True, string="Actions possibles",type="char", store=False),
        }

    """
    @param uid: user who whants the metadata
    @return: dict containing number of records (that user can see),
            model fields definition (another dict with field as key and list as definition),
            @todo: filters authorized for this user too ?
            in example : {'count':55, 'fields':{'name':{'string':'Kapweeee', type:'char', 'required':True}}, 'saved_filters': [TODO]}
    """
    def getModelMetadata(self, cr, uid, context=None):
        ret = {'count':0, 'fields':{}}
        #Comment count because special count from client (swif)
        #ret['count'] = self.search(cr, uid, [], count=True, context=context)
        #dict containing default keys to return, even if value is False (OpenERP does not return a key where the val is False)
        mandatory_vals = {'type':False,'required':False,'select':False,'readonly':False, 'help':False}
        #list containing key to return if set
        authorized_vals = ['selection','domain']
        vals_to_retrieve = authorized_vals + mandatory_vals.keys()

        #Get model id
        ret['model_id']  = self.pool.get('ir.model').search(cr, uid, [('model','=',self._name)])[0]

        #for each field, returns all mandatory fields, and return authorized fields if set
        for f, dict_vals in self.fields_get(cr, uid, context=context).items():
            final_val = mandatory_vals.copy()
            for key,val in dict_vals.items():
                if key in vals_to_retrieve:
                    final_val.update({key:val})
            ret['fields'].update({f:final_val})

        return ret

    def __init__(self, cr, pool):
        self._columns.update(self._columns_to_add)
        self._actions_to_eval.setdefault(self._name,{})
        self._fields_names_to_eval.setdefault(self._name,{})

        self._actions_to_eval[self._name].update(self._actions)
        self._fields_names_to_eval[self._name].update(self._fields_names)
        #method to retrieve many2many fields with custom format
        def _get_fields_names(self, cr, uid, ids, name, args, context=None):
            res = {}
            if not isinstance(name, list):
                name = [name]
            for obj in self.browse(cr, uid, ids, context=context):
                #for each field_names to read, retrieve their values
                res[obj.id] = {}
                for fname in name:
                    #many2many browse_record field to map
                    field_ids = obj[self._fields_names_to_eval[self._name][fname]]
                    val = []
                    for item in field_ids:
                        val.append([item.id,item.name_get()[0][1]])
                    res[obj.id].update({fname:val})
            return res

        super(OpenbaseCore,self).__init__(cr,pool)
         #add _field_names to fields definition of the model
        for f in self._fields_names.keys():
            #force name of new field with '_names' suffix
            self._columns.update({f:fields.function(_get_fields_names, type='char',method=True, multi='field_names',store=False)})

    def search(self, cr, uid, args, offset=0, limit=None, order=None, context=None, count=False):
        new_args = []
        #fields = self.fields_get(cr, uid, context=context).items()
        for id, domain  in enumerate(args) :
            #Test if domain tuple = ('key','operator','value')
            if len(domain) == 3 :
                #Get key, operator and domain
                k, o, v = domain
                #Get field's type
                try:
                    type = self._columns[k]._type
                except (KeyError):
                    type = None
                #if domain contains special keyword
                if v in self.DATE_KEYWORDS :
                    #Adapts keyword in domain to specials filter that need to be computed (cf get_date_from_keyword method)
                    domain[2] = self.get_date_from_keyword(v)
                elif  type != None and type == 'datetime':
                    try:
                        #Test if already format with hours
                        datetime.strptime(v,self.DATE_TIME_FMT)
                    except ValueError:
                        #Format date with hours
                        domain[2] = datetime.strftime(datetime.strptime(v,self.DATE_FMT), "%Y-%m-%d %H:%M:%S")
            new_args.extend([domain])
        return super(OpenbaseCore, self).search(cr, uid, new_args, offset, limit, order, context, count)

    """
    @param keyword: keyword to compute corresponding date
    @return: return string date for domain search
        domain is used to filter OpenBase object (ask (request), project (intervention)  :
        * from current week
        * from current month
        * delayed (deadline spent)
    """
    def get_date_from_keyword(self, keyword):
        val = ""
        timeDtFrmt = "%Y-%m-%d %H:%M:%S"
        today = date.today()
        start_day_month = dt.datetime(today.year, today.month, 1)
        dates = [today + dt.timedelta(days=i) for i in range(0 - today.weekday(), 7 - today.weekday())]
        if keyword == 'FIRSTDAYWEEK':
             return datetime.strftime(dates[0],timeDtFrmt)
        elif keyword == 'LASTDAYWEEK':
             return datetime.strftime(dates[6],timeDtFrmt)
        elif keyword == 'FIRSTDAYMONTH':
            return datetime.strftime(dt.datetime(today.year, today.month, 1),timeDtFrmt)
        elif keyword == 'LASTDAYMONTH':
            date_on_next_month = start_day_month + dt.timedelta(31)
            start_next_month = dt.datetime(date_on_next_month.year, date_on_next_month.month, 1)
            return datetime.strftime(start_next_month - dt.timedelta(1),timeDtFrmt)
        elif keyword == 'OVERMONTH':
             return datetime.strftime(start_day_month + dt.timedelta(31),timeDtFrmt)
        elif keyword == 'OUTDATED':
            return datetime.strftime(today,timeDtFrmt)
        return val

    def send_mail(self, cr, uid, id, vals, module, model, mail_templates):
        email_obj = self.pool.get("email.template")
        email_tmpl_id = 0
        data_obj = self.pool.get('ir.model.data')
        #first, retrieve template_id according to 'state' parameter
        if vals.get('state','') in mail_templates.keys():
            email_tmpl_id = data_obj.get_object_reference(cr, uid, module,mail_templates.get(vals.get('state')))[1]
            if email_tmpl_id:
                if isinstance(email_tmpl_id, list):
                    email_tmpl_id = email_tmpl_id[0]
                #generate mail and send it
                mail_id = email_obj.send_mail(cr, uid, email_tmpl_id, id)
                self.pool.get("mail.message").write(cr, uid, [mail_id], {})
                self.pool.get("mail.message").send(cr, uid, [mail_id])
<commit_msg> refs #2558 : Filtre avacné demande d'interventions Date de création égal ne fonctionne pas<commit_after># -*- coding: utf-8 -*-

##############################################################################
#    Copyright (C) 2012 SICLIC http://siclic.fr
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>
#
#############################################################################
from osv import fields, osv
import datetime as dt
from datetime import datetime, date, timedelta
from dateutil import *
from dateutil.tz import *
from copy import copy
#Core abstract model to add SICLIC custom features, such as actions rights calculation (to be used in SICLIC custom GUI)
class OpenbaseCore(osv.Model):
    _auto = True
    _register = False # not visible in ORM registry, meant to be python-inherited only
    _transient = False # True in a TransientModel

    _actions_to_eval = {}
    _fields_names_to_eval = {}
    _actions = {}
    _fields_names = {}

    #keywords to compute filter domain
    DATE_KEYWORDS = ['FIRSTDAYWEEK', 'LASTDAYWEEK',  'FIRSTDAYMONTH',  'LASTDAYMONTH', 'OVERMONTH', 'OUTDATED']
    DATE_FMT = "%Y-%m-%d"
    DATE_TIME_FMT = "%Y-%m-%d %H:%M:%S"

    def _get_actions(self, cr, uid, ids, myFields ,arg, context=None):
        #default value: empty string for each id
        ret = {}.fromkeys(ids,'')
        groups_code = []
        groups_code = [group.code for group in self.pool.get("res.users").browse(cr, uid, uid, context=context).groups_id if group.code]

        for record in self.browse(cr, uid, ids, context=context):
            ret.update({record.id:[key for key,func in self._actions_to_eval[self._name].items() if func(self,cr,uid,record,groups_code)]})
        return ret

    _columns_to_add = {
        'actions':fields.function(_get_actions, method=True, string="Actions possibles",type="char", store=False),
        }

    """
    @param uid: user who whants the metadata
    @return: dict containing number of records (that user can see),
            model fields definition (another dict with field as key and list as definition),
            @todo: filters authorized for this user too ?
            in example : {'count':55, 'fields':{'name':{'string':'Kapweeee', type:'char', 'required':True}}, 'saved_filters': [TODO]}
    """
    def getModelMetadata(self, cr, uid, context=None):
        ret = {'count':0, 'fields':{}}
        #Comment count because special count from client (swif)
        #ret['count'] = self.search(cr, uid, [], count=True, context=context)
        #dict containing default keys to return, even if value is False (OpenERP does not return a key where the val is False)
        mandatory_vals = {'type':False,'required':False,'select':False,'readonly':False, 'help':False}
        #list containing key to return if set
        authorized_vals = ['selection','domain']
        vals_to_retrieve = authorized_vals + mandatory_vals.keys()

        #Get model id
        ret['model_id']  = self.pool.get('ir.model').search(cr, uid, [('model','=',self._name)])[0]

        #for each field, returns all mandatory fields, and return authorized fields if set
        for f, dict_vals in self.fields_get(cr, uid, context=context).items():
            final_val = mandatory_vals.copy()
            for key,val in dict_vals.items():
                if key in vals_to_retrieve:
                    final_val.update({key:val})
            ret['fields'].update({f:final_val})

        return ret

    def __init__(self, cr, pool):
        self._columns.update(self._columns_to_add)
        self._actions_to_eval.setdefault(self._name,{})
        self._fields_names_to_eval.setdefault(self._name,{})

        self._actions_to_eval[self._name].update(self._actions)
        self._fields_names_to_eval[self._name].update(self._fields_names)
        #method to retrieve many2many fields with custom format
        def _get_fields_names(self, cr, uid, ids, name, args, context=None):
            res = {}
            if not isinstance(name, list):
                name = [name]
            for obj in self.browse(cr, uid, ids, context=context):
                #for each field_names to read, retrieve their values
                res[obj.id] = {}
                for fname in name:
                    #many2many browse_record field to map
                    field_ids = obj[self._fields_names_to_eval[self._name][fname]]
                    val = []
                    for item in field_ids:
                        val.append([item.id,item.name_get()[0][1]])
                    res[obj.id].update({fname:val})
            return res

        super(OpenbaseCore,self).__init__(cr,pool)
         #add _field_names to fields definition of the model
        for f in self._fields_names.keys():
            #force name of new field with '_names' suffix
            self._columns.update({f:fields.function(_get_fields_names, type='char',method=True, multi='field_names',store=False)})

    def search(self, cr, uid, args, offset=0, limit=None, order=None, context=None, count=False):
        new_args = []
        #fields = self.fields_get(cr, uid, context=context).items()
        for id, domain  in enumerate(args) :
            #Test if domain tuple = ('key','operator','value')
            if len(domain) == 3 :
                #Get key, operator and domain
                k, o, v = domain
                #Get field's type
                try:
                    type = self._columns[k]._type
                except (KeyError):
                    type = None
                #if domain contains special keyword
                if v in self.DATE_KEYWORDS :
                    #For records filters : Adapts keyword in domain to specials filter that need to be computed (cf get_date_from_keyword method)
                    domain[2] = self.get_date_from_keyword(v)
                elif  type != None and type == 'datetime':
                    try:
                        #Test if already format with hours
                        datetime.strptime(v,self.DATE_TIME_FMT)
                    except ValueError:
                        #Format date with hours
                        domain[2] = datetime.strftime(datetime.strptime(v,self.DATE_FMT), "%Y-%m-%d %H:%M:%S")
                        #if equal method in domain : build domain, example : [('date_to_compare', '>' ,'2014-03-05 00:00:00'), ('date_to_compare', '<' ,'2014-03-05 23:59:59')]
                        if domain[1] ==  "=":
                            #prepare first domain , example : [('date_to_compare', '>' ,'2014-03-05 00:00:00')]
                            domain[1] = ">"
                            #prepare second domain , example : [('date_to_compare', '<' ,'2014-03-05 23:59:59')]
                            new_domain = copy(domain)
                            new_domain[1] = "<"
                            new_domain[2] = datetime.strftime(datetime.strptime(v,self.DATE_FMT), "%Y-%m-%d 23:59:59")
                            new_args.extend([new_domain])
                new_args.extend([domain])

        return super(OpenbaseCore, self).search(cr, uid, new_args, offset, limit, order, context, count)

    """
    @param keyword: keyword to compute corresponding date
    @return: return string date for domain search
        domain is used to filter OpenBase object (ask (request), project (intervention)  :
        * from current week
        * from current month
        * delayed (deadline spent)
    """
    def get_date_from_keyword(self, keyword):
        val = ""
        timeDtFrmt = "%Y-%m-%d %H:%M:%S"
        today = date.today()
        start_day_month = dt.datetime(today.year, today.month, 1)
        dates = [today + dt.timedelta(days=i) for i in range(0 - today.weekday(), 7 - today.weekday())]
        if keyword == 'FIRSTDAYWEEK':
             return datetime.strftime(dates[0],timeDtFrmt)
        elif keyword == 'LASTDAYWEEK':
             return datetime.strftime(dates[6],timeDtFrmt)
        elif keyword == 'FIRSTDAYMONTH':
            return datetime.strftime(dt.datetime(today.year, today.month, 1),timeDtFrmt)
        elif keyword == 'LASTDAYMONTH':
            date_on_next_month = start_day_month + dt.timedelta(31)
            start_next_month = dt.datetime(date_on_next_month.year, date_on_next_month.month, 1)
            return datetime.strftime(start_next_month - dt.timedelta(1),timeDtFrmt)
        elif keyword == 'OVERMONTH':
             return datetime.strftime(start_day_month + dt.timedelta(31),timeDtFrmt)
        elif keyword == 'OUTDATED':
            return datetime.strftime(today,timeDtFrmt)
        return val

    def send_mail(self, cr, uid, id, vals, module, model, mail_templates):
        email_obj = self.pool.get("email.template")
        email_tmpl_id = 0
        data_obj = self.pool.get('ir.model.data')
        #first, retrieve template_id according to 'state' parameter
        if vals.get('state','') in mail_templates.keys():
            email_tmpl_id = data_obj.get_object_reference(cr, uid, module,mail_templates.get(vals.get('state')))[1]
            if email_tmpl_id:
                if isinstance(email_tmpl_id, list):
                    email_tmpl_id = email_tmpl_id[0]
                #generate mail and send it
                mail_id = email_obj.send_mail(cr, uid, email_tmpl_id, id)
                self.pool.get("mail.message").write(cr, uid, [mail_id], {})
                self.pool.get("mail.message").send(cr, uid, [mail_id])
<|endoftext|>


<commit_before>import numpy as np
import unittest

from chainer import testing
from chainer.dataset import TabularDataset

from .test_tabular_dataset import DummyDataset

@testing.parameterize(*testing.product_dict(
    testing.product({
        'mode': [tuple, dict],
        'integer': [int, np.int32],
        'seq': [list, np.array],
    }),
    [
        {'keys': None, 'expected_keys': ('a', 'b', 'c'),
         'expected_key_indices': None},
        {'keys': (1,), 'expected_keys': ('b',),
         'expected_key_indices': (1,)},
        {'keys': (3,), 'exception': IndexError},
        {'keys': ('c',), 'expected_keys': ('c',),
         'expected_key_indices': (2,)},
        {'keys': ('d',), 'exception': KeyError},
        {'keys': ('c', 0), 'expected_keys': ('c', 'a'),
         'expected_key_indices': (2, 0)},
    ],
    [
        {'indices': slice(None), 'expected_len': 10,
         'expected_indices': slice(0, 10, 1)},
        {'indices': [3, 1], 'expected_len': 2, 'expected_indices': [3, 1]},
        {'indices': [11, 1], 'exception': IndexError},
        {'indices': [i in {1, 3} for i in range(10)],
         'expected_len': 2, 'expected_indices': [1, 3]},
        {'indices': [True] * 11, 'exception': ValueError},
        {'indices': slice(3, None, -2), 'expected_len': 2,
         'expected_indices': slice(3, -1, -2)},
    ],
))
class TestSlice(unittest.TestCase):

    def setUp(self):
        if isinstance(self.indices, list):
            self.indices = self.seq(
                [index if isinstance(index, bool) else self.integer(index)
                 for index in self.indices])

    def test_slice(self):
        def callback(indices, key_indices):
            self.assertEqual(indices, self.expected_indices)
            self.assertEqual(key_indices, self.expected_key_indices)

        dataset = DummyDataset(mode=self.mode, callback=callback)

        if hasattr(self, 'exception'):
            with self.assertRaises(self.exception):
                if self.keys is None:
                    dataset.slice[self.indices]
                else:
                    dataset.slice[self.indices, self.keys]
            return

        if self.keys is None:
            view = dataset.slice[self.indices]
            data = dataset.data[:, self.indices]
        else:
            view = dataset.slice[self.indices, self.keys]
            key_indices = [{'a': 0, 'b': 1, 'c': 2}.get(key, key)
                           for key in self.keys]
            data = dataset.data[key_indices][:, self.indices]

        self.assertIsInstance(view, TabularDataset)
        self.assertEqual(len(view), self.expected_len)
        self.assertEqual(view.keys, self.expected_keys)
        self.assertEqual(view.mode, self.mode)

        if self.mode is tuple:
            expected_output = tuple(list(d) for d in data)
        elif self.mode is dict:
            expected_output = dict(zip(
                self.expected_keys, (list(d) for d in data)))

        self.assertEqual(view.fetch(), expected_output)

testing.run_module(__name__, __file__)
<commit_msg>call get_examples to check value<commit_after>import numpy as np
import unittest

from chainer import testing
from chainer.dataset import TabularDataset

from .test_tabular_dataset import DummyDataset

@testing.parameterize(*testing.product_dict(
    testing.product({
        'mode': [tuple, dict],
        'integer': [int, np.int32],
        'seq': [list, np.array],
    }),
    [
        {'keys': None, 'expected_keys': ('a', 'b', 'c'),
         'expected_key_indices': None},
        {'keys': (1,), 'expected_keys': ('b',),
         'expected_key_indices': (1,)},
        {'keys': (3,), 'exception': IndexError},
        {'keys': ('c',), 'expected_keys': ('c',),
         'expected_key_indices': (2,)},
        {'keys': ('d',), 'exception': KeyError},
        {'keys': ('c', 0), 'expected_keys': ('c', 'a'),
         'expected_key_indices': (2, 0)},
    ],
    [
        {'indices': slice(None), 'expected_len': 10,
         'expected_indices': slice(0, 10, 1)},
        {'indices': [3, 1], 'expected_len': 2, 'expected_indices': [3, 1]},
        {'indices': [11, 1], 'exception': IndexError},
        {'indices': [i in {1, 3} for i in range(10)],
         'expected_len': 2, 'expected_indices': [1, 3]},
        {'indices': [True] * 11, 'exception': ValueError},
        {'indices': slice(3, None, -2), 'expected_len': 2,
         'expected_indices': slice(3, -1, -2)},
    ],
))
class TestSlice(unittest.TestCase):

    def setUp(self):
        if isinstance(self.indices, list):
            self.indices = self.seq(
                [index if isinstance(index, bool) else self.integer(index)
                 for index in self.indices])

    def test_slice(self):
        def callback(indices, key_indices):
            self.assertEqual(indices, self.expected_indices)
            self.assertEqual(key_indices, self.expected_key_indices)

        dataset = DummyDataset(mode=self.mode, callback=callback)

        if hasattr(self, 'exception'):
            with self.assertRaises(self.exception):
                if self.keys is None:
                    dataset.slice[self.indices]
                else:
                    dataset.slice[self.indices, self.keys]
            return

        if self.keys is None:
            view = dataset.slice[self.indices]
            data = dataset.data[:, self.indices]
        else:
            view = dataset.slice[self.indices, self.keys]
            key_indices = [{'a': 0, 'b': 1, 'c': 2}.get(key, key)
                           for key in self.keys]
            data = dataset.data[key_indices][:, self.indices]

        self.assertIsInstance(view, TabularDataset)
        self.assertEqual(len(view), self.expected_len)
        self.assertEqual(view.keys, self.expected_keys)
        self.assertEqual(view.mode, self.mode)
        self.assertEqual(
            view.get_examples(None, None), tuple(list(d) for d in data))

testing.run_module(__name__, __file__)
<|endoftext|>