<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Modified from:

#--------------------------------------
#    ___  ___  _ ____
#   / _ \/ _ \(_) __/__  __ __
#  / , _/ ___/ /\ \/ _ \/ // /
# /_/|_/_/  /_/___/ .__/\_, /
#                /_/   /___/
#
#           Temperature Logger
#  Read data from a BMP180 sensor and
#  send to Thingspeak.com account.
#
# Author : Matt Hawkins
# Date   : 20/06/2015
#
# http://www.raspberrypi-spy.co.uk/
#--------------------------------------

# TODO:
# * make log machine readable
# * make a more verbose human readable log as well
# * save pictures and log to cloud

#import smbus
import time
import datetime
import os
import sys
import glob #to get lists of files etc.
import urllib            # URL functions
import urllib2           # URL functions
import random #for testing
import numpy as np #for simple computations such as median

# Modules for building email
from email.mime.text import MIMEText
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
import smtplib #for sending email

#import grovepi #for Grove sensors
import Adafruit_DHT #for DHT11 and DHT22 sensors
import RPi.GPIO as GPIO #for PIR pin read
from picamera import PiCamera #for camera

################# Default Constants #################
# These can be changed if required
OUTPATH = '/home/pi/data/mokinhenki'
if not os.path.exists(OUTPATH):
	os.makedirs(OUTPATH)
OUTPATH_FIG = os.path.join(OUTPATH, 'figs')
if not os.path.exists(OUTPATH_FIG):
	os.makedirs(OUTPATH_FIG)
LOGFILE = os.path.join(OUTPATH, 'log_mokinhenki.txt')

from_email = 'jussitapiokorpela@gmail.com' #seems to make no difference what this is...
to_email = ['pirkko.k.korpela@gmail.com']
app_token_file = '/home/pi/private/google_app_token'

TEMP_ALARM_TH = -20
HUM_ALARM_TH = 100

AUTOSHUTDOWN  = 1    # Set to 1 to shutdown on switch
THINGSPEAKKEY = 'WUKJPVAXWTTYQTFM' #channel: ""
THINGSPEAKURL = 'https://api.thingspeak.com/update'

PIN_PIR = 12 #PIR detector data GPIO pin

DHT_SENSOR = 22 #DHT 11 sensor submodel
PIN_DHT22 = 16 #DHT sensor data GPIO pin
TEMPHUM_INTERVAL_SEC = 60 #how often measurement values are read and sent to thingspeak
TEMPHUM_ALARM_INTERVAL_SEC = 60*60 #how often alarm emails are sent if alarm conditions persist
#####################################################

"""
def switchCallback(channel):

    global AUTOSHUTDOWN

    # Called if switch is pressed
    if AUTOSHUTDOWN==1:
        os.system('/sbin/shutdown -h now')
        sys.exit(0)
"""

def sendData(url, key, temp, hum):
    """
    Send event to internet site
    """

    global LOGFILE

    values = {'api_key' : key, 'field1' : temp, 'field2' : hum}

    postdata = urllib.urlencode(values)
    req = urllib2.Request(url, postdata)

    log = time.strftime("%d-%m-%Y,%H:%M:%S") + ","
    log = log + "{:.1f}C".format(temp) + ","
    log = log + "{:.2f}%".format(hum) + ","

    try:
        # Send data to Thingspeak
        response = urllib2.urlopen(req, None, 5)
        html_string = response.read()
        response.close()
        log = log + 'Update ' + html_string

    except urllib2.HTTPError as e:
        log = log + 'Server could not fulfill the request. Error code: ' + str(e.code)

        """
        Traceback (most recent call last):
		  File "thingspeak_test.py", line 146, in <module>
			main()
		  File "thingspeak_test.py", line 131, in main
			sendData(THINGSPEAKURL,THINGSPEAKKEY,temp,hum)
		  File "thingspeak_test.py", line 73, in sendData
			log = log + 'Server could not fulfill the request. Error code: ' + e.code
		TypeError: cannot concatenate 'str' and 'int' objects
        """

    except urllib2.URLError as e:
        log = log + 'Failed to reach server. Reason: ' + str(e.reason)
        """
        TODO: Find a way to avoid this:

        Traceback (most recent call last):
		File "thingspeak_envlogger.py", line 153, in <module>
		main()
		File "thingspeak_envlogger.py", line 137, in main
		sendData(THINGSPEAKURL,THINGSPEAKKEY,temp,hum)
		File "thingspeak_envlogger.py", line 76, in sendData
		log = log + 'Failed to reach server. Reason: ' + e.reason
		TypeError: cannot concatenate 'str' and 'gaierror' objects
		"""

    except:
        exctype, value = sys.exc_info()[:2]
        log = log + 'sendData error. Reason: ' + str(value)

    print log

    with open(LOGFILE, 'a') as file:
		file.write(log + '\n')

""" Create a multipart MIME message that can hold both text and images. """
def createMultipartEMail(from_email, to_email, subjectline, message):
    msg = MIMEMultipart()
    msg['Subject'] = subjectline
    msg['From'] = from_email
    msg['To'] = ",".join(to_email)
    #msg['Date'] = formatdate(localtime=True)
    msg.attach(MIMEText(message))
    return msg

""" Sends email using smtp.gmail.com. Authentication needs an App token since I use two-factor authentication.
    App token needs to be provided, since I do not dare to hard code it into this public code."""
def sendGMail(msg, google_app_token):
    status = [None] * 2
    log = time.strftime("%d-%m-%Y,%H:%M:%S")
    global LOGFILE

    try:
        server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
        status[0] = server.login('jussitapiokorpela@gmail.com', google_app_token)
        status[1] = server.sendmail(msg['from'], msg['to'], msg.as_string())
        server.quit()
        log = log + 'Send mail notification.'

    except:
        exctype, value = sys.exc_info()[:2]
        log = log + 'sendData error. Reason: ' + str(value)

    print log
    with open(LOGFILE, 'a') as file:
	    file.write(log + '\n')

    return status

""" Main program """
def main():

    global THINGSPEAKKEY
    global THINGSPEAKURL

    GPIO.setmode(GPIO.BCM)
    GPIO.setup(PIN_PIR, GPIO.IN)

    camera = PiCamera()
    camera.rotation = 180
    camera.resolution =  (1024, 576)
    day_pic_taken = False

    #camera.start_preview()
    #sleep(20)
    #camera.stop_preview()

    # initialize
    last_temphum_ts = time.time()
    hum = 0
    temp = -100
    last_temphum_email_ts = time.time()
    hum_hist = [HUM_ALARM_TH - 1] * 5
    temp_hist = [TEMP_ALARM_TH + 1] * 5

    # Load google app token from file
    fp = open(app_token_file, 'r')
    app_token = fp.readline()
    #app_token = app_token[0:-1] #remove \n
    fp.close()

    while True:

        try:
            # Take picture based on PIR
            if (GPIO.input(PIN_PIR)):
                #print("Taking a picture...")
                #camera.start_preview()

                time.sleep(0.2) #wait for the object to move fully into frame

                ts = time.time()
                tstr = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M-%S')
                fname = os.path.join(OUTPATH_FIG, "pirimage_%s.jpg" % (tstr))

                camera.capture(fname)

                # Create email
                msg = createMultipartEMail(from_email, to_email,
                        'Mokinhenki tiedottaa: liiketunnistin lauennut',
                        'Liiketunnistimeni laukesi. Ohessa kuva.')

                # Attach image
                fp = open(fname, 'rb')
                img = MIMEImage(fp.read())
                fp.close()
                msg.attach(img)

                # Send using gmail
                status = sendGMail(msg, app_token)

                #time.sleep(3) #sleep until PIR is ready again
                #camera.stop_preview()

            # Take picture based on time of day
            curts = time.localtime()
            if (curts[3]==12) & (curts[4]==0) & (day_pic_taken != True):
                ts = time.time()
                tstr = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M-%S')
                fname = os.path.join(OUTPATH_FIG, "dayimage_%s.jpg" % (tstr))
                camera.capture(fname)
                day_pic_taken = True

            if (curts[3]==12) & (curts[4]==1):
                day_pic_taken = False #reset flag

            # Log temperature and humidity
            if (time.time() - last_temphum_ts) > TEMPHUM_INTERVAL_SEC:

                #[temp, hum] = grovepi.dht(4, 1) #D4 input, the white version

                # Try to grab a sensor reading.  Use the read_retry method which will retry up
                # to 15 times to get a sensor reading (waiting 2 seconds between each retry).
                hum_old = hum
                temp_old = temp
                hum, temp = Adafruit_DHT.read_retry(DHT_SENSOR, PIN_DHT22)

                # Note that sometimes you won't get a reading and
                # the results will be null (because Linux can't
                # guarantee the timing of calls to read the sensor).
                # If this happens try again!
                if hum is None or temp is None:
                    hum = hum_old
                    temp = temp_old

                if all(x is not None for x in [temp, hum]) & (-40 < temp) & (temp < 50) & (0 < hum) & (hum < 100):
                                print("temp = %.02f C humidity =%.02f%%"%(temp, hum))
                                sendData(THINGSPEAKURL, THINGSPEAKKEY, temp, hum)
                                sys.stdout.flush()

                                # update history records: pop last element and add new to position 0
                                temp_hist.pop()
                                temp_hist.insert(0, temp)
                                hum_hist.pop()
                                hum_hist.insert(0, hum)

                                # send email warning
                                if ((np.median(temp_hist) < TEMP_ALARM_TH) | (np.median(hum_hist) > HUM_ALARM_TH)) & ((time.time() - last_temphum_email_ts) > TEMPHUM_ALARM_INTERVAL_SEC):
                                    msg = createMultipartEMail(from_email, to_email,
                                    'Mokinhenki tiedottaa: liian kylmaa tai kosteaa',
                                    'Majassani on liian kylmaa tai kosteaa. Lampotila on %2.1f astetta ja suht. kosteus %2.1f prosenttia. Tee jotain.' % (temp, hum) )
                                    status = sendGMail(msg, app_token)
                                    last_temphum_email_ts = time.time()

                                last_temphum_ts = time.time()

            time.sleep(0.5) #for the while loop

        except:
            exctype, value = sys.exc_info()[:2]
            log = 'Loop error: ' + str(value)
            print(log)
            with open(LOGFILE, 'a') as file:
                file.write(log + '\n')

if __name__=="__main__":
    main()

<commit_msg>Added dropbox upload to mokihenki.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Modified from:

#--------------------------------------
#    ___  ___  _ ____
#   / _ \/ _ \(_) __/__  __ __
#  / , _/ ___/ /\ \/ _ \/ // /
# /_/|_/_/  /_/___/ .__/\_, /
#                /_/   /___/
#
#           Temperature Logger
#  Read data from a BMP180 sensor and
#  send to Thingspeak.com account.
#
# Author : Matt Hawkins
# Date   : 20/06/2015
#
# http://www.raspberrypi-spy.co.uk/
#--------------------------------------

# TODO:
# * make log machine readable
# * make a more verbose human readable log as well
# * save pictures and log to cloud
# * add newline to google app token file

#import smbus
import time
import datetime
import os
import sys
import glob #to get lists of files etc.
import urllib            # URL functions
import urllib2           # URL functions
import random #for testing
import numpy as np #for simple computations such as median

# Modules for building email
from email.mime.text import MIMEText
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
import smtplib #for sending email

#import grovepi #for Grove sensors
import Adafruit_DHT #for DHT11 and DHT22 sensors
import RPi.GPIO as GPIO #for PIR pin read
from picamera import PiCamera #for camera

import dropbox #sudo pip2 install dropbox

################# Default Constants #################
# These can be changed if required
OUTPATH = '/home/pi/data/mokinhenki'
if not os.path.exists(OUTPATH):
	os.makedirs(OUTPATH)
OUTPATH_FIG = os.path.join(OUTPATH, 'figs')
if not os.path.exists(OUTPATH_FIG):
	os.makedirs(OUTPATH_FIG)
LOGFILE = os.path.join(OUTPATH, 'log_mokinhenki.txt')

from_email = 'jussitapiokorpela@gmail.com' #seems to make no difference what this is...
to_email = ['pirkko.k.korpela@gmail.com']
app_token_file = '/home/pi/private/google_app_token'
dbx_oauth2_token_file = '/home/pi/private/dbx_accesstoken_mokinhenki'

TEMP_ALARM_TH = -20
HUM_ALARM_TH = 100

AUTOSHUTDOWN  = 1    # Set to 1 to shutdown on switch
THINGSPEAKKEY = 'WUKJPVAXWTTYQTFM' #channel: ""
THINGSPEAKURL = 'https://api.thingspeak.com/update'

PIN_PIR = 12 #PIR detector data GPIO pin

DHT_SENSOR = 22 #DHT 11 sensor submodel
PIN_DHT22 = 16 #DHT sensor data GPIO pin
TEMPHUM_INTERVAL_SEC = 60 #how often measurement values are read and sent to thingspeak
TEMPHUM_ALARM_INTERVAL_SEC = 60*60 #how often alarm emails are sent if alarm conditions persist
#####################################################

"""
def switchCallback(channel):

    global AUTOSHUTDOWN

    # Called if switch is pressed
    if AUTOSHUTDOWN==1:
        os.system('/sbin/shutdown -h now')
        sys.exit(0)
"""

def sendData(url, key, temp, hum):
    """
    Send event to internet site
    """

    global LOGFILE

    values = {'api_key' : key, 'field1' : temp, 'field2' : hum}

    postdata = urllib.urlencode(values)
    req = urllib2.Request(url, postdata)

    log = time.strftime("%d-%m-%Y,%H:%M:%S") + ","
    log = log + "{:.1f}C".format(temp) + ","
    log = log + "{:.2f}%".format(hum) + ","

    try:
        # Send data to Thingspeak
        response = urllib2.urlopen(req, None, 5)
        html_string = response.read()
        response.close()
        log = log + 'Update ' + html_string

    except urllib2.HTTPError as e:
        log = log + 'Server could not fulfill the request. Error code: ' + str(e.code)

        """
        Traceback (most recent call last):
		  File "thingspeak_test.py", line 146, in <module>
			main()
		  File "thingspeak_test.py", line 131, in main
			sendData(THINGSPEAKURL,THINGSPEAKKEY,temp,hum)
		  File "thingspeak_test.py", line 73, in sendData
			log = log + 'Server could not fulfill the request. Error code: ' + e.code
		TypeError: cannot concatenate 'str' and 'int' objects
        """

    except urllib2.URLError as e:
        log = log + 'Failed to reach server. Reason: ' + str(e.reason)
        """
        TODO: Find a way to avoid this:

        Traceback (most recent call last):
		File "thingspeak_envlogger.py", line 153, in <module>
		main()
		File "thingspeak_envlogger.py", line 137, in main
		sendData(THINGSPEAKURL,THINGSPEAKKEY,temp,hum)
		File "thingspeak_envlogger.py", line 76, in sendData
		log = log + 'Failed to reach server. Reason: ' + e.reason
		TypeError: cannot concatenate 'str' and 'gaierror' objects
		"""

    except:
        exctype, value = sys.exc_info()[:2]
        log = log + 'sendData error. Reason: ' + str(value)

    print log

    with open(LOGFILE, 'a') as file:
		file.write(log + '\n')

""" Create a multipart MIME message that can hold both text and images. """
def createMultipartEMail(from_email, to_email, subjectline, message):
    msg = MIMEMultipart()
    msg['Subject'] = subjectline
    msg['From'] = from_email
    msg['To'] = ",".join(to_email)
    #msg['Date'] = formatdate(localtime=True)
    msg.attach(MIMEText(message))
    return msg

""" Sends email using smtp.gmail.com. Authentication needs an App token since I use two-factor authentication.
    App token needs to be provided, since I do not dare to hard code it into this public code."""
def sendGMail(msg, google_app_token):
    status = [None] * 2
    log = time.strftime("%d-%m-%Y,%H:%M:%S")
    global LOGFILE

    try:
        server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
        status[0] = server.login('jussitapiokorpela@gmail.com', google_app_token)
        status[1] = server.sendmail(msg['from'], msg['to'], msg.as_string())
        server.quit()
        log = log + 'Send mail notification.'

    except:
        exctype, value = sys.exc_info()[:2]
        log = log + 'sendData error. Reason: ' + str(value)

    print log
    with open(LOGFILE, 'a') as file:
	    file.write(log + '\n')

    return status

# Read token from file (read first line, assuming no line break)
def read_token(app_token_file):
    fp = open(app_token_file, 'r')
    app_token = fp.readline()
    app_token = app_token[0:-1] #remove \n
    fp.close()
    return app_token

# A class to interact with Dropbox
class DbxTransferData:

    # Contstructor
    def __init__(self, access_token):
        self.access_token = access_token

    # Upload file using Dropbox API v2
    def upload_file(self, file_from, file_to):
        """upload a file to Dropbox using API v2
        """
        dbx = dropbox.Dropbox(self.access_token)

        with open(file_from, 'rb') as f:
            dbx.files_upload(f.read(), file_to)

""" Main program """
def main():

    global THINGSPEAKKEY
    global THINGSPEAKURL

    GPIO.setmode(GPIO.BCM)
    GPIO.setup(PIN_PIR, GPIO.IN)

    camera = PiCamera()
    camera.rotation = 180
    camera.resolution =  (1024, 576)
    day_pic_taken = False

    #camera.start_preview()
    #sleep(20)
    #camera.stop_preview()

    # initialize
    last_temphum_ts = time.time()
    hum = 0
    temp = -100
    last_temphum_email_ts = time.time()
    hum_hist = [HUM_ALARM_TH - 1] * 5
    temp_hist = [TEMP_ALARM_TH + 1] * 5

    # Read needed oauth tokens from files
    # Load google app token from file
    # fp = open(app_token_file, 'r')
    # app_token = fp.readline()
    # #app_token = app_token[0:-1] #remove \n
    # fp.close()

    app_token = read_token(app_token_file)
    dbx_oauth2_token = read_token(dbx_oauth2_token_file)
    dbx = DbxTransferData(dbx_oauth2_token)

    while True:

        try:
            # Take picture based on PIR
            if (GPIO.input(PIN_PIR)):
                #print("Taking a picture...")
                #camera.start_preview()

                time.sleep(0.2) #wait for the object to move fully into frame

                ts = time.time()
                tstr = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M-%S')
                fname = os.path.join(OUTPATH_FIG, "pirimage_%s.jpg" % (tstr))

                camera.capture(fname)

                # Notify using GMail

                # Create email
                msg = createMultipartEMail(from_email, to_email,
                        'Mokinhenki tiedottaa: liiketunnistin lauennut',
                        'Liiketunnistimeni laukesi. Ohessa kuva.')

                # Attach image
                fp = open(fname, 'rb')
                img = MIMEImage(fp.read())
                fp.close()
                msg.attach(img)

                # Send using gmail
                status = sendGMail(msg, app_token)

                #time.sleep(3) #sleep until PIR is ready again
                #camera.stop_preview()

                # Save to dropbox
                file_to = os.path.join('/figs/pir/', os.path.basename(fname))
                dbx.upload_file(fname, file_to)

            # Take picture based on time of day
            curts = time.localtime()
            if (curts[3]==12) & (curts[4]==0) & (day_pic_taken != True):
                ts = time.time()
                tstr = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M-%S')
                fname = os.path.join(OUTPATH_FIG, "dayimage_%s.jpg" % (tstr))
                camera.capture(fname)
                day_pic_taken = True

                # path to upload the file to, including the file name
                # saves to dropbox under Apps/mokinhenki/
                file_to = os.path.join('/figs/daily/', os.path.basename(fname))
                dbx.upload_file(fname, file_to)

            if (curts[3]==12) & (curts[4]==1):
                day_pic_taken = False #reset flag

            # Log temperature and humidity
            if (time.time() - last_temphum_ts) > TEMPHUM_INTERVAL_SEC:

                #[temp, hum] = grovepi.dht(4, 1) #D4 input, the white version

                # Try to grab a sensor reading.  Use the read_retry method which will retry up
                # to 15 times to get a sensor reading (waiting 2 seconds between each retry).
                hum_old = hum
                temp_old = temp
                hum, temp = Adafruit_DHT.read_retry(DHT_SENSOR, PIN_DHT22)

                # Note that sometimes you won't get a reading and
                # the results will be null (because Linux can't
                # guarantee the timing of calls to read the sensor).
                # If this happens try again!
                if hum is None or temp is None:
                    hum = hum_old
                    temp = temp_old

                if all(x is not None for x in [temp, hum]) & (-40 < temp) & (temp < 50) & (0 < hum) & (hum < 100):
                                print("temp = %.02f C humidity =%.02f%%"%(temp, hum))
                                sendData(THINGSPEAKURL, THINGSPEAKKEY, temp, hum)
                                sys.stdout.flush()

                                # update history records: pop last element and add new to position 0
                                temp_hist.pop()
                                temp_hist.insert(0, temp)
                                hum_hist.pop()
                                hum_hist.insert(0, hum)

                                # send email warning
                                if ((np.median(temp_hist) < TEMP_ALARM_TH) | (np.median(hum_hist) > HUM_ALARM_TH)) & ((time.time() - last_temphum_email_ts) > TEMPHUM_ALARM_INTERVAL_SEC):
                                    msg = createMultipartEMail(from_email, to_email,
                                    'Mokinhenki tiedottaa: liian kylmaa tai kosteaa',
                                    'Majassani on liian kylmaa tai kosteaa. Lampotila on %2.1f astetta ja suht. kosteus %2.1f prosenttia. Tee jotain.' % (temp, hum) )
                                    status = sendGMail(msg, app_token)
                                    last_temphum_email_ts = time.time()

                                last_temphum_ts = time.time()

            time.sleep(0.5) #for the while loop

        except:
            exctype, value = sys.exc_info()[:2]
            log = 'Loop error: ' + str(value)
            print(log)
            with open(LOGFILE, 'a') as file:
                file.write(log + '\n')

if __name__=="__main__":
    main()

<|endoftext|>


<commit_before>"""
Property descriptors.
"""

from _compat import *

from functools import partial as _partial
import operator as _operator

class _func_deco(object):

    def __init__(self, func):
        super(_func_deco, self).__init__()
        self.func = func

class class_instance_method(_func_deco):
    """Non-data descriptor.

    Methods decorated by this class must accept two special arguments (go
    first): cls and self: cls is the same as for classmethod, self is None
    in case of invoking on the class and the instance otherwise.

    Usage example:

    >>> class C(object):
    ...     @class_instance_method
    ...     def meth(cls, self, arg):
    ...         print("Invoking {cls.__name__}.meth on {self} with {arg}"
    ...               .format(**locals()))
    ...         return arg
    ...     def __repr__(self):
    ...         return "<{cls.__name__} object>".format(cls=type(self))
    ...
    >>> C.meth(17)
    Invoking C.meth on None with 17
    17
    >>> x = C()
    >>> x.meth(42)
    Invoking C.meth on <C object> with 42
    42
    """

    def __get__(self, obj, objtype=None):
        if objtype is None:
            objtype = type(obj)
        return _partial(self.func, objtype, obj)

class default_property(_func_deco):
    """Non-data descriptor.

    Delegates to func everytime a property is accessed unless someone
    explicitly overrides it by setting a new value.

    Usage example:

    >>> class C(object):
    ...     @default_property
    ...     def default(self):
    ...         print("Accessing {cls.__name__}.default"
    ...               .format(cls=type(self)))
    ...         return 17
    ...
    >>> x = C()
    >>> x.default
    Accessing C.default
    17
    >>> x.default
    Accessing C.default
    17
    >>> x.default = 42
    >>> x.default
    42
    """

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        return self.func(obj)

class class_default_property(_func_deco):
    """Non-data descriptor.

    Calls func on an instance type everytime a property is accessed.

    Usage example:

    >>> class C(object):
    ...     @class_default_property
    ...     def prop(cls):
    ...         print("Accessing {cls.__name__}.prop"
    ...               .format(**locals()))
    ...         return 17
    ...
    >>> C.prop
    Accessing C.prop
    17
    >>> x = C()
    >>> x.prop
    Accessing C.prop
    17
    >>> x.prop = 42
    >>> x.prop
    42
    """

    def __get__(self, obj, objtype=None):
        if objtype is None:
            objtype = type(obj)
        return self.func(objtype)

class cached_property(default_property):
    """Non-data descriptor.

    Delegates to func only the first time a property is accessed.

    Usage example:

    >>> class C(object):
    ...     @cached_property
    ...     def cached(self):
    ...         print("Accessing {cls.__name__}.cached".format(cls=type(self)))
    ...         return 17
    ...
    >>> x = C()
    >>> x.cached
    Accessing C.cached
    17
    >>> x.cached
    17
    >>> x.cached = 42
    >>> x.cached
    42
    """

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        ret = super(cached_property, self).__get__(obj, objtype)
        setattr(obj, self.func.__name__, ret)
        return ret

class cumulative_sequence_property(object):

    def __init__(self, factory, attr_name, add_meth_name=None):
        super(cumulative_property, self).__init__()
        self.factory       = factory
        self.attr_name     = attr_name
        self.add_meth_name = add_meth_name

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        try:
            return getattr(obj, self.attr_name)
        except AttributeError:
            default = self.factory()
            setattr(obj, self.attr_name, default)
            return default

    def __set__(self, obj, items):
        try:
            old_items = getattr(obj, self.attr_name)
        except AttributeError:
            items = new_items = self.factory(items)
        else:
            new_items = self.factory(item for item in items
                                     if item not in old_items)
            items = old_items+new_items

        if self.add_meth_name is not None:
            func = getattr(obj, self.add_meth_name)

            for item in new_items:
                func(item)

        setattr(obj, self.attr_name, items)

class cumulative_property(object):

    def __init__(self, attr, factory):
        super(cumulative_property, self).__init__()
        self.attr    = attr
        self.factory = factory

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        try:
            return getattr(obj, self.attr)
        except AttributeError:
            default = self.factory()
            setattr(obj, self.attr, default)
            return default

    def split_old_new(self, obj, items):
        old_items = getattr(obj, self.attr, None)

        if old_items is not None:
            new_items = self.filter_new(items, old_items)
        else:
            new_items = items

        return old_items, self.factory(new_items)

    def filter_new(self, items, old_items):
        return items

class cumulative_mapping_property(cumulative_property):

    def __init__(self, attr, factory=dict):
        super(cumulative_mapping_property, self).__init__(attr, factory)

    def __set__(self, obj, items):
        old_items, items = self.split_old_new(obj, items)

        if old_items is not None:
            old_items.update(items)
            items = old_items

        setattr(obj, self.attr, items)

    def filter_new(self, items, old_items):
        if not isinstance(items, dict):
            items = dict(items)
        return (item for item in iteritems(items)
                if item[0] not in old_items)

class cumulative_sequence_property(cumulative_property):

    def __init__(self, attr, factory=list):
        super(cumulative_sequence_property, self).__init__(attr, factory)

    def __set__(self, obj, items):
        old_items, items = self.split_old_new(obj, items)

        if old_items is not None:
            items = _operator.__iadd__(old_items, items)

        setattr(obj, self.attr, items)

    def filter_new(self, items, old_items):
        return (item for item in items
                if item not in old_items)

class cumulative_tuple_property(cumulative_sequence_property):

    def __init__(self, attr, add_meth=None):
        super(cumulative_tuple_property, self).__init__(attr, factory=tuple)
        self.add_meth = add_meth

    def __set__(self, obj, items):
        old_items, items = self.split_old_new(obj, items)

        if self.add_meth is not None:
            func = getattr(obj, self.add_meth)

            for item in items:
                func(item)

        if old_items is not None:
            items = _operator.__iadd__(old_items, items)

        setattr(obj, self.attr, items)

class intercepting_property(property):
    """Data descriptor.

    Allows one to set hooks on get/set/del operations and to modify an actual
    value which is stored in a backing __dict__.

    Usage example:

    >>> class C(object):
    ...     @intercepting_property
    ...     def icp(self, value):
    ...         print("Get {cls.__name__}.icp: {value}"
    ...               .format(cls=type(self), **locals()))
    ...         return -value
    ...     @icp.setter
    ...     def icp(self, value):
    ...         print("Set {cls.__name__}.icp: {value}"
    ...               .format(cls=type(self), **locals()))
    ...         return -value
    ...     @icp.deleter
    ...     def icp(self):
    ...         print("Del {cls.__name__}.icp".format(cls=type(self)))
    ...
    >>> x = C()
    >>> x.icp
    Traceback (most recent call last):
        ...
    AttributeError: 'C' object has no attribute 'icp'
    >>> x.icp = 7
    Set C.icp: 7
    >>> x.__dict__['icp']
    -7
    >>> x.icp
    Get C.icp: -7
    7
    >>> del x.icp
    Del C.icp
    >>> 'icp' in x.__dict__
    False
    """

    def __init__(self, fget=None, fset=None, fdel=None, name=None, doc=None):
        super(intercepting_property, self).__init__(fget, fset, fdel, doc)

        if name is None:
            func = fget or fset or fdel
            if func is None:
                raise TypeError('no attribute name given')
            name = func.__name__

        self.name = name

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        try:
            ret = obj.__dict__[self.name]
        except KeyError:
            raise AttributeError("'{objtype.__name__}' object has no "
                                 "attribute '{self.name}'".format(**locals()))
        fget = self.fget
        if fget is not None:
            ret = fget(obj, ret)
        return ret

    def __set__(self, obj, value):
        fset = self.fset
        if fset is not None:
            value = fset(obj, value)
        obj.__dict__[self.name] = value

    def __delete__(self, obj):
        fdel = self.fdel
        if fdel is not None:
            fdel(obj)
        try:
            del obj.__dict__[self.name]
        except KeyError:
            objtype = type(obj)
            raise AttributeError("'{objtype.__name__}' object has no "
                                 "attribute '{self.name}'".format(**locals()))

if __name__ == '__main__':
    import doctest
    doctest.testmod()

<commit_msg>util: prop: prevent accessing default_property as a class attribute<commit_after>"""
Property descriptors.
"""

from _compat import *

from functools import partial as _partial
import operator as _operator

class _func_deco(object):

    def __init__(self, func):
        super(_func_deco, self).__init__()
        self.func = func

class class_instance_method(_func_deco):
    """Non-data descriptor.

    Methods decorated by this class must accept two special arguments (go
    first): cls and self: cls is the same as for classmethod, self is None
    in case of invoking on the class and the instance otherwise.

    Usage example:

    >>> class C(object):
    ...     @class_instance_method
    ...     def meth(cls, self, arg):
    ...         print("Invoking {cls.__name__}.meth on {self} with {arg}"
    ...               .format(**locals()))
    ...         return arg
    ...     def __repr__(self):
    ...         return "<{cls.__name__} object>".format(cls=type(self))
    ...
    >>> C.meth(17)
    Invoking C.meth on None with 17
    17
    >>> x = C()
    >>> x.meth(42)
    Invoking C.meth on <C object> with 42
    42
    """

    def __get__(self, obj, objtype=None):
        if objtype is None:
            objtype = type(obj)
        return _partial(self.func, objtype, obj)

class default_property(_func_deco):
    """Non-data descriptor.

    Delegates to func everytime a property is accessed unless someone
    explicitly overrides it by setting a new value.

    Usage example:

    >>> class C(object):
    ...     @default_property
    ...     def default(self):
    ...         print("Accessing {cls.__name__}.default"
    ...               .format(cls=type(self)))
    ...         return 17
    ...
    >>> C.default
    Traceback (most recent call last):
        ...
    AttributeError: 'default_property' descriptor 'default' of 'C' objects is not accessible as a class attribute
    >>> x = C()
    >>> x.default
    Accessing C.default
    17
    >>> x.default
    Accessing C.default
    17
    >>> x.default = 42
    >>> x.default
    42
    """

    def __get__(self, obj, objtype=None):
        if obj is None:
            raise AttributeError("'{self.__class__.__name__}' descriptor "
                                 "'{self.func.__name__}' "
                                 "of '{objtype.__name__}' objects is not "
                                 "accessible as a class attribute"
                                 .format(**locals()))
        return self.func(obj)

class class_default_property(_func_deco):
    """Non-data descriptor.

    Calls func on an instance type everytime a property is accessed.

    Usage example:

    >>> class C(object):
    ...     @class_default_property
    ...     def prop(cls):
    ...         print("Accessing {cls.__name__}.prop"
    ...               .format(**locals()))
    ...         return 17
    ...
    >>> C.prop
    Accessing C.prop
    17
    >>> x = C()
    >>> x.prop
    Accessing C.prop
    17
    >>> x.prop = 42
    >>> x.prop
    42
    """

    def __get__(self, obj, objtype=None):
        if objtype is None:
            objtype = type(obj)
        return self.func(objtype)

class cached_property(default_property):
    """Non-data descriptor.

    Delegates to func only the first time a property is accessed.

    Usage example:

    >>> class C(object):
    ...     @cached_property
    ...     def cached(self):
    ...         print("Accessing {cls.__name__}.cached".format(cls=type(self)))
    ...         return 17
    ...
    >>> x = C()
    >>> x.cached
    Accessing C.cached
    17
    >>> x.cached
    17
    >>> x.cached = 42
    >>> x.cached
    42
    """

    def __get__(self, obj, objtype=None):
        ret = super(cached_property, self).__get__(obj, objtype)
        setattr(obj, self.func.__name__, ret)
        return ret

class cumulative_sequence_property(object):

    def __init__(self, factory, attr_name, add_meth_name=None):
        super(cumulative_property, self).__init__()
        self.factory       = factory
        self.attr_name     = attr_name
        self.add_meth_name = add_meth_name

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        try:
            return getattr(obj, self.attr_name)
        except AttributeError:
            default = self.factory()
            setattr(obj, self.attr_name, default)
            return default

    def __set__(self, obj, items):
        try:
            old_items = getattr(obj, self.attr_name)
        except AttributeError:
            items = new_items = self.factory(items)
        else:
            new_items = self.factory(item for item in items
                                     if item not in old_items)
            items = old_items+new_items

        if self.add_meth_name is not None:
            func = getattr(obj, self.add_meth_name)

            for item in new_items:
                func(item)

        setattr(obj, self.attr_name, items)

class cumulative_property(object):

    def __init__(self, attr, factory):
        super(cumulative_property, self).__init__()
        self.attr    = attr
        self.factory = factory

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        try:
            return getattr(obj, self.attr)
        except AttributeError:
            default = self.factory()
            setattr(obj, self.attr, default)
            return default

    def split_old_new(self, obj, items):
        old_items = getattr(obj, self.attr, None)

        if old_items is not None:
            new_items = self.filter_new(items, old_items)
        else:
            new_items = items

        return old_items, self.factory(new_items)

    def filter_new(self, items, old_items):
        return items

class cumulative_mapping_property(cumulative_property):

    def __init__(self, attr, factory=dict):
        super(cumulative_mapping_property, self).__init__(attr, factory)

    def __set__(self, obj, items):
        old_items, items = self.split_old_new(obj, items)

        if old_items is not None:
            old_items.update(items)
            items = old_items

        setattr(obj, self.attr, items)

    def filter_new(self, items, old_items):
        if not isinstance(items, dict):
            items = dict(items)
        return (item for item in iteritems(items)
                if item[0] not in old_items)

class cumulative_sequence_property(cumulative_property):

    def __init__(self, attr, factory=list):
        super(cumulative_sequence_property, self).__init__(attr, factory)

    def __set__(self, obj, items):
        old_items, items = self.split_old_new(obj, items)

        if old_items is not None:
            items = _operator.__iadd__(old_items, items)

        setattr(obj, self.attr, items)

    def filter_new(self, items, old_items):
        return (item for item in items
                if item not in old_items)

class cumulative_tuple_property(cumulative_sequence_property):

    def __init__(self, attr, add_meth=None):
        super(cumulative_tuple_property, self).__init__(attr, factory=tuple)
        self.add_meth = add_meth

    def __set__(self, obj, items):
        old_items, items = self.split_old_new(obj, items)

        if self.add_meth is not None:
            func = getattr(obj, self.add_meth)

            for item in items:
                func(item)

        if old_items is not None:
            items = _operator.__iadd__(old_items, items)

        setattr(obj, self.attr, items)

class intercepting_property(property):
    """Data descriptor.

    Allows one to set hooks on get/set/del operations and to modify an actual
    value which is stored in a backing __dict__.

    Usage example:

    >>> class C(object):
    ...     @intercepting_property
    ...     def icp(self, value):
    ...         print("Get {cls.__name__}.icp: {value}"
    ...               .format(cls=type(self), **locals()))
    ...         return -value
    ...     @icp.setter
    ...     def icp(self, value):
    ...         print("Set {cls.__name__}.icp: {value}"
    ...               .format(cls=type(self), **locals()))
    ...         return -value
    ...     @icp.deleter
    ...     def icp(self):
    ...         print("Del {cls.__name__}.icp".format(cls=type(self)))
    ...
    >>> x = C()
    >>> x.icp
    Traceback (most recent call last):
        ...
    AttributeError: 'C' object has no attribute 'icp'
    >>> x.icp = 7
    Set C.icp: 7
    >>> x.__dict__['icp']
    -7
    >>> x.icp
    Get C.icp: -7
    7
    >>> del x.icp
    Del C.icp
    >>> 'icp' in x.__dict__
    False
    """

    def __init__(self, fget=None, fset=None, fdel=None, name=None, doc=None):
        super(intercepting_property, self).__init__(fget, fset, fdel, doc)

        if name is None:
            func = fget or fset or fdel
            if func is None:
                raise TypeError('no attribute name given')
            name = func.__name__

        self.name = name

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        try:
            ret = obj.__dict__[self.name]
        except KeyError:
            raise AttributeError("'{objtype.__name__}' object has no "
                                 "attribute '{self.name}'".format(**locals()))
        fget = self.fget
        if fget is not None:
            ret = fget(obj, ret)
        return ret

    def __set__(self, obj, value):
        fset = self.fset
        if fset is not None:
            value = fset(obj, value)
        obj.__dict__[self.name] = value

    def __delete__(self, obj):
        fdel = self.fdel
        if fdel is not None:
            fdel(obj)
        try:
            del obj.__dict__[self.name]
        except KeyError:
            objtype = type(obj)
            raise AttributeError("'{objtype.__name__}' object has no "
                                 "attribute '{self.name}'".format(**locals()))

if __name__ == '__main__':
    import doctest
    doctest.testmod()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import tweepy
import os
from time import sleep
from random import randrange
import re

def file_to_dict(file_name):
	"""opens a file and put tab separated values in a dict"""
	f = open(file_name, 'r')
	d_keys = {}
	for i in f.readlines():
		i = i[:len(i) - 1]		#Removing the \n char
		j = i.split('\t')
		d_keys[j[0]] = j[1]
	return d_keys

def twitter_authentification(keys_file):
	"""return a tweepy.api object connected to your account"""
	d_keys = file_to_dict(keys_file)
	auth = tweepy.OAuthHandler(d_keys["consumer_key"],
		d_keys["consumer_secret"])
	auth.set_access_token(d_keys["access_token"],
		d_keys["access_secret"])
	api = tweepy.API(auth)
	return api

def daemonize():
    try:
        pid = os.fork()
    except OSError, e:
        raise Exception, "%s [%d]" % (e.argsstrerror, e.argserrno)
    if (pid == 0):      # Ayanami Rei
        os.setsid()
        try:
            pid = os.fork()
        except OSError, e:
            raise Exception, "%s [%d]" % (e.argsstrerror, e.argserrno)
        if (pid == 0):      # Langley Soryu Asuka
            os.chdir("/")
            os.umask(0)
        else:
            os._exit(0)
    else:
        os._exit(0)
    os.open("/dev/null",os.O_RDWR)
    os.dup2(0,1)
    os.dup2(0,2)
    print pid
    return(0)

def is_direct_mention(text):
	pattern_direct_mention = re.compile("^@.+")
	return re.match(pattern_direct_mention, text)

def parsing_dm_list(dm_list):
	dm_list_copy = []
	for dm in dm_list:
		if not is_direct_mention(dm.text):
			dm_list_copy.append(dm)
	return dm_list_copy

if __name__ == "__main__":
	#retCode = daemonize()
	cur_dir = os.path.dirname(os.path.realpath(__file__))

	api = twitter_authentification(cur_dir + "/keys.conf")
	dms = api.direct_messages()
	dms = parsing_dm_list(dms)

	while 1:
		try:
			print "Nb de DM : " + str(len(dms))
			print "Follow back"
			followers = api.followers()
			for follower in followers:
				if not follower.following:
					try:
						follower.follow()
					except tweepy.error.TweepError:
						print "Demande déjà faite pour " + follower.screen_name

			print "sleeping before tweeting"
			sleep(1800)

			if len(dms) != 0:
				print "Picking dm"
				picked_dm = dms[randrange(len(dms))]
				dms.remove(picked_dm)
				#api.destroy_direct_message(picked_dm.id)
				print picked_dm.text
				api.update_status(picked_dm.text)

				last_dm_id = dms[len(dms) - 1]
				first_dm_id = last_dm_id

			print "sleeping before following"
			sleep(1800)

			try:
				dms = api.direct_messages(first_dm_id)
				dms = parsing_dm_list(dms)
			except tweepy.error.TweepError:pass
		except IndexError:pass

<commit_msg>encoding<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import tweepy
import os
from time import sleep
from random import randrange
import re

def file_to_dict(file_name):
	"""opens a file and put tab separated values in a dict"""
	f = open(file_name, 'r')
	d_keys = {}
	for i in f.readlines():
		i = i[:len(i) - 1]		#Removing the \n char
		j = i.split('\t')
		d_keys[j[0]] = j[1]
	return d_keys

def twitter_authentification(keys_file):
	"""return a tweepy.api object connected to your account"""
	d_keys = file_to_dict(keys_file)
	auth = tweepy.OAuthHandler(d_keys["consumer_key"],
		d_keys["consumer_secret"])
	auth.set_access_token(d_keys["access_token"],
		d_keys["access_secret"])
	api = tweepy.API(auth)
	return api

def daemonize():
    try:
        pid = os.fork()
    except OSError, e:
        raise Exception, "%s [%d]" % (e.argsstrerror, e.argserrno)
    if (pid == 0):      # Ayanami Rei
        os.setsid()
        try:
            pid = os.fork()
        except OSError, e:
            raise Exception, "%s [%d]" % (e.argsstrerror, e.argserrno)
        if (pid == 0):      # Langley Soryu Asuka
            os.chdir("/")
            os.umask(0)
        else:
            os._exit(0)
    else:
        os._exit(0)
    os.open("/dev/null",os.O_RDWR)
    os.dup2(0,1)
    os.dup2(0,2)
    print pid
    return(0)

def is_direct_mention(text):
	pattern_direct_mention = re.compile("^@.+")
	return re.match(pattern_direct_mention, text)

def parsing_dm_list(dm_list):
	dm_list_copy = []
	for dm in dm_list:
		if not is_direct_mention(dm.text):
			dm_list_copy.append(dm)
	return dm_list_copy

if __name__ == "__main__":
	#retCode = daemonize()
	cur_dir = os.path.dirname(os.path.realpath(__file__))

	api = twitter_authentification(cur_dir + "/keys.conf")
	dms = api.direct_messages()
	dms = parsing_dm_list(dms)

	while 1:
		try:
			print "Nb de DM : " + str(len(dms))
			print "Follow back"
			followers = api.followers()
			for follower in followers:
				if not follower.following:
					try:
						follower.follow()
					except tweepy.error.TweepError:
						print "Demande déjà faite pour " + follower.name

			print "sleeping before tweeting"
			sleep(1800)

			if len(dms) != 0:
				print "Picking dm"
				picked_dm = dms[randrange(len(dms))]
				dms.remove(picked_dm)
				#api.destroy_direct_message(picked_dm.id)
				print picked_dm.text
				api.update_status(picked_dm.text)

				last_dm_id = dms[len(dms) - 1]
				first_dm_id = last_dm_id

			print "sleeping before following"
			sleep(1800)

			try:
				dms = api.direct_messages(first_dm_id)
				dms = parsing_dm_list(dms)
			except tweepy.error.TweepError:pass
		except IndexError:pass

<|endoftext|>


<commit_before>import logging
import os
import time
import unittest

import pytest

from tests.cook import util

@pytest.mark.prodskip
@pytest.mark.timeout(util.DEFAULT_TEST_TIMEOUT_SECS)  # individual test timeout
class TestDynamicClusters(util.CookTest):

    @classmethod
    def setUpClass(cls):
        cls.cook_url = util.retrieve_cook_url()
        util.init_cook_session(cls.cook_url)

    def setUp(self):
        self.cook_url = type(self).cook_url
        self.logger = logging.getLogger(__name__)
        self.user_factory = util.UserFactory(self)

    @unittest.skipUnless(util.using_kubernetes(), 'Test requires kubernetes')
    @unittest.skipUnless(os.getenv('COOK_TEST_DYNAMIC_CLUSTERS') is not None,
                         'Requires setting the COOK_TEST_DYNAMIC_CLUSTERS environment variable')
    @pytest.mark.serial
    def test_dynamic_clusters(self):
        """
        Test that dynamic cluster configuration functionality is working.
        """
        docker_image = util.docker_image()
        container = {'type': 'docker',
                     'docker': {'image': docker_image}}
        admin = self.user_factory.admin()
        # Force all clusters to have state = deleted via the API
        clusters = [cluster for cluster in util.compute_clusters(self.cook_url)['db-configs'] if cluster["state"] == "running"]
        with admin:
            self.logger.info(f'Clusters {clusters}')
            # First state = draining
            for cluster in clusters:
                cluster["state"] = "draining"
                cluster["state-locked?"] = True
                self.logger.info(f'Trying to update cluster {cluster}')
                data, resp = util.update_compute_cluster(self.cook_url, cluster)
                self.assertEqual(201, resp.status_code, resp.content)
            # Then state = deleted
            for cluster in clusters:
                cluster["state"] = "deleted"
                util.wait_until(lambda: util.update_compute_cluster(self.cook_url, cluster),
                                lambda x: 201 == x[1].status_code,
                                300000, 5000)
            # Create at least one new cluster with a unique test name (using one of the existing cluster's IP and cert)
            test_cluster_name = f'test_cluster_{round(time.time() * 1000)}'
            test_cluster = {
                "name": test_cluster_name,
                "state": "running",
                "base-path": clusters[0]["base-path"],
                "ca-cert": clusters[0]["ca-cert"],
                "template": clusters[0]["template"]
            }
            data, resp = util.create_compute_cluster(self.cook_url, test_cluster)
            self.assertEqual(201, resp.status_code, resp.content)
            # Test create cluster with duplicate name
            data, resp = util.create_compute_cluster(self.cook_url, test_cluster)
            self.assertEqual(422, resp.status_code, resp.content)
            self.assertEqual(f'Compute cluster with name {test_cluster_name} already exists',
                             data['error']['message'],
                             resp.content)

        # Check that a job schedules successfully
        command = "true"
        job_uuid, resp = util.submit_job(self.cook_url, command=command, container=container)
        self.assertEqual(201, resp.status_code, resp.content)
        instance = util.wait_for_instance(self.cook_url, job_uuid)
        message = repr(instance)
        self.assertIsNotNone(instance['compute-cluster'], message)
        instance_compute_cluster_name = instance['compute-cluster']['name']
        self.assertEqual(test_cluster["name"], instance_compute_cluster_name, instance['compute-cluster'])
        util.wait_for_instance(self.cook_url, job_uuid, status='success')
        running_clusters = [cluster for cluster in util.compute_clusters(self.cook_url)['db-configs'] if cluster["state"] == "running"]
        self.assertEqual(1, len(running_clusters), running_clusters)
        self.assertEqual(test_cluster["name"], running_clusters[0]["name"], running_clusters)

        with admin:
            # Delete test cluster
            # First state = draining
            test_cluster["state"] = "draining"
            data, resp = util.update_compute_cluster(self.cook_url, test_cluster)
            self.assertEqual(201, resp.status_code, resp.content)
            # Then state = deleted
            test_cluster["state"] = "deleted"
            util.wait_until(lambda: util.update_compute_cluster(self.cook_url, test_cluster),
                            lambda x: 201 == x[1].status_code,
                            300000, 5000)
            # Hard-delete the original non-test clusters
            for cluster in clusters:
                self.logger.info(f'Trying to delete cluster {cluster}')
                resp = util.delete_compute_cluster(self.cook_url, cluster)
                self.assertEqual(204, resp.status_code, resp.content)
            # Force give up leadership
            resp = util.shutdown_leader(self.cook_url, "test_dynamic_clusters")
            self.assertEqual(b'Accepted', resp)

        # Old clusters should be re-created
        # wait for cook to come up
        util.wait_until(lambda: [cluster for cluster in util.compute_clusters(self.cook_url)['db-configs'] if cluster["state"] == "running"],
                        lambda x: len(x) == len(clusters),
                        420000, 5000)
        # Check that a job schedules successfully
        command = "true"
        job_uuid, resp = util.submit_job(self.cook_url, command=command, container=container)
        self.assertEqual(201, resp.status_code, resp.content)
        util.wait_for_instance(self.cook_url, job_uuid, status='success')

        with admin:
            # Hard-delete test cluster
            resp = util.delete_compute_cluster(self.cook_url, test_cluster)
            self.assertEqual(204, resp.status_code, resp.content)

    @unittest.skipUnless(util.using_kubernetes(), 'Test requires kubernetes')
    def test_checkpoint_locality(self):
        """
        Test that restored instances run in the same location as their checkpointed instances.
        """
        # Get the set of clusters that correspond to the pool under test and are running
        pool = util.default_submit_pool()
        clusters = util.compute_clusters(self.cook_url)
        running_clusters = [c for c in clusters['in-mem-configs']
                            if pool in c['cluster-definition']['config']['synthetic-pods']['pools']
                            and c['state'] == 'running']
        self.logger.info(f'Running clusters for pool {pool}: {running_clusters}')
        if len(running_clusters) == 0:
            self.skipTest(f'Requires at least 1 running compute cluster for pool {pool}')

        # Submit an initial canary job
        job_uuid, resp = util.submit_job(self.cook_url, pool=pool, command='true')
        self.assertEqual(201, resp.status_code, resp.content)
        util.wait_for_instance(self.cook_url, job_uuid, status='success', indent=None)

        # Submit a long-running job with checkpointing
        checkpoint_job_uuid, resp = util.submit_job(
            self.cook_url,
            pool=pool,
            command=f'sleep {util.DEFAULT_TEST_TIMEOUT_SECS}',
            max_retries=5,
            checkpoint={'mode': 'auto'})
        self.assertEqual(201, resp.status_code, resp.content)

        try:
            # Wait for the job to be running
            checkpoint_instance = util.wait_for_instance(
                self.cook_url,
                checkpoint_job_uuid,
                status='running',
                indent=None)
            checkpoint_instance_uuid = checkpoint_instance['task_id']
            checkpoint_location = next(c['location'] for c in running_clusters
                                       if c['name'] == checkpoint_instance['compute-cluster']['name'])

            admin = self.user_factory.admin()
            try:
                # Force all clusters in the instance's location to have state = draining
                with admin:
                    for cluster in running_clusters:
                        if cluster['location'] == checkpoint_location:
                            cluster_update = dict(cluster)
                            # Set state = draining
                            cluster_update['state'] = 'draining'
                            cluster_update['state-locked?'] = True
                            # The location, cluster-definition, and features fields cannot be sent in the update
                            cluster_update.pop('location', None)
                            cluster_update.pop('cluster-definition', None)
                            cluster_update.pop('features', None)
                            self.logger.info(f'Trying to update cluster to draining: {cluster_update}')
                            util.wait_until(
                                lambda: util.update_compute_cluster(self.cook_url, cluster_update)[1],
                                lambda response: response.status_code == 201 and len(response.json()) > 0)
                        else:
                            self.logger.info(f'Not updating cluster - not in location {checkpoint_location}: {cluster}')

                # Kill the running checkpoint job instance
                util.kill_instance(self.cook_url, checkpoint_instance_uuid)

                # Submit another canary job
                job_uuid, resp = util.submit_job(self.cook_url, pool=pool, command='true')
                self.assertEqual(201, resp.status_code, resp.content)

                cluster_locations = set(c['location'] for c in running_clusters)
                if len(cluster_locations) > 1:
                    # The canary job should run in the non-draining location
                    self.logger.info(f'There are > 1 cluster locations under test: {cluster_locations}')
                    util.wait_for_instance(self.cook_url, job_uuid, status='success', indent=None)
                else:
                    self.logger.info(f'There is only 1 cluster location under test: {cluster_locations}')

                # The checkpoint job should be waiting
                util.wait_for_instance(self.cook_url, checkpoint_job_uuid, status='failed', indent=None)
                util.wait_for_job_in_statuses(self.cook_url, checkpoint_job_uuid, ['waiting'])
            finally:
                # Revert all clusters in the instance's location to state = running
                with admin:
                    for cluster in running_clusters:
                        if cluster['location'] == checkpoint_location:
                            cluster_update = dict(cluster)
                            # Set state = running
                            cluster_update['state'] = 'running'
                            cluster_update['state-locked?'] = False
                            # The location, cluster-definition, and features fields cannot be sent in the update
                            cluster_update.pop('location', None)
                            cluster_update.pop('cluster-definition', None)
                            cluster_update.pop('features', None)
                            self.logger.info(f'Trying to update cluster to running: {cluster_update}')
                            util.wait_until(
                                lambda: util.update_compute_cluster(self.cook_url, cluster_update)[1],
                                lambda response: response.status_code == 201 and len(response.json()) > 0)
                        else:
                            self.logger.info(f'Not updating cluster - not in location {checkpoint_location}: {cluster}')

                # Wait for the checkpoint job to be running again, in the same location as before
                checkpoint_instance = util.wait_for_instance(
                    self.cook_url,
                    checkpoint_job_uuid,
                    status='running',
                    indent=None)
                self.assertEqual(checkpoint_location,
                                 next(c['location'] for c in running_clusters
                                      if c['name'] == checkpoint_instance['compute-cluster']['name']))
        finally:
            # Kill the checkpoint job to not leave it running
            util.kill_jobs(self.cook_url, [checkpoint_job_uuid])
<commit_msg>clean up jobs in dynamic cluster test (#2027)<commit_after>import logging
import os
import time
import unittest

import pytest

from tests.cook import util

@pytest.mark.prodskip
@pytest.mark.timeout(util.DEFAULT_TEST_TIMEOUT_SECS)  # individual test timeout
class TestDynamicClusters(util.CookTest):

    @classmethod
    def setUpClass(cls):
        cls.cook_url = util.retrieve_cook_url()
        util.init_cook_session(cls.cook_url)

    def setUp(self):
        self.cook_url = type(self).cook_url
        self.logger = logging.getLogger(__name__)
        self.user_factory = util.UserFactory(self)

    @unittest.skipUnless(util.using_kubernetes(), 'Test requires kubernetes')
    @unittest.skipUnless(os.getenv('COOK_TEST_DYNAMIC_CLUSTERS') is not None,
                         'Requires setting the COOK_TEST_DYNAMIC_CLUSTERS environment variable')
    @pytest.mark.serial
    def test_dynamic_clusters(self):
        """
        Test that dynamic cluster configuration functionality is working.
        """
        docker_image = util.docker_image()
        container = {'type': 'docker',
                     'docker': {'image': docker_image}}
        admin = self.user_factory.admin()
        # Force all clusters to have state = deleted via the API
        clusters = [cluster for cluster in util.compute_clusters(self.cook_url)['db-configs'] if cluster["state"] == "running"]
        with admin:
            self.logger.info(f'Clusters {clusters}')
            # First state = draining
            for cluster in clusters:
                cluster["state"] = "draining"
                cluster["state-locked?"] = True
                self.logger.info(f'Trying to update cluster {cluster}')
                data, resp = util.update_compute_cluster(self.cook_url, cluster)
                self.assertEqual(201, resp.status_code, resp.content)
            # Then state = deleted
            for cluster in clusters:
                cluster["state"] = "deleted"
                util.wait_until(lambda: util.update_compute_cluster(self.cook_url, cluster),
                                lambda x: 201 == x[1].status_code,
                                300000, 5000)
            # Create at least one new cluster with a unique test name (using one of the existing cluster's IP and cert)
            test_cluster_name = f'test_cluster_{round(time.time() * 1000)}'
            test_cluster = {
                "name": test_cluster_name,
                "state": "running",
                "base-path": clusters[0]["base-path"],
                "ca-cert": clusters[0]["ca-cert"],
                "template": clusters[0]["template"]
            }
            data, resp = util.create_compute_cluster(self.cook_url, test_cluster)
            self.assertEqual(201, resp.status_code, resp.content)
            # Test create cluster with duplicate name
            data, resp = util.create_compute_cluster(self.cook_url, test_cluster)
            self.assertEqual(422, resp.status_code, resp.content)
            self.assertEqual(f'Compute cluster with name {test_cluster_name} already exists',
                             data['error']['message'],
                             resp.content)

        # Check that a job schedules successfully
        command = "true"
        job_uuid, resp = util.submit_job(self.cook_url, command=command, container=container)
        self.assertEqual(201, resp.status_code, resp.content)
        try:
            instance = util.wait_for_instance(self.cook_url, job_uuid)
            message = repr(instance)
            self.assertIsNotNone(instance['compute-cluster'], message)
            instance_compute_cluster_name = instance['compute-cluster']['name']
            self.assertEqual(test_cluster["name"], instance_compute_cluster_name, instance['compute-cluster'])
            util.wait_for_instance(self.cook_url, job_uuid, status='success')
            running_clusters = [cluster for cluster in util.compute_clusters(self.cook_url)['db-configs'] if cluster["state"] == "running"]
            self.assertEqual(1, len(running_clusters), running_clusters)
            self.assertEqual(test_cluster["name"], running_clusters[0]["name"], running_clusters)
        finally:
            util.kill_jobs(self.cook_url, [job_uuid], assert_response=False)

        with admin:
            # Delete test cluster
            # First state = draining
            test_cluster["state"] = "draining"
            data, resp = util.update_compute_cluster(self.cook_url, test_cluster)
            self.assertEqual(201, resp.status_code, resp.content)
            # Then state = deleted
            test_cluster["state"] = "deleted"
            util.wait_until(lambda: util.update_compute_cluster(self.cook_url, test_cluster),
                            lambda x: 201 == x[1].status_code,
                            300000, 5000)
            # Hard-delete the original non-test clusters
            for cluster in clusters:
                self.logger.info(f'Trying to delete cluster {cluster}')
                resp = util.delete_compute_cluster(self.cook_url, cluster)
                self.assertEqual(204, resp.status_code, resp.content)
            # Force give up leadership
            resp = util.shutdown_leader(self.cook_url, "test_dynamic_clusters")
            self.assertEqual(b'Accepted', resp)

        # Old clusters should be re-created
        # wait for cook to come up
        util.wait_until(lambda: [cluster for cluster in util.compute_clusters(self.cook_url)['db-configs'] if cluster["state"] == "running"],
                        lambda x: len(x) == len(clusters),
                        420000, 5000)
        # Check that a job schedules successfully
        command = "true"
        job_uuid, resp = util.submit_job(self.cook_url, command=command, container=container)
        self.assertEqual(201, resp.status_code, resp.content)
        try:
            util.wait_for_instance(self.cook_url, job_uuid, status='success')
        finally:
            util.kill_jobs(self.cook_url, [job_uuid], assert_response=False)

        with admin:
            # Hard-delete test cluster
            resp = util.delete_compute_cluster(self.cook_url, test_cluster)
            self.assertEqual(204, resp.status_code, resp.content)

    @unittest.skipUnless(util.using_kubernetes(), 'Test requires kubernetes')
    def test_checkpoint_locality(self):
        """
        Test that restored instances run in the same location as their checkpointed instances.
        """
        # Get the set of clusters that correspond to the pool under test and are running
        pool = util.default_submit_pool()
        clusters = util.compute_clusters(self.cook_url)
        running_clusters = [c for c in clusters['in-mem-configs']
                            if pool in c['cluster-definition']['config']['synthetic-pods']['pools']
                            and c['state'] == 'running']
        self.logger.info(f'Running clusters for pool {pool}: {running_clusters}')
        if len(running_clusters) == 0:
            self.skipTest(f'Requires at least 1 running compute cluster for pool {pool}')

        # Submit an initial canary job
        job_uuid, resp = util.submit_job(self.cook_url, pool=pool, command='true')
        self.assertEqual(201, resp.status_code, resp.content)
        util.wait_for_instance(self.cook_url, job_uuid, status='success', indent=None)

        # Submit a long-running job with checkpointing
        checkpoint_job_uuid, resp = util.submit_job(
            self.cook_url,
            pool=pool,
            command=f'sleep {util.DEFAULT_TEST_TIMEOUT_SECS}',
            max_retries=5,
            checkpoint={'mode': 'auto'})
        self.assertEqual(201, resp.status_code, resp.content)

        try:
            # Wait for the job to be running
            checkpoint_instance = util.wait_for_instance(
                self.cook_url,
                checkpoint_job_uuid,
                status='running',
                indent=None)
            checkpoint_instance_uuid = checkpoint_instance['task_id']
            checkpoint_location = next(c['location'] for c in running_clusters
                                       if c['name'] == checkpoint_instance['compute-cluster']['name'])

            admin = self.user_factory.admin()
            try:
                # Force all clusters in the instance's location to have state = draining
                with admin:
                    for cluster in running_clusters:
                        if cluster['location'] == checkpoint_location:
                            cluster_update = dict(cluster)
                            # Set state = draining
                            cluster_update['state'] = 'draining'
                            cluster_update['state-locked?'] = True
                            # The location, cluster-definition, and features fields cannot be sent in the update
                            cluster_update.pop('location', None)
                            cluster_update.pop('cluster-definition', None)
                            cluster_update.pop('features', None)
                            self.logger.info(f'Trying to update cluster to draining: {cluster_update}')
                            util.wait_until(
                                lambda: util.update_compute_cluster(self.cook_url, cluster_update)[1],
                                lambda response: response.status_code == 201 and len(response.json()) > 0)
                        else:
                            self.logger.info(f'Not updating cluster - not in location {checkpoint_location}: {cluster}')

                # Kill the running checkpoint job instance
                util.kill_instance(self.cook_url, checkpoint_instance_uuid)

                # Submit another canary job
                job_uuid, resp = util.submit_job(self.cook_url, pool=pool, command='true')
                self.assertEqual(201, resp.status_code, resp.content)

                cluster_locations = set(c['location'] for c in running_clusters)
                if len(cluster_locations) > 1:
                    # The canary job should run in the non-draining location
                    self.logger.info(f'There are > 1 cluster locations under test: {cluster_locations}')
                    util.wait_for_instance(self.cook_url, job_uuid, status='success', indent=None)
                else:
                    self.logger.info(f'There is only 1 cluster location under test: {cluster_locations}')

                # The checkpoint job should be waiting
                util.wait_for_instance(self.cook_url, checkpoint_job_uuid, status='failed', indent=None)
                util.wait_for_job_in_statuses(self.cook_url, checkpoint_job_uuid, ['waiting'])
            finally:
                # Revert all clusters in the instance's location to state = running
                with admin:
                    for cluster in running_clusters:
                        if cluster['location'] == checkpoint_location:
                            cluster_update = dict(cluster)
                            # Set state = running
                            cluster_update['state'] = 'running'
                            cluster_update['state-locked?'] = False
                            # The location, cluster-definition, and features fields cannot be sent in the update
                            cluster_update.pop('location', None)
                            cluster_update.pop('cluster-definition', None)
                            cluster_update.pop('features', None)
                            self.logger.info(f'Trying to update cluster to running: {cluster_update}')
                            util.wait_until(
                                lambda: util.update_compute_cluster(self.cook_url, cluster_update)[1],
                                lambda response: response.status_code == 201 and len(response.json()) > 0)
                        else:
                            self.logger.info(f'Not updating cluster - not in location {checkpoint_location}: {cluster}')

                # Wait for the checkpoint job to be running again, in the same location as before
                checkpoint_instance = util.wait_for_instance(
                    self.cook_url,
                    checkpoint_job_uuid,
                    status='running',
                    indent=None)
                self.assertEqual(checkpoint_location,
                                 next(c['location'] for c in running_clusters
                                      if c['name'] == checkpoint_instance['compute-cluster']['name']))
        finally:
            # Kill the checkpoint job to not leave it running
            util.kill_jobs(self.cook_url, [checkpoint_job_uuid])
<|endoftext|>


<commit_before>import discord
import re
import asyncio
import json
from handlers.issue_handler import IssueHandler
from handlers.card_handler import CardHandler
from handlers.enum_handler import EnumHandler

CMD_CARD = "!card "
CMD_CARD_COLLECTIBLE = "!cardc "
CMD_CARD_NONCOLLECTIBLE = "!cardn "
CMD_TAG = "!tag "
CMD_ENUM = "!enum "

with open('config.json', 'r') as f:
    config = json.load(f)

client = discord.Client()
issue_handler = IssueHandler(config["repos"])
card_handler = CardHandler()
enum_handler = EnumHandler()

@client.event
async def on_ready():
    print('Logged in as', client.user.name)

@client.event
async def on_message(message):
	if message.author.id == client.user.id:
		return
	if message.content.startswith(CMD_CARD):
		await handle_card(message, CMD_CARD);
		return
	if message.content.startswith(CMD_CARD_COLLECTIBLE):
		await handle_card(message, CMD_CARD_COLLECTIBLE, True);
		return
	if message.content.startswith(CMD_CARD_NONCOLLECTIBLE):
		await handle_card(message, CMD_CARD_NONCOLLECTIBLE, False);
		return
	if message.content.startswith(CMD_TAG):
		response = enum_handler.handle("GameTag " + message.content[len(CMD_TAG):])
		log(message, response)
		await client.send_message(message.channel, response);
		return
	if message.content.startswith(CMD_ENUM):
		response = enum_handler.handle(message.content[len(CMD_ENUM):])
		log(message, response)
		await client.send_message(message.channel, response);
		return

	matches = re.findall("(\w+)?#(\d+)", message.content)
	if len(matches):
		print("[%s]" % (message.channel), message.content)
	for match in matches:
		prefix = match[0]
		issue = match[1]
		response = issue_handler.handle(message.channel.name, prefix, issue)
		print("Reponse:", response)
		if response is not None:
			await client.send_message(message.channel, response)

async def handle_card(message, cmd, collectible = None):
		response = card_handler.handle(message.content[len(cmd):], max_response(message), collectible)
		log(message, response)
		await client.send_message(message.channel, response);

def max_response(message):
	return 10 if message.channel.is_private else 2

def log(message, response):
	print("[%s]" % (message.channel), message.content)
	print("Reponse:", response)

client.run(config["token"])<commit_msg>fix triggering github issues on github channel links<commit_after>import discord
import re
import asyncio
import json
from handlers.issue_handler import IssueHandler
from handlers.card_handler import CardHandler
from handlers.enum_handler import EnumHandler

CMD_CARD = "!card "
CMD_CARD_COLLECTIBLE = "!cardc "
CMD_CARD_NONCOLLECTIBLE = "!cardn "
CMD_TAG = "!tag "
CMD_ENUM = "!enum "

with open('config.json', 'r') as f:
    config = json.load(f)

client = discord.Client()
issue_handler = IssueHandler(config["repos"])
card_handler = CardHandler()
enum_handler = EnumHandler()

@client.event
async def on_ready():
    print('Logged in as', client.user.name)

@client.event
async def on_message(message):
	if message.author.id == client.user.id:
		return
	if message.content.startswith(CMD_CARD):
		await handle_card(message, CMD_CARD);
		return
	if message.content.startswith(CMD_CARD_COLLECTIBLE):
		await handle_card(message, CMD_CARD_COLLECTIBLE, True);
		return
	if message.content.startswith(CMD_CARD_NONCOLLECTIBLE):
		await handle_card(message, CMD_CARD_NONCOLLECTIBLE, False);
		return
	if message.content.startswith(CMD_TAG):
		response = enum_handler.handle("GameTag " + message.content[len(CMD_TAG):])
		log(message, response)
		await client.send_message(message.channel, response);
		return
	if message.content.startswith(CMD_ENUM):
		response = enum_handler.handle(message.content[len(CMD_ENUM):])
		log(message, response)
		await client.send_message(message.channel, response);
		return

	matches = re.findall("(?<!<)(\w+)?#(\d+)", message.content)
	if len(matches):
		print("[%s]" % (message.channel), message.content)
	for match in matches:
		prefix = match[0]
		issue = match[1]
		response = issue_handler.handle(message.channel.name, prefix, issue)
		print("Reponse:", response)
		if response is not None:
			await client.send_message(message.channel, response)

async def handle_card(message, cmd, collectible = None):
		response = card_handler.handle(message.content[len(cmd):], max_response(message), collectible)
		log(message, response)
		await client.send_message(message.channel, response);

def max_response(message):
	return 10 if message.channel.is_private else 2

def log(message, response):
	print("[%s]" % (message.channel), message.content)
	print("Reponse:", response)

client.run(config["token"])<|endoftext|>


<commit_before>import os
from sqlalchemy import func

from db import session
import model
import util

PROFILE_PICTURE_URL = '/images/profile/%d'

def active_years(user_id):
	if user_id is None:
		return []

	a = session.query(model.Year.id). \
		join(model.Wave, model.Wave.year == model.Year.id).\
		join(model.Task, model.Task.wave == model.Wave.id).\
		join(model.Module, model.Module.task == model.Task.id).\
		join(model.Evaluation, model.Evaluation.module == model.Task.id).\
		filter(model.Evaluation.user == user_id).\
		group_by(model.Year).all()
	return a

def any_task_submitted(user_id, year_id):
	if user_id is None:
		return False

	return session.query(model.Evaluation). \
		join(model.Module, model.Module.id == model.Evaluation.module).\
		filter(model.Evaluation.user == user_id).\
		join(model.Task, model.Task.id == model.Module.task).\
		join(model.Wave, model.Wave.id == model.Task.wave).\
		filter(model.Wave.year == year_id).count() > 0

def points_per_task(user_id):
        tasks = session.query(model.Task)

        return { task: util.task.points(task.id, user_id) for task in tasks }

def sum_points(user_id, year_id):
        points = session.query(model.Evaluation.module, func.max(model.Evaluation.points).label('points')).\
		filter(model.Evaluation.user == user_id).\
		join(model.Module, model.Evaluation.module == model.Module.id).\
		join(model.Task, model.Task.id == model.Module.task).\
		filter(model.Task.evaluation_public).\
		join(model.Wave, model.Wave.id == model.Task.wave).\
		filter(model.Wave.year == year_id).\
		group_by(model.Evaluation.module).all()

        return sum([ item.points for item in points if item.points is not None ])

def percentile(user_id, year_id):
        query = session.query(model.User).filter(model.User.role == 'participant')
        count = query.count()
        user_points = { user.id: sum_points(user.id, year_id) for user in query.all() }
        points_order = sorted(user_points.values(), reverse=True)

        if user_id not in user_points:
                return 0

        rank = 0.0
        for points in points_order:
                if points == user_points[user_id]:
                        return round((1 - (rank / count)) * 100)
                rank += 1

        return 0

def get_profile_picture(user):
        return PROFILE_PICTURE_URL % user.id if user.profile_picture and os.path.isfile(user.profile_picture) else None

def to_json(user, year_id):
        data = { 'id': user.id, 'first_name': user.first_name, 'last_name': user.last_name, 'profile_picture': get_profile_picture(user), 'gender': user.sex }

        data['role'] = user.role

        if user.role == 'participant':
                data['score'] =  sum_points(user.id, year_id)
                data['tasks_num'] = len(util.task.fully_submitted(user.id, year_id))
                data['achievements'] = list(util.achievement.ids_set(filter(lambda a: a.year == year_id, user.achievements)))

                profile = session.query(model.Profile).get(user.id)
                data['addr_country'] = profile.addr_country
                data['school_name'] = profile.school_name
                data['seasons'] = active_years(user.id)
        else:
                data['nick_name'] = user.nick_name
                data['tasks'] = [ task.id for task in user.tasks ]
                data['short_info'] = user.short_info

        return data
<commit_msg>Add orgs score :P<commit_after>import os
from sqlalchemy import func

from db import session
import model
import util

PROFILE_PICTURE_URL = '/images/profile/%d'

def active_years(user_id):
	if user_id is None:
		return []

	a = session.query(model.Year.id). \
		join(model.Wave, model.Wave.year == model.Year.id).\
		join(model.Task, model.Task.wave == model.Wave.id).\
		join(model.Module, model.Module.task == model.Task.id).\
		join(model.Evaluation, model.Evaluation.module == model.Task.id).\
		filter(model.Evaluation.user == user_id).\
		group_by(model.Year).all()
	return a

def any_task_submitted(user_id, year_id):
	if user_id is None:
		return False

	return session.query(model.Evaluation). \
		join(model.Module, model.Module.id == model.Evaluation.module).\
		filter(model.Evaluation.user == user_id).\
		join(model.Task, model.Task.id == model.Module.task).\
		join(model.Wave, model.Wave.id == model.Task.wave).\
		filter(model.Wave.year == year_id).count() > 0

def points_per_task(user_id):
        tasks = session.query(model.Task)

        return { task: util.task.points(task.id, user_id) for task in tasks }

def sum_points(user_id, year_id):
        points = session.query(model.Evaluation.module, func.max(model.Evaluation.points).label('points')).\
		filter(model.Evaluation.user == user_id).\
		join(model.Module, model.Evaluation.module == model.Module.id).\
		join(model.Task, model.Task.id == model.Module.task).\
		filter(model.Task.evaluation_public).\
		join(model.Wave, model.Wave.id == model.Task.wave).\
		filter(model.Wave.year == year_id).\
		group_by(model.Evaluation.module).all()

        return sum([ item.points for item in points if item.points is not None ])

def percentile(user_id, year_id):
        query = session.query(model.User).filter(model.User.role == 'participant')
        count = query.count()
        user_points = { user.id: sum_points(user.id, year_id) for user in query.all() }
        points_order = sorted(user_points.values(), reverse=True)

        if user_id not in user_points:
                return 0

        rank = 0.0
        for points in points_order:
                if points == user_points[user_id]:
                        return round((1 - (rank / count)) * 100)
                rank += 1

        return 0

def get_profile_picture(user):
        return PROFILE_PICTURE_URL % user.id if user.profile_picture and os.path.isfile(user.profile_picture) else None

def to_json(user, year_id):
        data = { 'id': user.id, 'first_name': user.first_name, 'last_name': user.last_name, 'profile_picture': get_profile_picture(user), 'gender': user.sex }

        data['role'] = user.role
        data['score'] =  sum_points(user.id, year_id)
        data['tasks_num'] = len(util.task.fully_submitted(user.id, year_id))
        data['achievements'] = list(util.achievement.ids_set(filter(lambda a: a.year == year_id, user.achievements)))

        if user.role == 'participant':
                profile = session.query(model.Profile).get(user.id)
                data['addr_country'] = profile.addr_country
                data['school_name'] = profile.school_name
                data['seasons'] = active_years(user.id)
        else:
                data['nick_name'] = user.nick_name
                data['tasks'] = [ task.id for task in user.tasks ]
                data['short_info'] = user.short_info

        return data
<|endoftext|>