<commit_before># find blobs in the incPCP foreground mask and code different blobs using different colors
import os
import cv2
import pdb
import pickle
import numpy as np
import glob as glob
import matplotlib.pyplot as plt
from scipy.sparse import csr_matrix

"""blob detector doesn't work..."""
"""
def find_blob(ori_img):
	# doesn't work!! segmentation fault (core dumped)
	# Setup SimpleBlobDetector parameters.
	params = cv2.SimpleBlobDetector_Params()

	# Change thresholds
	params.minThreshold = 10;
	params.maxThreshold = 200;

	# Filter by Area.
	params.filterByArea = False
	params.minArea = 10 #1500

	# Filter by Circularity
	params.filterByCircularity = False
	params.minCircularity = 0.1

	# Filter by Convexity
	params.filterByConvexity = False
	params.minConvexity = 0.87

	# Filter by Inertia
	params.filterByInertia = False
	params.minInertiaRatio = 0.01

	# Create a detector with the parameters
	ver = (cv2.__version__).split('.')
	if int(ver[0]) < 3 :
		detector = cv2.SimpleBlobDetector(params)
	else:
		detector = cv2.SimpleBlobDetector_create(params)

	keypoints = detector.detect(ori_img)
	# Draw detected blobs as red circles.
	# cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS ensures the size of the circle corresponds to the size of blob
	im_with_keypoints = cv2.drawKeypoints(ori_img, keypoints, np.array([]), (0,0,255), cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
	plt.imshow(im_with_keypoints)

	return keypoints
"""

def blobImg2blobmatrix(mask,ori_img):
	maskgray            = cv2.cvtColor(mask,cv2.COLOR_BGR2GRAY)
	ret,thresh          = cv2.threshold(maskgray,127,255,0)
	contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)

	# draw all the contours
	# ori_imgcopy         = ori_img.copy()
	# cv2.drawContours(ori_imgcopy, contours, -1, (0,255,0),thickness = 1)
	# plt.imshow(ori_imgcopy[:,:,::-1])
	# plt.pause(0.00001)

	#filter the contours by area, only keep the first 20
	cnts       = sorted(contours, key = cv2.contourArea, reverse = True)[:40]
	blobImage  = np.zeros(ori_img.shape)
	blobMatrix = np.zeros(ori_img.shape[:-1])
	color      = np.array([np.random.randint(0,255) for _ in range(3*int(len(cnts)))]).reshape(int(len(cnts)),3)

	for k in range(len(cnts)):
		if cv2.contourArea(cnts[k])<20:
			break
		# draw in class number, starts from 1
		cv2.drawContours(blobImage, cnts, k,(k+1,k+1,k+1),thickness = cv2.cv.CV_FILLED) # fill the contour for the blob
		# draw in real color
		# cv2.drawContours(blobImage, cnts, k,(color[k]),thickness = cv2.cv.CV_FILLED) # fill the contour for the blob
		blobMatrix = np.sum(blobImage,2)/3
		# if want to save the colored img
		# blobname = '/media/TOSHIBA/DoTdata/CanalSt@BaxterSt-96.106/incPCP/Canal_blobImage/'+str(frame_idx).zfill(7)+'.jpg'
		# cv2.imwrite(blobname, blobImage)
	# plt.imshow(blobImagee)
	# plt.pause(0.00001)

	return blobMatrix

if __name__ == '__main__':
	mask_list = sorted(glob.glob('/media/TOSHIBA/DoTdata/CanalSt@BaxterSt-96.106/incPCP/Canal_filled_hole/*.jpg'))
	ori_list  = sorted(glob.glob('../DoT/CanalSt@BaxterSt-96.106/CanalSt@BaxterSt-96.106_2015-06-16_16h03min52s762ms/*.jpg'))
	nframe    = min(len(mask_list),len(ori_list))
	frame_idx = 7800 #resume here, from 7800
	trunclen  = 600
	plt.figure()

	# temp_img   = cv2.imread(ori_list[0])
	# blobTensor = np.zeros((temp_img.shape[0],temp_img.shape[1],nframe))
	blob_sparse_list = []
	while frame_idx <nframe:
		print "frame_idx: ", frame_idx
		mask    = cv2.imread(mask_list[frame_idx])
		ori_img = cv2.imread(ori_list[frame_idx])

		"""blob detector doesn't work..."""
		# keypoints = find_blob(ori_img)
		"""use cv2.findContours, and then fill the contours..."""
		blobMatrix = blobImg2blobmatrix(mask,ori_img)
		# blobTensor[:,:,frame_idx] = blobMatrix
		sparse_slice = csr_matrix(blobMatrix)
		blob_sparse_list.append(sparse_slice)

		frame_idx = frame_idx+1
		#end of while loop
		if frame_idx>0 and np.mod(frame_idx,trunclen)==0:
			print "Save the blob index tensor into a pickle file:"
			# savePath = '../DoT/CanalSt@BaxterSt-96.106/'
			savePath = '/media/TOSHIBA/DoTdata/CanalSt@BaxterSt-96.106/incPCP/'
			savename = os.path.join(savePath,'blobTensorList'+str(frame_idx/trunclen).zfill(3)+'.p')
			pickle.dump(blob_sparse_list, open( savename, "wb" ))
			blob_sparse_list = []

	"""directly dumping to pickle not allowed, memory error! """
	# savename = os.path.join(savePath,'blobTensor.p')
	# pickle.dump( blobTensor, open( savename, "wb" ) )

<commit_msg>use blob center to replace SBS<commit_after># find blobs in the incPCP foreground mask and code different blobs using different colors
import os
import cv2
import pdb
import pickle
import numpy as np
import glob as glob
import matplotlib.pyplot as plt
from scipy.sparse import csr_matrix

import scipy as scipy

"""blob detector doesn't work..."""
"""
def find_blob(ori_img):
	# doesn't work!! segmentation fault (core dumped)
	# Setup SimpleBlobDetector parameters.
	params = cv2.SimpleBlobDetector_Params()

	# Change thresholds
	params.minThreshold = 10;
	params.maxThreshold = 200;

	# Filter by Area.
	params.filterByArea = False
	params.minArea = 10 #1500

	# Filter by Circularity
	params.filterByCircularity = False
	params.minCircularity = 0.1

	# Filter by Convexity
	params.filterByConvexity = False
	params.minConvexity = 0.87

	# Filter by Inertia
	params.filterByInertia = False
	params.minInertiaRatio = 0.01

	# Create a detector with the parameters
	ver = (cv2.__version__).split('.')
	if int(ver[0]) < 3 :
		detector = cv2.SimpleBlobDetector(params)
	else:
		detector = cv2.SimpleBlobDetector_create(params)

	keypoints = detector.detect(ori_img)
	# Draw detected blobs as red circles.
	# cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS ensures the size of the circle corresponds to the size of blob
	im_with_keypoints = cv2.drawKeypoints(ori_img, keypoints, np.array([]), (0,0,255), cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
	plt.imshow(im_with_keypoints)

	return keypoints
"""

def blobImg2blobmatrix(mask,ori_img):
	maskgray            = cv2.cvtColor(mask,cv2.COLOR_BGR2GRAY)
	ret,thresh          = cv2.threshold(maskgray,127,255,0)
	contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)

	# draw all the contours
	# ori_imgcopy         = ori_img.copy()
	# cv2.drawContours(ori_imgcopy, contours, -1, (0,255,0),thickness = 1)
	# plt.imshow(ori_imgcopy[:,:,::-1])
	# plt.pause(0.00001)

	#filter the contours by area, only keep the first 20
	cnts       = sorted(contours, key = cv2.contourArea, reverse = True)[:40]
	blobImage  = np.zeros(ori_img.shape)
	blobMatrix = np.zeros(ori_img.shape[:-1])
	color      = np.array([np.random.randint(0,255) for _ in range(3*int(len(cnts)))]).reshape(int(len(cnts)),3)

	for k in range(len(cnts)):
		if cv2.contourArea(cnts[k])<20:
			break
		# draw in class number, starts from 1
		cv2.drawContours(blobImage, cnts, k,(k+1,k+1,k+1),thickness = cv2.cv.CV_FILLED) # fill the contour for the blob
		# draw in real color
		# cv2.drawContours(blobImage, cnts, k,(color[k]),thickness = cv2.cv.CV_FILLED) # fill the contour for the blob
		blobMatrix = np.sum(blobImage,2)/3
		# if want to save the colored img
		# blobname = '/media/TOSHIBA/DoTdata/CanalSt@BaxterSt-96.106/incPCP/Canal_blobImage/'+str(frame_idx).zfill(7)+'.jpg'
		# cv2.imwrite(blobname, blobImage)
	# plt.imshow(blobImagee)
	# plt.pause(0.00001)

	labelMatrix = scipy.ndimage.measurements.label(mask)
	 = scipy.ndimage.measurements.center_of_mass(mask)

	return blobMatrix

if __name__ == '__main__':
	mask_list = sorted(glob.glob('/media/TOSHIBA/DoTdata/CanalSt@BaxterSt-96.106/incPCP/Canal_filled_hole/*.jpg'))
	ori_list  = sorted(glob.glob('../DoT/CanalSt@BaxterSt-96.106/CanalSt@BaxterSt-96.106_2015-06-16_16h03min52s762ms/*.jpg'))
	nframe    = min(len(mask_list),len(ori_list))
	frame_idx = 7800 #resume here, from 7800
	trunclen  = 600
	plt.figure()

	# temp_img   = cv2.imread(ori_list[0])
	# blobTensor = np.zeros((temp_img.shape[0],temp_img.shape[1],nframe))
	blob_sparse_list = []
	while frame_idx <nframe:
		print "frame_idx: ", frame_idx
		mask    = cv2.imread(mask_list[frame_idx])
		ori_img = cv2.imread(ori_list[frame_idx])

		"""blob detector doesn't work..."""
		# keypoints = find_blob(ori_img)
		"""use cv2.findContours, and then fill the contours..."""
		blobMatrix = blobImg2blobmatrix(mask,ori_img)
		# blobTensor[:,:,frame_idx] = blobMatrix
		sparse_slice = csr_matrix(blobMatrix)
		blob_sparse_list.append(sparse_slice)

		frame_idx = frame_idx+1
		#end of while loop
		if frame_idx>0 and np.mod(frame_idx,trunclen)==0:
			print "Save the blob index tensor into a pickle file:"
			# savePath = '../DoT/CanalSt@BaxterSt-96.106/'
			savePath = '/media/TOSHIBA/DoTdata/CanalSt@BaxterSt-96.106/incPCP/'
			savename = os.path.join(savePath,'blobTensorList'+str(frame_idx/trunclen).zfill(3)+'.p')
			pickle.dump(blob_sparse_list, open( savename, "wb" ))
			blob_sparse_list = []

	"""directly dumping to pickle not allowed, memory error! """
	# savename = os.path.join(savePath,'blobTensor.p')
	# pickle.dump( blobTensor, open( savename, "wb" ) )

<|endoftext|>


<commit_before>from __future__ import unicode_literals

import logging

from mopidy import backend
from mopidy.models import Ref

import requests

import spotipy

from translator import to_mopidy_track

logger = logging.getLogger(__name__)

def get_tracks_from_web_api(token):
    try:
        logger.debug(
            'Loading spotify-web library from web-api using token: %s', token)

        sp = spotipy.Spotify(auth=token)
        offset = 0
        tracks = []

        while True:
            results = sp.current_user_saved_tracks(limit=50, offset=offset)
            logger.debug('On spotify-web Got results %s for offset %s',
                         len(results['items']), offset)
            for item in results['items']:
                tracks.append(to_mopidy_track(item['track']))
                offset += 1
            if results['next'] is None:
                break

        return tracks
    except spotipy.SpotifyException as e:
        logger.error('Spotipy error(%s): %s', e.code, e.msg)
        return []

def get_fresh_token(config):
    try:
        logger.debug("authenticating")
        if config['use_mopidy_oauth_bridge']:
            response = get_fresh_token_from_mopidy(config)
        else:
            response = get_fresh_token_from_spotify(config)
        logger.debug("authentication response: %s", response.content)
        access_token = response.json()['access_token']
        logger.debug("authentication token: %s", access_token)
        return access_token
    except requests.exceptions.RequestException as e:
        logger.error('Refreshing the auth token failed: %s', e)
    except ValueError as e:
        logger.error('Decoding the JSON auth token response failed: %s', e)

def get_fresh_token_from_mopidy(config):
    mopidy_token_url = config['mopidy_token_url']
    logger.debug("authentication using mopidy swap service on: %s", mopidy_token_url)
    auth = (config['client_id'], config['client_secret'])
    return requests.post(mopidy_token_url, auth=auth, data={
        'grant_type': 'client_credentials',
    })

def get_fresh_token_from_spotify(config):
    spotify_token_url = config['spotify_token_url'];
    logger.debug("authentication using spotify on: %s", spotify_token_url)
    auth = (config['client_id'], config['client_secret'])
    return requests.post(spotify_token_url, auth=auth, data={
        'grant_type': 'refresh_token',
        'refresh_token': config['refresh_token'],
    })

class SpotifyWebLibraryProvider(backend.LibraryProvider):
    root_directory = Ref.directory(uri='spotifyweb:directory',
                                   name='Spotify Web Browse')

    def __init__(self, *args, **kwargs):
        super(SpotifyWebLibraryProvider, self).__init__(*args, **kwargs)
        self._cache = None
        self._root = [
            Ref.directory(uri='spotifyweb:artists', name='Artists'),
            Ref.directory(uri='spotifyweb:albums', name='Albums')]

    def refresh(self, uri=None):
        token = get_fresh_token(self.backend.config)
        if token is not None:
            tracks = get_tracks_from_web_api(token)
        else:
            tracks = []
        self._cache = Cache(tracks)

    def lookup(self, uri):
        pass

    def browse(self, uri):
        logger.debug("Request to browse %s in SpotifyWebLibraryProvider", uri)
        if uri == self.root_directory.uri:
            return self._root
        elif uri == 'spotifyweb:artists':
            return self._cache.sortedArtists
            # return Ref directory with all artists
        elif uri.startswith('spotifyweb:artist:'):
            # get artist uri
            return self._cache.artists2albums.get(uri)
            # return Ref directory with all albums for artist
        elif uri.startswith('spotifyweb:album:'):
            # get album uri
            return self._cache.albums2tracks.get(uri)
            # return Ref directory with all tracks for album
        elif uri == 'spotifyweb:albums':
            return self._cache.sortedAlbums
            # return Ref directory for all albums
        else:
            return []

class Cache:
    def __init__(self, tracks, *args, **kwargs):
        logger.debug("initializing SpotifyWebLibraryProvider cache")
        self.albums2tracks = {}
        self.artists2albums = {}
        self.sortedAlbums = []
        self.sortedArtists = []
        self.tracks = []
        for t in tracks:
            logger.debug("Adding track %s", t.name)
            self.tracks.append(Ref.track(name=t.name, uri=t.uri))
            if hasattr(t, 'album'):
                self.add_album_and_artists(t)

        logger.debug('Sorting albums and artists')
        cmp_dir_names = lambda x, y: cmp(x.name, y.name)
        self.sortedAlbums.sort(cmp_dir_names)
        self.sortedArtists.sort(cmp_dir_names)

    def add_album_and_artists(self, track):
        album = track.album

        name = album.name
        logger.debug('Going to add album %s', name)
        album_dir = Ref.directory(uri=album.uri, name=name)
        track_ref = Ref.track(name=track.name, uri=track.uri)
        if album.uri not in self.albums2tracks:
            self.sortedAlbums.append(album_dir)

        # adding track to album2tracks
        if album.uri in self.albums2tracks:
            if track_ref not in self.albums2tracks[album.uri]:
                self.albums2tracks[album.uri].append(track_ref)
        else:
            self.albums2tracks[album.uri] = [track_ref]

        if hasattr(track, 'artists'):
            self.add_artists(track, album_dir)

    def add_artists(self, track, album_dir):
        for artist in track.artists:
            artist_name = artist.name
            logger.debug('Going to add artist %s', artist_name)

            artist_dir = Ref.directory(uri=artist.uri, name=artist_name)
            if artist.uri not in self.artists2albums:
                self.sortedArtists.append(artist_dir)

            if artist.uri in self.artists2albums:
                if album_dir not in self.artists2albums[artist.uri]:
                    self.artists2albums[artist.uri].append(album_dir)
            else:
                self.artists2albums[artist.uri] = [album_dir]
<commit_msg>flake8 cleanup<commit_after>from __future__ import unicode_literals

import logging

from mopidy import backend
from mopidy.models import Ref

import requests

import spotipy

from translator import to_mopidy_track

logger = logging.getLogger(__name__)

def get_tracks_from_web_api(token):
    try:
        logger.debug(
            'Loading spotify-web library from web-api using token: %s', token)

        sp = spotipy.Spotify(auth=token)
        offset = 0
        tracks = []

        while True:
            results = sp.current_user_saved_tracks(limit=50, offset=offset)
            logger.debug('On spotify-web Got results %s for offset %s',
                         len(results['items']), offset)
            for item in results['items']:
                tracks.append(to_mopidy_track(item['track']))
                offset += 1
            if results['next'] is None:
                break

        return tracks
    except spotipy.SpotifyException as e:
        logger.error('Spotipy error(%s): %s', e.code, e.msg)
        return []

def get_fresh_token(config):
    try:
        logger.debug("authenticating")
        if config['use_mopidy_oauth_bridge']:
            response = get_fresh_token_from_mopidy(config)
        else:
            response = get_fresh_token_from_spotify(config)
        logger.debug("authentication response: %s", response.content)
        access_token = response.json()['access_token']
        logger.debug("authentication token: %s", access_token)
        return access_token
    except requests.exceptions.RequestException as e:
        logger.error('Refreshing the auth token failed: %s', e)
    except ValueError as e:
        logger.error('Decoding the JSON auth token response failed: %s', e)

def get_fresh_token_from_mopidy(config):
    mopidy_token_url = config['mopidy_token_url']
    logger.debug("authentication using mopidy swap service on: %s",
                 mopidy_token_url)
    auth = (config['client_id'], config['client_secret'])
    return requests.post(mopidy_token_url, auth=auth, data={
        'grant_type': 'client_credentials',
    })

def get_fresh_token_from_spotify(config):
    spotify_token_url = config['spotify_token_url']
    logger.debug("authentication using spotify on: %s", spotify_token_url)
    auth = (config['client_id'], config['client_secret'])
    return requests.post(spotify_token_url, auth=auth, data={
        'grant_type': 'refresh_token',
        'refresh_token': config['refresh_token'],
    })

class SpotifyWebLibraryProvider(backend.LibraryProvider):
    root_directory = Ref.directory(uri='spotifyweb:directory',
                                   name='Spotify Web Browse')

    def __init__(self, *args, **kwargs):
        super(SpotifyWebLibraryProvider, self).__init__(*args, **kwargs)
        self._cache = None
        self._root = [
            Ref.directory(uri='spotifyweb:artists', name='Artists'),
            Ref.directory(uri='spotifyweb:albums', name='Albums')]

    def refresh(self, uri=None):
        token = get_fresh_token(self.backend.config)
        if token is not None:
            tracks = get_tracks_from_web_api(token)
        else:
            tracks = []
        self._cache = Cache(tracks)

    def lookup(self, uri):
        pass

    def browse(self, uri):
        logger.debug("Request to browse %s in SpotifyWebLibraryProvider", uri)
        if uri == self.root_directory.uri:
            return self._root
        elif uri == 'spotifyweb:artists':
            return self._cache.sortedArtists
            # return Ref directory with all artists
        elif uri.startswith('spotifyweb:artist:'):
            # get artist uri
            return self._cache.artists2albums.get(uri)
            # return Ref directory with all albums for artist
        elif uri.startswith('spotifyweb:album:'):
            # get album uri
            return self._cache.albums2tracks.get(uri)
            # return Ref directory with all tracks for album
        elif uri == 'spotifyweb:albums':
            return self._cache.sortedAlbums
            # return Ref directory for all albums
        else:
            return []

class Cache:
    def __init__(self, tracks, *args, **kwargs):
        logger.debug("initializing SpotifyWebLibraryProvider cache")
        self.albums2tracks = {}
        self.artists2albums = {}
        self.sortedAlbums = []
        self.sortedArtists = []
        self.tracks = []
        for t in tracks:
            logger.debug("Adding track %s", t.name)
            self.tracks.append(Ref.track(name=t.name, uri=t.uri))
            if hasattr(t, 'album'):
                self.add_album_and_artists(t)

        logger.debug('Sorting albums and artists')
        cmp_dir_names = lambda x, y: cmp(x.name, y.name)
        self.sortedAlbums.sort(cmp_dir_names)
        self.sortedArtists.sort(cmp_dir_names)

    def add_album_and_artists(self, track):
        album = track.album

        name = album.name
        logger.debug('Going to add album %s', name)
        album_dir = Ref.directory(uri=album.uri, name=name)
        track_ref = Ref.track(name=track.name, uri=track.uri)
        if album.uri not in self.albums2tracks:
            self.sortedAlbums.append(album_dir)

        # adding track to album2tracks
        if album.uri in self.albums2tracks:
            if track_ref not in self.albums2tracks[album.uri]:
                self.albums2tracks[album.uri].append(track_ref)
        else:
            self.albums2tracks[album.uri] = [track_ref]

        if hasattr(track, 'artists'):
            self.add_artists(track, album_dir)

    def add_artists(self, track, album_dir):
        for artist in track.artists:
            artist_name = artist.name
            logger.debug('Going to add artist %s', artist_name)

            artist_dir = Ref.directory(uri=artist.uri, name=artist_name)
            if artist.uri not in self.artists2albums:
                self.sortedArtists.append(artist_dir)

            if artist.uri in self.artists2albums:
                if album_dir not in self.artists2albums[artist.uri]:
                    self.artists2albums[artist.uri].append(album_dir)
            else:
                self.artists2albums[artist.uri] = [album_dir]
<|endoftext|>


<commit_before>import unittest

import country, creature

class TestCreature(unittest.TestCase):
	def setUp(self):
		self.c = creature.Creature()

	def test_where_do_i_live(self):
		country1 = country.Country()
		country1.add_area([(0,0), (100,0), (100, 100)])
		country2 = country.Country()
		country2.add_area([(0,0), (100, 0), (100, -100)])
		self.assertIsNone(self.c.where_do_i_live())
		country1.area.add_resident(self.c)
		self.assertEqual(self.c.where_do_i_live(), country1)
		country2.area.add_resident(self.c)
		self.assertEqual(self.c.where_do_i_live(), country2)
<commit_msg>Adds test_travel<commit_after>import unittest

import country, creature

class TestCreature(unittest.TestCase):
	def setUp(self):
		self.c = creature.Creature()

	def test_where_do_i_live(self):
		country1 = country.Country()
		country1.add_area([(0,0), (100,0), (100, 100)])
		country2 = country.Country()
		country2.add_area([(0,0), (100, 0), (100, -100)])
		self.assertIsNone(self.c.where_do_i_live())
		country1.area.add_resident(self.c)
		self.assertEqual(self.c.where_do_i_live(), country1)
		country2.area.add_resident(self.c)
		self.assertEqual(self.c.where_do_i_live(), country2)

	#A criature can only travel to adjacent country
	def test_travel(self):
		country1 = country.Country()
		country1.add_area([(0,0), (100,0), (100, 100)])
		country2 = country.Country()
		country2.add_area([(0,0), (100, 0), (100, -100)])
		country3 = country.Country()
		country3.add_area([(200,0), (500, 0), (200, 200)])
		country1.area.add_resident(self.c)
		self.c.travel(country2)
		self.assertEqual(self.c.where_do_i_live(), c2)
		self.c.travel(country3)
		self.assertEqual(self.c.where_do_i_live(), c2)<|endoftext|>


<commit_before>from pywps.Process import WPSProcess

import icclim
import icclim.util.callback as callback

import dateutil.parser
from datetime import datetime
import os
from os.path import expanduser
from mkdir_p import *

transfer_limit_Mb = 100

class ProcessPercentileIndice(WPSProcess):

    def __init__(self):
        WPSProcess.__init__(self,
                            identifier = 'wps_percentile_indice', # only mandatary attribute = same file name
                            title = 'PercentileIndices',
                            abstract = 'Computes single input, percentile based indices of temperature: TG10p, TX10p, TN10p, TG90p, TX90p, TN90p, WSDI, CSDI; and of rainfall R75p, R95p, R99p, R75TOT, R95TOT, R99TOT.',
                            version = "1.0",
                            storeSupported = True,
                            statusSupported = True,
                            grassLocation =False)

        self.indiceNameIn = self.addLiteralInput(identifier = 'indiceName',
                                               title = 'Indice name',
                                               type="String",
                                               default = 'TG90p')

        self.indiceNameIn.values = ['TG10p', 'TX10p', 'TN10p', 'TG90p', 'TX90p', 'TN90p', 'WSDI', 'CSDI', 'R75p', 'R75TOT', 'R95p', 'R95TOT', 'R99p', 'R99TOT']

        self.sliceModeIn = self.addLiteralInput(identifier = 'sliceMode',
                                               title = 'Slice mode (temporal grouping to apply for calculations)',
                                               type="String",
                                               default = 'year')
        self.sliceModeIn.values = ["year","month","ONDJFM","AMJJAS","DJF","MAM","JJA","SON"]

        self.filesBasePeriodIn = self.addLiteralInput(identifier = 'filesBasePeriod',
                                               title = 'Input netCDF files list (base (reference) period)',
                                               abstract="application/netcdf",
                                               type=type("S"),
                                               minOccurs=0,
                                               maxOccurs=1024,
                                               default = 'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20060101-20251231.nc,' +
                                                          'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20260101-20501231.nc,' +
                                                          'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20510101-20751231.nc,' +
                                                          'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20760101-21001231.nc')

        self.timeRangeBasePeriodIn = self.addLiteralInput(identifier = 'timeRangeBasePeriod',
                                               title = 'Time range of base (reference) period, e.g. 1961-01-01/1990-12-31',
                                               type="String",
                                               default = 'Please Fill-in')

        self.filesStudyPeriodIn = self.addLiteralInput(identifier = 'filesStudyPeriod',
                                               title = 'Input netCDF files list (study period)',
                                               abstract="application/netcdf",
                                               type=type("S"),
                                               minOccurs=0,
                                               maxOccurs=1024,
                                               default = 'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20060101-20251231.nc,' +
                                                          'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20260101-20501231.nc,' +
                                                          'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20510101-20751231.nc,' +
                                                          'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20760101-21001231.nc')

        self.timeRangeStudyPeriodIn = self.addLiteralInput(identifier = 'timeRangeStudyPeriod',
                                               title = 'Time range, e.g. 2010-01-01/2012-12-31',
                                               type="String",
                                               default = 'Please Fill-in')

        self.varNameIn = self.addLiteralInput(identifier = 'varName',
                                               title = 'Variable name to process',
                                               type="String",
                                               default = 'tas')

        self.leapNonLeapYearsIn = self.addLiteralInput(identifier = 'leapNonLeapYears',
                                               title = 'Method for computing a percentile value for the calendar day of February 29th',
                                               type="String",
                                               default = "take all years (leap and non-leap)")
        self.leapNonLeapYearsIn.values = ["take all years (leap and non-leap)", "take only leap years"]

        self.outputFileNameIn = self.addLiteralInput(identifier = 'outputFileName',
                                               title = 'Name of output netCDF file',
                                               type="String",
                                               default = './out_icclim.nc')

        self.NLevelIn = self.addLiteralInput(identifier = 'NLevel',
                                               title = 'Number of level (if 4D variable)',
                                               type="String",
                                               default = None)

        self.opendapURL = self.addLiteralOutput(identifier = "opendapURL",title = "opendapURL");

    def callback(self,message,percentage):
        self.status.set("%s" % str(message),str(percentage));

    def execute(self):
        # Very important: This allows the NetCDF library to find the users credentials (X509 cert)
        homedir = os.environ['HOME']
        os.chdir(homedir)
        def callback(b):
          self.callback("Processing",b)

        indice_name = self.indiceNameIn.getValue()

        in_files = []
        in_files.extend(self.filesBasePeriodIn.getValue())

        time_range_base_period = self.timeRangeBasePeriodIn.getValue()

        var_name = self.varNameIn.getValue()

        leap_nonleap_years = self.leapNonLeapYearsIn.getValue()

        in_files.extend(self.filesStudyPeriodIn.getValue())

        time_range_study_period = self.timeRangeStudyPeriodIn.getValue()

        slice_mode = self.sliceModeIn.getValue()
        out_file_name = self.outputFileNameIn.getValue()
        level = self.NLevelIn.getValue()

        if (level == "None"):
            level = None

        if (time_range_base_period == "None"):
            time_range_base_period = None
        else:
            startdate = dateutil.parser.parse(time_range_base_period.split("/")[0])
            stopdate  = dateutil.parser.parse(time_range_base_period.split("/")[1])
            time_range_base_period = [startdate,stopdate]

        if(time_range_study_period == "None"):
            time_range_study_period = None
        else:
            startdate = dateutil.parser.parse(time_range_study_period.split("/")[0])
            stopdate  = dateutil.parser.parse(time_range_study_period.split("/")[1])
            time_range_study_period = [startdate,stopdate]

        if (leap_nonleap_years == "take all years (leap and non-leap)"):
            leap_nonleap_years = False
        else:
            leap_nonleap_years = True

        home = expanduser("~")

        self.status.set("Preparing....", 0)

        pathToAppendToOutputDirectory = "/WPS_"+self.identifier+"_" + datetime.now().strftime("%Y%m%dT%H%M%SZ")

        """ URL output path """
        fileOutURL  = os.environ['POF_OUTPUT_URL']  + pathToAppendToOutputDirectory+"/"

        """ Internal output path"""
        fileOutPath = os.environ['POF_OUTPUT_PATH']  + pathToAppendToOutputDirectory +"/"

        """ Create output directory """
        mkdir_p(fileOutPath)

        self.status.set("Processing input list: " + str(in_files), 0)

        icclim.indice(indice_name=indice_name,
                      in_files=in_files,
                      var_name=var_name,
                      slice_mode=slice_mode,
                      time_range=time_range_study_period,
                      out_file=fileOutPath+out_file_name,
                      N_lev=level,
                      transfer_limit_Mbytes=transfer_limit_Mb,
                      callback=callback,
                      callback_percentage_start_value=0,
                      callback_percentage_total=100,
                      base_period_time_range=time_range_base_period,
                      window_width=5,
                      only_leap_years=leap_nonleap_years,
                      ignore_Feb29th=True,
                      interpolation='hyndman_fan',
                      netcdf_version='NETCDF4_CLASSIC',
                      out_unit='days')

        """ Set output """
        url = fileOutURL+"/"+out_file_name;
        self.opendapURL.setValue(url);
        self.status.set("ready",100);

<commit_msg>adapted inputs and outputs<commit_after>from pywps.Process import WPSProcess

import icclim
import icclim.util.callback as callback

import dateutil.parser
from datetime import datetime
import os
from os.path import expanduser
#from mkdir_p import *

from flyingpigeon.utils import make_dirs

transfer_limit_Mb = 100

import logging
logger = logging.getLogger()

class ProcessPercentileIndice(WPSProcess):

    def __init__(self):
        WPSProcess.__init__(self,
                            identifier = 'wps_percentile_indice', # only mandatory attribute = same file name
                            title = 'c4i - Percentile Indices',
                            abstract = 'Computes single input, percentile based indices of temperature: TG10p, TX10p, TN10p, TG90p, TX90p, TN90p, WSDI, CSDI; and of rainfall R75p, R95p, R99p, R75TOT, R95TOT, R99TOT. This processes is also available in Climate4Impact and uses ICCLIM.',
                            version = "1.0",
                            metadata = [
                                {"title": "ICCLIM" , "href": "http://icclim.readthedocs.io/en/latest/"},
                                {"title": "Climate4Impact", "href": "http://climate4impact.eu/impactportal/general/index.jsp"},
                            ],
                            storeSupported = True,
                            statusSupported = True,
                            grassLocation =False)

        ## self.filesBasePeriodIn = self.addLiteralInput(identifier = 'filesBasePeriod',
        ##                                        title = 'Input netCDF files list (base (reference) period)',
        ##                                        abstract="application/netcdf",
        ##                                        type=type("S"),
        ##                                        minOccurs=0,
        ##                                        maxOccurs=1024,
        ##                                        default = 'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20060101-20251231.nc,' +
        ##                                                   'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20260101-20501231.nc,' +
        ##                                                   'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20510101-20751231.nc,' +
        ##                                                   'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20760101-21001231.nc')

        self.filesBasePeriodIn = self.addComplexInput(
            identifier="filesBasePeriod",
            title="URL to your NetCDF File",
            abstract="Input netCDF files list (base (reference) period). You may provide a URL or upload a NetCDF file.",
            minOccurs=1,
            maxOccurs=100,
            maxmegabites=10000,
            formats=[{"mimeType":"application/x-netcdf"}],
            )

        self.indiceNameIn = self.addLiteralInput(identifier = 'indiceName',
                                               title = 'Indice name',
                                               type=type("String"),
                                               minOccurs = 1,
                                               maxOccurs = 1,
                                               default = 'TG90p')

        self.indiceNameIn.values = ['TG10p', 'TX10p', 'TN10p', 'TG90p', 'TX90p', 'TN90p', 'WSDI', 'CSDI', 'R75p', 'R75TOT', 'R95p', 'R95TOT', 'R99p', 'R99TOT']

        self.sliceModeIn = self.addLiteralInput(identifier = 'sliceMode',
                                               title = 'Slice mode (temporal grouping to apply for calculations)',
                                               type=type("String"),
                                               minOccurs = 1,
                                               maxOccurs = 1,
                                               default = 'year')
        self.sliceModeIn.values = ["year","month","ONDJFM","AMJJAS","DJF","MAM","JJA","SON"]

        self.timeRangeBasePeriodIn = self.addLiteralInput(identifier = 'timeRangeBasePeriod',
                                               title = 'Time range of base (reference) period, e.g. 1961-01-01/1990-12-31',
                                               abstract = "Time range is mandatory. Please fill-in.",
                                               type=type("String"),
                                               minOccurs = 1,
                                               maxOccurs = 1,
                                               default = '1961-01-01/1990-12-31')

        self.filesStudyPeriodIn = self.addLiteralInput(identifier = 'filesStudyPeriod',
                                               title = 'Input netCDF files list (study period)',
                                               abstract="application/netcdf",
                                               type=type("S"),
                                               minOccurs=0,
                                               maxOccurs=1024,
                                               default = 'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20060101-20251231.nc,' +
                                                          'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20260101-20501231.nc,' +
                                                          'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20510101-20751231.nc,' +
                                                          'http://opendap.knmi.nl/knmi/thredds/dodsC/IS-ENES/TESTSETS/tas_day_EC-EARTH_rcp26_r8i1p1_20760101-21001231.nc')

        self.timeRangeStudyPeriodIn = self.addLiteralInput(identifier = 'timeRangeStudyPeriod',
                                               title = 'Time range, e.g. 2010-01-01/2012-12-31',
                                               abstract = "Time range is mandatory. Please fill in.",
                                               type=type("String"),
                                               default = '2010-01-01/2012-12-31')

        self.varNameIn = self.addLiteralInput(identifier = 'varName',
                                               title = 'Variable name to process',
                                               type=type("String"),
                                               default = 'tas')

        self.leapNonLeapYearsIn = self.addLiteralInput(
            identifier = 'leapNonLeapYears',
            title = 'Take only leap years?',
            abstract = "Method for computing a percentile value for the calendar day of February 29th. Default: take all years (leap and non-leap)",
            type=type(False),
            minOccurs=1,
            maxOccurs=1,
            default = False)
        #self.leapNonLeapYearsIn.values = ["take all years (leap and non-leap)", "take only leap years"]

        ## self.outputFileNameIn = self.addLiteralInput(identifier = 'outputFileName',
        ##                                        title = 'Name of output netCDF file',
        ##                                        type=type("String"),
        ##                                        default = './out_icclim.nc')

        self.NLevelIn = self.addLiteralInput(
            identifier = 'NLevel',
            title = 'Number of level (if 4D variable)',
            type=type(1),
            minOccurs = 0)

        #self.opendapURL = self.addLiteralOutput(identifier = "opendapURL",title = "opendapURL");

        self.output = self.addComplexOutput(
            identifier="output",
            title="Climate Indice",
            abstract="Calculated climate indice with icclim.",
            formats=[{"mimeType":"application/x-netcdf"}],
            asReference=True)

    def callback(self,message,percentage):
        self.status.set("%s" % str(message),str(percentage));

    def execute(self):
        # Very important: This allows the NetCDF library to find the users credentials (X509 cert)
        #homedir = os.environ['HOME']
        #os.chdir(homedir)
        def callback(b):
          self.callback("Processing",b)

        indice_name = self.indiceNameIn.getValue()

        in_files = self.getInputValues(identifier='filesBasePeriod')

        time_range_base_period = self.timeRangeBasePeriodIn.getValue()

        var_name = self.varNameIn.getValue()

        leap_nonleap_years = self.leapNonLeapYearsIn.getValue()

        in_files.extend(self.getInputValues(identifier='filesStudyPeriod'))

        time_range_study_period = self.timeRangeStudyPeriodIn.getValue()

        slice_mode = self.sliceModeIn.getValue()
        #out_file_name = self.outputFileNameIn.getValue()
        out_file_name = out.nc
        level = self.NLevelIn.getValue()

        if time_range_base_period:
            startdate = dateutil.parser.parse(time_range_base_period.split("/")[0])
            stopdate  = dateutil.parser.parse(time_range_base_period.split("/")[1])
            time_range_base_period = [startdate,stopdate]

        if time_range_study_period:
            startdate = dateutil.parser.parse(time_range_study_period.split("/")[0])
            stopdate  = dateutil.parser.parse(time_range_study_period.split("/")[1])
            time_range_study_period = [startdate,stopdate]

        ## if (leap_nonleap_years == "take all years (leap and non-leap)"):
        ##     leap_nonleap_years = False
        ## else:
        ##     leap_nonleap_years = True

        #home = expanduser("~")

        self.status.set("Preparing....", 0)

        #pathToAppendToOutputDirectory = "/WPS_"+self.identifier+"_" + datetime.now().strftime("%Y%m%dT%H%M%SZ")

        """ URL output path """
        #fileOutURL  = os.environ['POF_OUTPUT_URL']  + pathToAppendToOutputDirectory+"/"

        """ Internal output path"""
        #fileOutPath = os.environ['POF_OUTPUT_PATH']  + pathToAppendToOutputDirectory +"/"

        """ Create output directory """
        #mkdir_p(fileOutPath)

        self.status.set("Processing input list: " + str(in_files), 0)

        icclim.indice(indice_name=indice_name,
                      in_files=in_files,
                      var_name=var_name,
                      slice_mode=slice_mode,
                      time_range=time_range_study_period,
                      out_file=out_file_name,
                      N_lev=level,
                      transfer_limit_Mbytes=transfer_limit_Mb,
                      callback=callback,
                      callback_percentage_start_value=0,
                      callback_percentage_total=100,
                      base_period_time_range=time_range_base_period,
                      window_width=5,
                      only_leap_years=leap_nonleap_years,
                      ignore_Feb29th=True,
                      interpolation='hyndman_fan',
                      netcdf_version='NETCDF4_CLASSIC',
                      out_unit='days')

        """ Set output """
        #url = fileOutURL+"/"+out_file_name;
        #self.opendapURL.setValue(url);
        self.output.setValue(out_file_name)
        self.status.set("ready",100);

<|endoftext|>


<commit_before><commit_msg>Simplify<commit_after><|endoftext|>


<commit_before>def create_service_js(stream, module):
    """
    Creates AngularJS service from module
    :param stream:
    :param module:
    :return:
    """
    from wutu.util import get_implemented_methods
    from wutu.compiler.grammar import Provider, Function, Object, SimpleDeclare, String, Expression
    stream.write("wutu.factory('{0}Service', ['$http', ".format(module.__class__.__name__))
    http = Provider("$http")
    params = module.get_identifier()
    obj = Object()
    full_url = "base_url() + url + \"/\" + {0} + \"/\"".format(" + \"/\" + ".join(params))
    obj.add_member("get", Function(params=params, returns=http.get(full_url)))
    obj.add_member("put", Function(params=("data",), returns=http.put("base_url() + url", "data")))
    obj.add_member("post", Function(params=params + ("data",), returns=http.post(full_url, "data")))
    obj.add_member("delete", Function(params=params, returns=http.delete(full_url)))
    impl = Function([http.name],
                    body=[SimpleDeclare("url", String(module.__name__)), SimpleDeclare("service", obj, private=True)],
                    returns=Expression("service"))
    stream.write(impl.compile())
    stream.write("]);\n")
<commit_msg>fix for full url creation<commit_after>def create_service_js(stream, module):
    """
    Creates AngularJS service from module
    :param stream:
    :param module:
    :return:
    """
    from wutu.util import get_implemented_methods
    from wutu.compiler.grammar import Provider, Function, Object, SimpleDeclare, String, Expression
    stream.write("wutu.factory('{0}Service', ['$http', ".format(module.__class__.__name__))
    http = Provider("$http")
    params = module.get_identifier()
    obj = Object()
    extended = "+ \"/\" + {0} + \"/\"".format(" + \"/\" + ".join(params)) if len(params) > 0 else ""
    full_url = "base_url() + url {0}".format(extended)
    obj.add_member("get", Function(params=params, returns=http.get(full_url)))
    obj.add_member("put", Function(params=("data",), returns=http.put("base_url() + url", "data")))
    obj.add_member("post", Function(params=params + ("data",), returns=http.post(full_url, "data")))
    obj.add_member("delete", Function(params=params, returns=http.delete(full_url)))
    impl = Function([http.name],
                    body=[SimpleDeclare("url", String(module.__name__)), SimpleDeclare("service", obj, private=True)],
                    returns=Expression("service"))
    stream.write(impl.compile())
    stream.write("]);\n")
<|endoftext|>


<commit_before>"""
sentry.utils.http
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
import urllib
from urlparse import urlparse, urljoin

from django.conf import settings

def absolute_uri(url=None):
    if not url:
        return settings.SENTRY_URL_PREFIX
    return urljoin(settings.SENTRY_URL_PREFIX.rstrip('/') + '/', url.lstrip('/'))

def safe_urlencode(params, doseq=0):
    """
    UTF-8-safe version of safe_urlencode

    The stdlib safe_urlencode prior to Python 3.x chokes on UTF-8 values
    which can't fail down to ascii.
    """
    # Snippet originally from pysolr: https://github.com/toastdriven/pysolr

    if hasattr(params, "items"):
        params = params.items()

    new_params = list()

    for k, v in params:
        k = k.encode("utf-8")

        if isinstance(v, basestring):
            new_params.append((k, v.encode("utf-8")))
        elif isinstance(v, (list, tuple)):
            new_params.append((k, [i.encode("utf-8") for i in v]))
        else:
            new_params.append((k, unicode(v)))

    return urllib.urlencode(new_params, doseq)

def is_same_domain(url1, url2):
    """
    Returns true if the two urls should be treated as if they're from the same
    domain (trusted).
    """
    url1 = urlparse(url1)
    url2 = urlparse(url2)
    return url1.netloc == url2.netloc

def get_origins(project=None):
    if settings.SENTRY_ALLOW_ORIGIN == '*':
        return frozenset(['*'])

    if settings.SENTRY_ALLOW_ORIGIN:
        result = settings.SENTRY_ALLOW_ORIGIN.split(' ')
    else:
        result = []

    if project:
        # TODO: we should cache this
        from sentry.plugins.helpers import get_option
        optval = get_option('sentry:origins', project)
        if optval:
            result.extend(optval)

    # lowercase and strip the trailing slash from all origin values
    # filter out empty values
    return frozenset(filter(bool, map(lambda x: x.lower().rstrip('/'), result)))

def is_valid_origin(origin, project=None):
    """
    Given an ``origin`` which matches a base URI (e.g. http://example.com)
    determine if a valid origin is present in the project settings.

    Origins may be defined in several ways:

    - http://domain.com[:port]: exact match for base URI (must include port)
    - *: allow any domain
    - *.domain.com: matches domain.com and all subdomains, on any port
    - domain.com: matches domain.com on any port
    """
    allowed = get_origins(project)
    if '*' in allowed:
        return True

    if not origin:
        return False

    # we always run a case insensitive check
    origin = origin.lower()

    # Fast check
    if origin in allowed:
        return True

    # XXX: In some cases origin might be localhost (or something similar) which causes a string value
    # of 'null' to be sent as the origin
    if origin == 'null':
        return False

    parsed = urlparse(origin)

    for valid in allowed:
        if '://' in valid:
            # Support partial uri matches that may include path
            if origin.startswith(valid):
                return True
            continue

        if valid[:2] == '*.':
            # check foo.domain.com and domain.com
            if parsed.hostname.endswith(valid[1:]) or parsed.hostname == valid[2:]:
                return True
            continue

        if parsed.hostname == valid:
            return True

    return False
<commit_msg>Don't assume that origin is formatted correctly<commit_after>"""
sentry.utils.http
~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
import urllib
from urlparse import urlparse, urljoin

from django.conf import settings

def absolute_uri(url=None):
    if not url:
        return settings.SENTRY_URL_PREFIX
    return urljoin(settings.SENTRY_URL_PREFIX.rstrip('/') + '/', url.lstrip('/'))

def safe_urlencode(params, doseq=0):
    """
    UTF-8-safe version of safe_urlencode

    The stdlib safe_urlencode prior to Python 3.x chokes on UTF-8 values
    which can't fail down to ascii.
    """
    # Snippet originally from pysolr: https://github.com/toastdriven/pysolr

    if hasattr(params, "items"):
        params = params.items()

    new_params = list()

    for k, v in params:
        k = k.encode("utf-8")

        if isinstance(v, basestring):
            new_params.append((k, v.encode("utf-8")))
        elif isinstance(v, (list, tuple)):
            new_params.append((k, [i.encode("utf-8") for i in v]))
        else:
            new_params.append((k, unicode(v)))

    return urllib.urlencode(new_params, doseq)

def is_same_domain(url1, url2):
    """
    Returns true if the two urls should be treated as if they're from the same
    domain (trusted).
    """
    url1 = urlparse(url1)
    url2 = urlparse(url2)
    return url1.netloc == url2.netloc

def get_origins(project=None):
    if settings.SENTRY_ALLOW_ORIGIN == '*':
        return frozenset(['*'])

    if settings.SENTRY_ALLOW_ORIGIN:
        result = settings.SENTRY_ALLOW_ORIGIN.split(' ')
    else:
        result = []

    if project:
        # TODO: we should cache this
        from sentry.plugins.helpers import get_option
        optval = get_option('sentry:origins', project)
        if optval:
            result.extend(optval)

    # lowercase and strip the trailing slash from all origin values
    # filter out empty values
    return frozenset(filter(bool, map(lambda x: x.lower().rstrip('/'), result)))

def is_valid_origin(origin, project=None):
    """
    Given an ``origin`` which matches a base URI (e.g. http://example.com)
    determine if a valid origin is present in the project settings.

    Origins may be defined in several ways:

    - http://domain.com[:port]: exact match for base URI (must include port)
    - *: allow any domain
    - *.domain.com: matches domain.com and all subdomains, on any port
    - domain.com: matches domain.com on any port
    """
    allowed = get_origins(project)
    if '*' in allowed:
        return True

    if not origin:
        return False

    # we always run a case insensitive check
    origin = origin.lower()

    # Fast check
    if origin in allowed:
        return True

    # XXX: In some cases origin might be localhost (or something similar) which causes a string value
    # of 'null' to be sent as the origin
    if origin == 'null':
        return False

    parsed = urlparse(origin)

    # There is no hostname, so the header is probably invalid
    if parsed.hostname is None:
        return False

    for valid in allowed:
        if '://' in valid:
            # Support partial uri matches that may include path
            if origin.startswith(valid):
                return True
            continue

        if valid[:2] == '*.':
            # check foo.domain.com and domain.com
            if parsed.hostname.endswith(valid[1:]) or parsed.hostname == valid[2:]:
                return True
            continue

        if parsed.hostname == valid:
            return True

    return False
<|endoftext|>


<commit_before>import datetime
import yaml
from pprint import pformat, pprint

import django.forms
from django.contrib.auth.mixins import (
    LoginRequiredMixin,
    PermissionRequiredMixin,
)
from django.http import HttpResponse
from django.shortcuts import (
    get_object_or_404,
    redirect,
    render,
)
from django.template import loader
from django.urls import reverse
from django.utils import timezone
from django.views import View

from fo2.connections import db_cursor_so

from base.views import O2BaseGetPostView
from utils.functions.format import format_cnpj

from systextil.queries.tabela.deposito import query_deposito

import geral.forms as forms
import geral.queries as queries
from geral.dados.fluxo_roteiros import get_roteiros_de_fluxo
from geral.dados.fluxos import dict_fluxo
from geral.functions import (
    config_get_value,
    config_set_value,
    get_empresa,
)
from geral.models import (
    InformacaoModulo,
    Painel,
    PainelModulo,
    Pop,
    PopAssunto,
    UsuarioPainelModulo,
    UsuarioPopAssunto,
)

def index(request):
    if get_empresa(request) == 'agator':
        return render(request, 'geral/index_agator.html')
    else:
        return render(request, 'geral/index.html')

def estagio(request):
    cursor = db_cursor_so(request)
    data = queries.estagios(cursor)
    context = {
        'titulo': 'Estágios',
        'headers': ('Estágio', 'Descrição', 'Lead time', 'Depósito'),
        'fields': ('EST', 'DESCR', 'LT', 'DEP'),
        'data': data,
    }
    return render(request, 'geral/tabela_geral.html', context)

def periodo_confeccao(request):
    cursor = db_cursor_so(request)
    data = queries.periodos_confeccao(cursor)

    for row in data:
        row['INI'] = row['INI'].date()
        row['FIM'] = row['FIM'].date()

    context = {
        'titulo': 'Períodos de confecção',
        'headers': ('Período', 'Data de início', 'Data de fim'),
        'fields': ('PERIODO', 'INI', 'FIM'),
        'data': data,
    }
    return render(request, 'geral/tabela_geral.html', context)

class PainelView(View):

    def get(self, request, *args, **kwargs):
        try:
            painel = Painel.objects.get(slug=kwargs['painel'], habilitado=True)
        except Painel.DoesNotExist:
            return redirect('apoio_ao_erp')

        ultimo_mes = timezone.now() - datetime.timedelta(days=61)

        layout = painel.layout
        config = yaml.load(layout)
        for dado in config['dados']:
            try:
                modulo = PainelModulo.objects.get(slug=dado['modulo'])
            except Exception:
                return redirect('apoio_ao_erp')

            if modulo.tipo == 'I':
                dado['modulo_nome'] = modulo.nome
                dado['dados'] = InformacaoModulo.objects.filter(
                    painel_modulo__slug=dado['modulo'],
                    habilitado=True,
                    data__gt=ultimo_mes,
                ).order_by('-data')

        context = {
            'titulo': painel.nome,
            'config': config,
            }
        return render(
            request, f"geral/{config['template']}.html", context)

class InformativoView(LoginRequiredMixin, View):
    Form_class = forms.InformacaoModuloForm
    template_name = 'geral/informativo.html'
    title_name = 'Informativos'
    context = {}
    informativo_id = None

    def list_informativo(self):
        self.context['informativos'] = InformacaoModulo.objects.filter(
            painel_modulo=self.modulo).order_by('-data')

    def tem_permissao(self, request, **kwargs):
        modulo_slug = kwargs['modulo']
        self.modulo = PainelModulo.objects.get(slug=modulo_slug)
        self.context = {
            'titulo': '{} - {}'.format(self.modulo.nome, self.title_name),
            'modulo_slug': modulo_slug,
            }

        verificacao = UsuarioPainelModulo.objects.filter(
            usuario=request.user, painel_modulo=self.modulo)
        if len(verificacao) == 0:
            self.context['msg_erro'] =\
                'Usuário não tem direito de manter o informativo "{}"'.format(
                    self.modulo.nome
                )
            return False
        return True

    def get_informativo_id(self, **kwargs):
        if 'id' in kwargs:
            self.informativo_id = kwargs['id']

    def get(self, request, *args, **kwargs):
        if not self.tem_permissao(request, **kwargs):
            return render(request, self.template_name, self.context)

        self.get_informativo_id(**kwargs)
        if self.informativo_id == 'add':
            form = self.Form_class()
            self.context['form'] = form
        else:
            informativo_por_id = InformacaoModulo.objects.filter(
                id=self.informativo_id)
            if len(informativo_por_id) == 0:
                self.list_informativo()
            else:
                self.context['informativo_id'] = self.informativo_id
                form = self.Form_class(
                    initial={'chamada': informativo_por_id[0].chamada,
                             'habilitado': informativo_por_id[0].habilitado})
                self.context['form'] = form

        return render(request, self.template_name, self.context)

    def post(self, request, *args, **kwargs):
        if not self.tem_permissao(request, **kwargs):
            return render(request, self.template_name, self.context)

        form = None
        self.get_informativo_id(**kwargs)
        if self.informativo_id == 'add':
            form = self.Form_class(request.POST)
        else:
            informativo_por_id = InformacaoModulo.objects.filter(
                id=self.informativo_id)
            if len(informativo_por_id) == 0:
                self.list_informativo()
            else:
                self.context['informativo_id'] = self.informativo_id
                form = self.Form_class(
                    request.POST,
                    initial={'chamada': informativo_por_id[0].chamada,
                             'habilitado': informativo_por_id[0].habilitado})

        if form is not None:
            if form.is_valid():
                chamada = form.cleaned_data['chamada']
                habilitado = form.cleaned_data['habilitado']
                if self.informativo_id == 'add':
                    informativo = InformacaoModulo(
                        usuario=request.user)
                else:
                    informativo = InformacaoModulo.objects.get(
                        id=self.informativo_id)
                informativo.chamada = chamada
                informativo.habilitado = habilitado
                informativo.painel_modulo = self.modulo
                informativo.save()
                self.list_informativo()
            else:
                self.context['form'] = form
        return render(request, self.template_name, self.context)

def pop(request, pop_assunto=None, id=None):
    assunto = PopAssunto.objects.get(slug=pop_assunto)
    if assunto.grupo_slug == '':
        titulo = 'POPs de {}'.format(assunto.nome)
    else:
        titulo = assunto.nome
    context = {'titulo': titulo}

    can_edit = False
    user = None
    if request.user.is_authenticated:
        user = request.user
    if user:
        can_edit = user.has_perm('geral.can_manage_pop')
        if can_edit:
            verificacao = UsuarioPopAssunto.objects.filter(
                usuario=request.user, assunto=assunto)
            can_edit = len(verificacao) != 0

    if can_edit:
        if id:
            instance = get_object_or_404(Pop, id=id)
            context.update({'edit': True})
        else:
            instance = None
            context.update({'insert': True})
        if request.method == 'POST':
            form = forms.PopForm(
                request.POST, request.FILES, instance=instance)
            if form.is_valid():
                form.save()
                return redirect('geral:pop', pop_assunto)
        else:
            if instance is None:
                form = forms.PopForm()
                form.fields['assunto'].initial = assunto.id
            else:
                form = forms.PopForm(instance=instance)
        form.fields['assunto'].widget = django.forms.HiddenInput()
        context.update({'form': form})

    if can_edit:
        select = Pop.objects.filter(assunto__slug=pop_assunto)
        select = select.order_by('-uploaded_at')
    else:
        select = Pop.objects.filter(assunto__slug=pop_assunto, habilitado=True)
        select = select.order_by('descricao')
    select = select.values()
    data = list(select)
    for row in data:
        row['descricao|LINK'] = '/media/{}'.format(row['pop'])
        row['descricao|TARGET'] = '_blank'
        row['habilitado'] = 'sim' if row['habilitado'] else 'não'
        row['edit'] = ''
        row['edit|LINK'] = reverse('geral:pop', args=[pop_assunto, row['id']])
    context.update({
        'data': data,
    })
    if can_edit:
        context.update({
            'headers': ('Adicionado em', 'Título', 'Arquivo POP',
                        'Habilitado', 'Editar'),
            'fields': ('uploaded_at', 'descricao', 'pop',
                       'habilitado', 'edit'),
        })
    else:
        context.update({
            'headers': ['Título'],
            'fields': ['descricao'],
        })
    if get_empresa(request) == 'agator':
        context.update({
            'extends_html': 'geral/index_agator.html',
        })
    else:
        context.update({
            'extends_html': 'geral/index.html'
        })
    return render(request, 'geral/pop.html', context)

def gera_fluxo(request, destino, id):
    fluxo = dict_fluxo(id)
    if fluxo is None:
        return HttpResponse("Fluxo {} não encontrado".format(id))

    if destino in ['a', 'f']:
        filename = \
            'roteiros_alt{id}_{versao_num}_{versao_sufixo}' \
            '.dot'.format(**fluxo)
        templ = loader.get_template(fluxo['template_base'])
        http_resp = HttpResponse(
            templ.render(fluxo, request), content_type='text/plain')
        http_resp['Content-Disposition'] = \
            'attachment; filename="{filename}"'.format(filename=filename)
        return http_resp

    else:
        return render(
            request, fluxo['template_base'], fluxo,
            content_type='text/plain')

class ExecGeraFluxo(O2BaseGetPostView):
    def __init__(self, *args, **kwargs):
        super(ExecGeraFluxo, self).__init__(*args, **kwargs)
        self.Form_class = forms.ExecGeraFluxoForm
        self.template_name = 'geral/exec_gera_fluxo.html'
        self.title_name = 'Gera fluxo'
        self.get_args = ['destino', 'id']

    def mount_context(self):
        destino = self.form.cleaned_data['destino']
        id = self.form.cleaned_data['id']
        self.context.update({
            'destino': destino,
            'id': id,
        })
        fluxo = dict_fluxo(id)
        if fluxo is None:
            self.context.update({
                'erro': "Fluxo {} não encontrado".format(id),
            })

def roteiros_de_fluxo(request, id):
    roteiros = get_roteiros_de_fluxo(id)
    return HttpResponse(
        pformat(roteiros, indent=4),
        content_type='text/plain')

def unidade(request):
    cursor = db_cursor_so(request)
    data = queries.unidades(cursor)
    context = {
        'titulo': 'Unidades / Divisões',
        'headers': ('Código ', 'Descrição', 'UF', 'Cidade',
                    '(CNPJ) Razão social'),
        'fields': ('DIV', 'DESCR', 'UF', 'CIDADE',
                   'NOME'),
        'data': data,
    }
    return render(request, 'geral/unidade.html', context)

class Configuracao(PermissionRequiredMixin, View):
    def __init__(self, *args, **kwargs):
        self.permission_required = 'geral.change_config'
        self.Form_class = forms.ConfigForm
        self.template_name = 'geral/config.html'
        self.title_name = 'Configuração'

    def get_values(self, request):
        values = {}
        for field in self.Form_class.field_param:
            param = self.Form_class.field_param[field]
            if request.user.is_superuser:
                values[field] = config_get_value(param)
            else:
                values[field] = config_get_value(param, request.user)
        return values

    def set_values(self, request, values):
        if request.user.is_superuser:
            usuario = None
        else:
            usuario = request.user
        ok = True
        for field in self.Form_class.field_param:
            param = self.Form_class.field_param[field]
            ok = ok and config_set_value(param, values[field], usuario=usuario)
        return ok

    def get(self, request, *args, **kwargs):
        context = {'titulo': self.title_name}
        form = self.Form_class()
        values = self.get_values(request)
        form.fields["op_unidade"].initial = values['op_unidade']
        form.fields["dias_alem_lead"].initial = values['dias_alem_lead']
        context['form'] = form
        return render(request, self.template_name, context)

    def post(self, request, *args, **kwargs):
        context = {'titulo': self.title_name}
        form = self.Form_class(request.POST)
        if form.is_valid():
            values = {}
            values['op_unidade'] = form.cleaned_data['op_unidade']
            values['dias_alem_lead'] = form.cleaned_data['dias_alem_lead']
            if self.set_values(request, values):
                context['msg'] = 'Valores salvos!'
            else:
                context['msg'] = 'Houve algum erro ao salvar os valores!'
        context['form'] = form
        return render(request, self.template_name, context)
<commit_msg>view PainelView bug no yaml.load<commit_after>import datetime
import yaml
from pprint import pformat, pprint

import django.forms
from django.contrib.auth.mixins import (
    LoginRequiredMixin,
    PermissionRequiredMixin,
)
from django.http import HttpResponse
from django.shortcuts import (
    get_object_or_404,
    redirect,
    render,
)
from django.template import loader
from django.urls import reverse
from django.utils import timezone
from django.views import View

from fo2.connections import db_cursor_so

from base.views import O2BaseGetPostView
from utils.functions.format import format_cnpj

from systextil.queries.tabela.deposito import query_deposito

import geral.forms as forms
import geral.queries as queries
from geral.dados.fluxo_roteiros import get_roteiros_de_fluxo
from geral.dados.fluxos import dict_fluxo
from geral.functions import (
    config_get_value,
    config_set_value,
    get_empresa,
)
from geral.models import (
    InformacaoModulo,
    Painel,
    PainelModulo,
    Pop,
    PopAssunto,
    UsuarioPainelModulo,
    UsuarioPopAssunto,
)

def index(request):
    if get_empresa(request) == 'agator':
        return render(request, 'geral/index_agator.html')
    else:
        return render(request, 'geral/index.html')

def estagio(request):
    cursor = db_cursor_so(request)
    data = queries.estagios(cursor)
    context = {
        'titulo': 'Estágios',
        'headers': ('Estágio', 'Descrição', 'Lead time', 'Depósito'),
        'fields': ('EST', 'DESCR', 'LT', 'DEP'),
        'data': data,
    }
    return render(request, 'geral/tabela_geral.html', context)

def periodo_confeccao(request):
    cursor = db_cursor_so(request)
    data = queries.periodos_confeccao(cursor)

    for row in data:
        row['INI'] = row['INI'].date()
        row['FIM'] = row['FIM'].date()

    context = {
        'titulo': 'Períodos de confecção',
        'headers': ('Período', 'Data de início', 'Data de fim'),
        'fields': ('PERIODO', 'INI', 'FIM'),
        'data': data,
    }
    return render(request, 'geral/tabela_geral.html', context)

class PainelView(View):

    def get(self, request, *args, **kwargs):
        try:
            painel = Painel.objects.get(slug=kwargs['painel'], habilitado=True)
        except Painel.DoesNotExist:
            return redirect('apoio_ao_erp')

        ultimo_mes = timezone.now() - datetime.timedelta(days=61)

        layout = painel.layout
        pprint(layout)
        config = yaml.load(layout, Loader=yaml.Loader)
        pprint(config)
        for dado in config['dados']:
            try:
                modulo = PainelModulo.objects.get(slug=dado['modulo'])
            except Exception:
                return redirect('apoio_ao_erp')

            if modulo.tipo == 'I':
                dado['modulo_nome'] = modulo.nome
                dado['dados'] = InformacaoModulo.objects.filter(
                    painel_modulo__slug=dado['modulo'],
                    habilitado=True,
                    data__gt=ultimo_mes,
                ).order_by('-data')

        context = {
            'titulo': painel.nome,
            'config': config,
            }
        return render(
            request, f"geral/{config['template']}.html", context)

class InformativoView(LoginRequiredMixin, View):
    Form_class = forms.InformacaoModuloForm
    template_name = 'geral/informativo.html'
    title_name = 'Informativos'
    context = {}
    informativo_id = None

    def list_informativo(self):
        self.context['informativos'] = InformacaoModulo.objects.filter(
            painel_modulo=self.modulo).order_by('-data')

    def tem_permissao(self, request, **kwargs):
        modulo_slug = kwargs['modulo']
        self.modulo = PainelModulo.objects.get(slug=modulo_slug)
        self.context = {
            'titulo': '{} - {}'.format(self.modulo.nome, self.title_name),
            'modulo_slug': modulo_slug,
            }

        verificacao = UsuarioPainelModulo.objects.filter(
            usuario=request.user, painel_modulo=self.modulo)
        if len(verificacao) == 0:
            self.context['msg_erro'] =\
                'Usuário não tem direito de manter o informativo "{}"'.format(
                    self.modulo.nome
                )
            return False
        return True

    def get_informativo_id(self, **kwargs):
        if 'id' in kwargs:
            self.informativo_id = kwargs['id']

    def get(self, request, *args, **kwargs):
        if not self.tem_permissao(request, **kwargs):
            return render(request, self.template_name, self.context)

        self.get_informativo_id(**kwargs)
        if self.informativo_id == 'add':
            form = self.Form_class()
            self.context['form'] = form
        else:
            informativo_por_id = InformacaoModulo.objects.filter(
                id=self.informativo_id)
            if len(informativo_por_id) == 0:
                self.list_informativo()
            else:
                self.context['informativo_id'] = self.informativo_id
                form = self.Form_class(
                    initial={'chamada': informativo_por_id[0].chamada,
                             'habilitado': informativo_por_id[0].habilitado})
                self.context['form'] = form

        return render(request, self.template_name, self.context)

    def post(self, request, *args, **kwargs):
        if not self.tem_permissao(request, **kwargs):
            return render(request, self.template_name, self.context)

        form = None
        self.get_informativo_id(**kwargs)
        if self.informativo_id == 'add':
            form = self.Form_class(request.POST)
        else:
            informativo_por_id = InformacaoModulo.objects.filter(
                id=self.informativo_id)
            if len(informativo_por_id) == 0:
                self.list_informativo()
            else:
                self.context['informativo_id'] = self.informativo_id
                form = self.Form_class(
                    request.POST,
                    initial={'chamada': informativo_por_id[0].chamada,
                             'habilitado': informativo_por_id[0].habilitado})

        if form is not None:
            if form.is_valid():
                chamada = form.cleaned_data['chamada']
                habilitado = form.cleaned_data['habilitado']
                if self.informativo_id == 'add':
                    informativo = InformacaoModulo(
                        usuario=request.user)
                else:
                    informativo = InformacaoModulo.objects.get(
                        id=self.informativo_id)
                informativo.chamada = chamada
                informativo.habilitado = habilitado
                informativo.painel_modulo = self.modulo
                informativo.save()
                self.list_informativo()
            else:
                self.context['form'] = form
        return render(request, self.template_name, self.context)

def pop(request, pop_assunto=None, id=None):
    assunto = PopAssunto.objects.get(slug=pop_assunto)
    if assunto.grupo_slug == '':
        titulo = 'POPs de {}'.format(assunto.nome)
    else:
        titulo = assunto.nome
    context = {'titulo': titulo}

    can_edit = False
    user = None
    if request.user.is_authenticated:
        user = request.user
    if user:
        can_edit = user.has_perm('geral.can_manage_pop')
        if can_edit:
            verificacao = UsuarioPopAssunto.objects.filter(
                usuario=request.user, assunto=assunto)
            can_edit = len(verificacao) != 0

    if can_edit:
        if id:
            instance = get_object_or_404(Pop, id=id)
            context.update({'edit': True})
        else:
            instance = None
            context.update({'insert': True})
        if request.method == 'POST':
            form = forms.PopForm(
                request.POST, request.FILES, instance=instance)
            if form.is_valid():
                form.save()
                return redirect('geral:pop', pop_assunto)
        else:
            if instance is None:
                form = forms.PopForm()
                form.fields['assunto'].initial = assunto.id
            else:
                form = forms.PopForm(instance=instance)
        form.fields['assunto'].widget = django.forms.HiddenInput()
        context.update({'form': form})

    if can_edit:
        select = Pop.objects.filter(assunto__slug=pop_assunto)
        select = select.order_by('-uploaded_at')
    else:
        select = Pop.objects.filter(assunto__slug=pop_assunto, habilitado=True)
        select = select.order_by('descricao')
    select = select.values()
    data = list(select)
    for row in data:
        row['descricao|LINK'] = '/media/{}'.format(row['pop'])
        row['descricao|TARGET'] = '_blank'
        row['habilitado'] = 'sim' if row['habilitado'] else 'não'
        row['edit'] = ''
        row['edit|LINK'] = reverse('geral:pop', args=[pop_assunto, row['id']])
    context.update({
        'data': data,
    })
    if can_edit:
        context.update({
            'headers': ('Adicionado em', 'Título', 'Arquivo POP',
                        'Habilitado', 'Editar'),
            'fields': ('uploaded_at', 'descricao', 'pop',
                       'habilitado', 'edit'),
        })
    else:
        context.update({
            'headers': ['Título'],
            'fields': ['descricao'],
        })
    if get_empresa(request) == 'agator':
        context.update({
            'extends_html': 'geral/index_agator.html',
        })
    else:
        context.update({
            'extends_html': 'geral/index.html'
        })
    return render(request, 'geral/pop.html', context)

def gera_fluxo(request, destino, id):
    fluxo = dict_fluxo(id)
    if fluxo is None:
        return HttpResponse("Fluxo {} não encontrado".format(id))

    if destino in ['a', 'f']:
        filename = \
            'roteiros_alt{id}_{versao_num}_{versao_sufixo}' \
            '.dot'.format(**fluxo)
        templ = loader.get_template(fluxo['template_base'])
        http_resp = HttpResponse(
            templ.render(fluxo, request), content_type='text/plain')
        http_resp['Content-Disposition'] = \
            'attachment; filename="{filename}"'.format(filename=filename)
        return http_resp

    else:
        return render(
            request, fluxo['template_base'], fluxo,
            content_type='text/plain')

class ExecGeraFluxo(O2BaseGetPostView):
    def __init__(self, *args, **kwargs):
        super(ExecGeraFluxo, self).__init__(*args, **kwargs)
        self.Form_class = forms.ExecGeraFluxoForm
        self.template_name = 'geral/exec_gera_fluxo.html'
        self.title_name = 'Gera fluxo'
        self.get_args = ['destino', 'id']

    def mount_context(self):
        destino = self.form.cleaned_data['destino']
        id = self.form.cleaned_data['id']
        self.context.update({
            'destino': destino,
            'id': id,
        })
        fluxo = dict_fluxo(id)
        if fluxo is None:
            self.context.update({
                'erro': "Fluxo {} não encontrado".format(id),
            })

def roteiros_de_fluxo(request, id):
    roteiros = get_roteiros_de_fluxo(id)
    return HttpResponse(
        pformat(roteiros, indent=4),
        content_type='text/plain')

def unidade(request):
    cursor = db_cursor_so(request)
    data = queries.unidades(cursor)
    context = {
        'titulo': 'Unidades / Divisões',
        'headers': ('Código ', 'Descrição', 'UF', 'Cidade',
                    '(CNPJ) Razão social'),
        'fields': ('DIV', 'DESCR', 'UF', 'CIDADE',
                   'NOME'),
        'data': data,
    }
    return render(request, 'geral/unidade.html', context)

class Configuracao(PermissionRequiredMixin, View):
    def __init__(self, *args, **kwargs):
        self.permission_required = 'geral.change_config'
        self.Form_class = forms.ConfigForm
        self.template_name = 'geral/config.html'
        self.title_name = 'Configuração'

    def get_values(self, request):
        values = {}
        for field in self.Form_class.field_param:
            param = self.Form_class.field_param[field]
            if request.user.is_superuser:
                values[field] = config_get_value(param)
            else:
                values[field] = config_get_value(param, request.user)
        return values

    def set_values(self, request, values):
        if request.user.is_superuser:
            usuario = None
        else:
            usuario = request.user
        ok = True
        for field in self.Form_class.field_param:
            param = self.Form_class.field_param[field]
            ok = ok and config_set_value(param, values[field], usuario=usuario)
        return ok

    def get(self, request, *args, **kwargs):
        context = {'titulo': self.title_name}
        form = self.Form_class()
        values = self.get_values(request)
        form.fields["op_unidade"].initial = values['op_unidade']
        form.fields["dias_alem_lead"].initial = values['dias_alem_lead']
        context['form'] = form
        return render(request, self.template_name, context)

    def post(self, request, *args, **kwargs):
        context = {'titulo': self.title_name}
        form = self.Form_class(request.POST)
        if form.is_valid():
            values = {}
            values['op_unidade'] = form.cleaned_data['op_unidade']
            values['dias_alem_lead'] = form.cleaned_data['dias_alem_lead']
            if self.set_values(request, values):
                context['msg'] = 'Valores salvos!'
            else:
                context['msg'] = 'Houve algum erro ao salvar os valores!'
        context['form'] = form
        return render(request, self.template_name, context)
<|endoftext|>


<commit_before>"""
Author: BIT Community

Description:

bitbot is a simple slack greetings bot developed by the BIT community
to help promote opensource and BIT code collaboration.

"""
import time
from slackclient import SlackClient
from helpers import load_config

slack_settings = dict()
slack_settings['slack'] = ['slack_token', {'channels': 'introduction'}, 'bot_user', {'messages': 'greeting_message'}]

try:
    slack_config = load_config(get_settings=slack_settings)
except:
    print("Problem loading config settings")

if slack_config:
    slack_token = slack_config['slack_token']
    introduction_channel_id = slack_config['introduction']
    bot_user = slack_config['bot_user']
    greeting_message = slack_config['greeting_message']
else:
    exit()

def slack_client_connect():
    try:
        slack_client = SlackClient(slack_token)
    except:
        slack_client = False
    return slack_client

def slack_client_test():
     slack_client = slack_client_connect()
     print(slack_client.api_call('api.test'))
     print(slack_client.api_call('auth.test'))

def slack_post_message(message=None):
    slack_client = slack_client_connect()
    if message:
        post = slack_client.api_call("chat.postMessage", as_user="true:",
                                     channel=introduction_channel_id,
                                     text=message)
    else:
        post = "There was no message to send."
    return post

def slack_event_parser(output):
    output_list = output
    if output_list and len(output_list) > 0:
        for output in output_list:
            #print(output['text'], output['channel'])
            return output

#slack_client_test()

if __name__ == "__main__":
    slack_client = slack_client_connect()
    socket_delay = 1
    if slack_client.rtm_connect():
        print("BITBOT is connected to Slack")
        while True:
            output = slack_event_parser(slack_client.rtm_read())
            if output['type'] == 'team_join':
                slack_post_message(message=greeting_message)
            time.sleep(socket_delay)
    else:
        print("Could not connect to slack. Check your token or network connection.")
<commit_msg>updated main check for output type<commit_after>"""
Author: BIT Community

Description:

bitbot is a simple slack greetings bot developed by the BIT community
to help promote opensource and BIT code collaboration.

"""
import time
import sys
from slackclient import SlackClient
from helpers import load_config

slack_settings = dict()
slack_settings['slack'] = ['slack_token', {'channels': 'introduction'}, 'bot_user', {'messages': 'greeting_message'}]

try:
    slack_config = load_config(get_settings=slack_settings)
except:
    print("Problem loading config settings")

if slack_config:
    slack_token = slack_config['slack_token']
    introduction_channel_id = slack_config['introduction']
    bot_user = slack_config['bot_user']
    greeting_message = slack_config['greeting_message']
else:
    sys.exit("Couldn't set variables")

def slack_client_connect():
    try:
        slack_client = SlackClient(slack_token)
    except:
        slack_client = False
    return slack_client

def slack_client_test():
     slack_client = slack_client_connect()
     print(slack_client.api_call('api.test'))
     print(slack_client.api_call('auth.test'))

def slack_post_message(message=None):
    slack_client = slack_client_connect()
    if message:
        post = slack_client.api_call("chat.postMessage", as_user="true:",
                                     channel=introduction_channel_id,
                                     text=message)
    else:
        post = "There was no message to send."
    return post

def slack_event_parser(output):
    output_list = output
    if output_list and len(output_list) > 0:
        for output in output_list:
            #print(output['text'], output['channel'])
            return output

#slack_client_test()

if __name__ == "__main__":
    slack_client = slack_client_connect()
    socket_delay = 1
    if slack_client.rtm_connect():
        print("BITBOT is connected to Slack")
        while True:
            output = slack_event_parser(slack_client.rtm_read())
            if output:
                if output['type'] == 'team_join':
                    slack_post_message(message=greeting_message)
            time.sleep(socket_delay)
    else:
        print("Could not connect to slack. Check your token or network connection.")
<|endoftext|>


<commit_before>#! /usr/bin/env python

import argparse

# First off, read the arguments
parser = argparse.ArgumentParser(description='Run forgetful workers on contrived jobs.')

parser.add_argument('--forgetfulness', dest='forgetfulness', default=0.1, type=float,
    help='What portion of jobs should be randomly dropped by workers')
parser.add_argument('--jobs', dest='numJobs', default=1000, type=int,
    help='How many jobs to schedule for the test')
parser.add_argument('--workers', dest='numWorkers', default=10, type=int,
    help='How many workers should do the work')
parser.add_argument('--quiet', dest='verbose', default=True, action='store_false',
    help='Reduce all the output')

args = parser.parse_args()

import time
import qless
import random
import logging
import threading

logger = logging.getLogger('qless-bench')
formatter = logging.Formatter('[%(asctime)s] %(threadName)s => %(message)s')
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
logger.addHandler(handler)
if args.verbose:
    logger.setLevel(logging.DEBUG)
else:
    logger.setLevel(logging.WARN)

class ForgetfulWorker(threading.Thread):
    def __init__(self, *args, **kwargs):
        threading.Thread.__init__(self, *args, **kwargs)
        self.q = qless.Queue('testing')
        self.q._hb = 1
        self.q.worker += '-' + self.getName()

    def run(self):
        while len(self.q):
            job = self.q.pop()
            if not job:
                # Sleep a little bit
                time.sleep(0.1)
                logger.debug('No jobs available. Sleeping.')
                continue
            # Randomly drop a job?
            if random.random() < args.forgetfulness:
                logger.debug('Randomly dropping job!')
                continue
            else:
                logger.debug('Completing job!')
                self.q.complete(job)

# Make sure that the redis instance is empty first
if len(qless.r.keys('*')):
    print 'Must begin on an empty Redis instance'
    exit(1)

# This is how much CPU Redis had used /before/
cpuBefore = qless.r.info()['used_cpu_user'] + qless.r.info()['used_cpu_sys']
# This is how long it took to add the jobs
putTime = -time.time()
# Alright, let's make a bunch of jobs
jids = [qless.Job.put('testing', {'test': 'benchmark', 'count': c}) for c in range(args.numJobs)]
putTime += time.time()

# This is how long it took to run the workers
workTime = -time.time()
# And now let's make some workers to deal with 'em!
workers = [ForgetfulWorker() for i in range(args.numWorkers)]
for worker in workers:
    worker.start()

for worker in workers:
    worker.join()

workTime += time.time()

import unittest
class TestQlessBench(unittest.TestCase):
    def test_stats(self):
        # In this test, we want to make sure that the stats we get out of
        # the system seem in accordance with what we'd expect. Ensure:
        #   1) Completed all of the jobs
        #   2) Non-zero mean, non-zero std deviation
        #   3) Wait stats for all jobs
        #   4) Non-zero
        stats = qless.Stats.get('testing', time.time())
        self.assertEqual(stats['run']['count'], args.numJobs)
        self.assertTrue( stats['run']['mean' ], 0)
        self.assertTrue( stats['run']['std'  ], 0)
        # We may want to ensure that it's within certain expectation
        # values based on the failure rate
        self.assertTrue(stats['wait']['count'] > args.numJobs)
        self.assertTrue(stats['wait']['mean' ] > 0)
        self.assertTrue(stats['wait']['std'  ] > 0)

# This needs some work
#unittest.main(exit=False)

def histo(l):
    count = sum(l)
    l = list(o for o in l if o)
    for i in range(len(l)):
        print '\t\t%2i, %10.9f, %i' % (i, float(l[i]) / count, l[i])

# Now we'll print out some interesting stats
stats = qless.Stats.get('testing', time.time())
print 'Wait:'
print '\tCount: %i'  % stats['wait']['count']
print '\tMean : %fs' % stats['wait']['mean']
print '\tSDev : %f'  % stats['wait']['std']
print '\tHist :'
histo(stats['wait']['histogram'])

print 'Run:'
print '\tCount: %i'  % stats['run']['count']
print '\tMean : %fs' % stats['run']['mean']
print '\tSDev : %f'  % stats['run']['std']
histo(stats['run']['histogram'])

print '=' * 50
print 'Put jobs : %fs' % putTime
print 'Do jobs  : %fs' % workTime
info = qless.r.info()
print 'Redis Mem: %s'  % info['used_memory_human']
print 'Redis Lua: %s'  % info['used_memory_lua']
print 'Redis CPU: %fs' % (info['used_cpu_user'] + info['used_cpu_sys'] - cpuBefore)

# Flush the database when we're done
print 'Flushing'
qless.r.flushdb()<commit_msg>Updated the forgetful bench to reflect the new interface.<commit_after>#! /usr/bin/env python

import argparse

# First off, read the arguments
parser = argparse.ArgumentParser(description='Run forgetful workers on contrived jobs.')

parser.add_argument('--forgetfulness', dest='forgetfulness', default=0.1, type=float,
    help='What portion of jobs should be randomly dropped by workers')
parser.add_argument('--jobs', dest='numJobs', default=1000, type=int,
    help='How many jobs to schedule for the test')
parser.add_argument('--workers', dest='numWorkers', default=10, type=int,
    help='How many workers should do the work')
parser.add_argument('--quiet', dest='verbose', default=True, action='store_false',
    help='Reduce all the output')

args = parser.parse_args()

import time
import qless
import random
import logging
import threading

logger = logging.getLogger('qless-bench')
formatter = logging.Formatter('[%(asctime)s] %(threadName)s => %(message)s')
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
logger.addHandler(handler)
if args.verbose:
    logger.setLevel(logging.DEBUG)
else:
    logger.setLevel(logging.WARN)

# Our qless client
client = qless.client()

class ForgetfulWorker(threading.Thread):
    def __init__(self, *args, **kwargs):
        threading.Thread.__init__(self, *args, **kwargs)
        self.q = client.queue('testing')
        self.q._hb = 1
        self.q.worker += '-' + self.getName()

    def run(self):
        while len(self.q):
            job = self.q.pop()
            if not job:
                # Sleep a little bit
                time.sleep(0.1)
                logger.debug('No jobs available. Sleeping.')
                continue
            # Randomly drop a job?
            if random.random() < args.forgetfulness:
                logger.debug('Randomly dropping job!')
                continue
            else:
                logger.debug('Completing job!')
                self.q.complete(job)

# Make sure that the redis instance is empty first
if len(client.redis.keys('*')):
    print 'Must begin on an empty Redis instance'
    exit(1)

# This is how much CPU Redis had used /before/
cpuBefore = client.redis.info()['used_cpu_user'] + client.redis.info()['used_cpu_sys']
# This is how long it took to add the jobs
putTime = -time.time()
# Alright, let's make a bunch of jobs
testing = client.queue('testing')
jids = [testing.put({'test': 'benchmark', 'count': c}) for c in range(args.numJobs)]
putTime += time.time()

# This is how long it took to run the workers
workTime = -time.time()
# And now let's make some workers to deal with 'em!
workers = [ForgetfulWorker() for i in range(args.numWorkers)]
for worker in workers:
    worker.start()

for worker in workers:
    worker.join()

workTime += time.time()

import unittest
class TestQlessBench(unittest.TestCase):
    def test_stats(self):
        # In this test, we want to make sure that the stats we get out of
        # the system seem in accordance with what we'd expect. Ensure:
        #   1) Completed all of the jobs
        #   2) Non-zero mean, non-zero std deviation
        #   3) Wait stats for all jobs
        #   4) Non-zero
        stats = client.stats.get('testing', time.time())
        self.assertEqual(stats['run']['count'], args.numJobs)
        self.assertTrue( stats['run']['mean' ], 0)
        self.assertTrue( stats['run']['std'  ], 0)
        # We may want to ensure that it's within certain expectation
        # values based on the failure rate
        self.assertTrue(stats['wait']['count'] > args.numJobs)
        self.assertTrue(stats['wait']['mean' ] > 0)
        self.assertTrue(stats['wait']['std'  ] > 0)

# This needs some work
#unittest.main(exit=False)

def histo(l):
    count = sum(l)
    l = list(o for o in l if o)
    for i in range(len(l)):
        print '\t\t%2i, %10.9f, %i' % (i, float(l[i]) / count, l[i])

# Now we'll print out some interesting stats
stats = client.stats.get('testing', time.time())
print 'Wait:'
print '\tCount: %i'  % stats['wait']['count']
print '\tMean : %fs' % stats['wait']['mean']
print '\tSDev : %f'  % stats['wait']['std']
print '\tWait Time Histogram:'
histo(stats['wait']['histogram'])

print 'Run:'
print '\tCount: %i'  % stats['run']['count']
print '\tMean : %fs' % stats['run']['mean']
print '\tSDev : %f'  % stats['run']['std']
print '\tCompletion Time Histogram:'
histo(stats['run']['histogram'])

print '=' * 50
print 'Put jobs : %fs' % putTime
print 'Do jobs  : %fs' % workTime
info = client.redis.info()
print 'Redis Mem: %s'  % info['used_memory_human']
print 'Redis Lua: %s'  % info['used_memory_lua']
print 'Redis CPU: %fs' % (info['used_cpu_user'] + info['used_cpu_sys'] - cpuBefore)

# Flush the database when we're done
print 'Flushing'
client.redis.flushdb()<|endoftext|>


<commit_before># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Test the design_matrix utilities.

Note that the tests just looks whether the data produces has correct dimension,
not whether it is exact
"""

import numpy as np
from nipy.neurospin.graph.BPmatch import *
from numpy.testing import assert_almost_equal

def test_match_trivial_0():
    c1 = np.array([[0],[1],[2]])
    c2 = c1+0.7
    dmax = 1.0
    i, j, k = match_trivial(c1, c2, dmax)
    assert(len(i)==9)

def test_match_trivial_1():
    c1 = np.array([[0],[1],[2]])
    c2 = c1+0.7
    dmax = 1.0
    i, j, k = match_trivial(c1, c2, dmax)
    a = np.zeros((3,3))
    a[i,j]=k
    assert_almost_equal(a.sum(1),[1,1,1])

def test_match_trivial_2():
    c1 = np.array([[0],[1],[2]])
    c2 = c1+0.4
    dmax = 1.0
    i, j, k = match_trivial(c1, c2, dmax)
    a = np.zeros((3,3))
    a[i,j] = k
    assert_almost_equal(a.argmax(1),[0,1,2])

def test_match_trivial_3():
    c1 = np.array([[0],[1],[2]])
    c2 = c1+0.6
    dmax = 1.0
    i, j, k = match_trivial(c1, c2, dmax)
    a = np.zeros((3,3))
    a[i,j] = k
    assert_almost_equal(a.argmax(1),[0,0,1])

def test_match_2():
    c1 = np.array([[0],[1],[2]])
    c2 = c1+0.6
    adjacency = np.ones((3,3))-np.eye(3)
    adjacency[2,0] = 0
    adjacency[0,2] = 0
    dmax = 1.0
    i, j , k = BPmatch(c1, c2, adjacency, dmax)
    a = np.zeros((3,3))
    a[i,j] = k
    assert_almost_equal(a.argmax(1),[0,1,2])

if __name__ == "__main__":
    import nose
    nose.run(argv=['', __file__])

<commit_msg>fixed a small broken test<commit_after># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Test the design_matrix utilities.

Note that the tests just looks whether the data produces has correct dimension,
not whether it is exact
"""

import numpy as np
from nipy.neurospin.graph.BPmatch import *
from numpy.testing import assert_almost_equal

def test_match_trivial_0():
    c1 = np.array([[0],[1],[2]])
    c2 = c1+0.7
    dmax = 1.0
    i, j, k = match_trivial(c1, c2, dmax)
    assert(len(i)==9)

def test_match_trivial_1():
    c1 = np.array([[0],[1],[2]])
    c2 = c1+0.7
    dmax = 1.0
    i, j, k = match_trivial(c1, c2, dmax)
    a = np.zeros((3,3))
    a[i,j]=k
    assert_almost_equal(a.sum(1),[1,1,1])

def test_match_trivial_2():
    c1 = np.array([[0],[1],[2]]).astype(np.float)
    c2 = c1+0.4
    dmax = 1.0
    i, j, k = match_trivial(c1, c2, dmax)
    a = np.zeros((3,3))
    a[i,j] = k
    assert_almost_equal(a.argmax(1),[0,1,2])

def test_match_trivial_3():
    c1 = np.array([[0],[1],[2]])
    c2 = c1+0.6
    dmax = 1.0
    i, j, k = match_trivial(c1, c2, dmax)
    a = np.zeros((3,3))
    a[i,j] = k
    assert_almost_equal(a.argmax(1),[0,0,1])

def test_match_2():
    c1 = np.array([[0],[1],[2]])
    c2 = c1+0.6
    adjacency = np.ones((3,3))-np.eye(3)
    adjacency[2,0] = 0
    adjacency[0,2] = 0
    dmax = 1.0
    i, j , k = BPmatch(c1, c2, adjacency, dmax)
    a = np.zeros((3,3))
    a[i,j] = k
    assert_almost_equal(a.argmax(1),[0,1,2])

if __name__ == "__main__":
    import nose
    nose.run(argv=['', __file__])

<|endoftext|>