<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Per ogni Lug indicato nella LugMap effettuo un insieme di controlli di validità.
   Se qualcosa non torna, avverto chi di dovere"""

if True: # import dei moduli
	try:
		import csv, glob, os, socket, sys, smtplib, syslog, urllib2
	except:
		import sys
		print "Non sono disponibili tutti i moduli standard necessari."
		sys.exit(-1)
	try:
		import ZODB, persistent, transaction
	except:
		print "Non sono disponibili uno o più di questi moduli: ZODB, persistent, transaction"

if True: # attiva DB
	from ZODB.FileStorage import FileStorage
	from ZODB.DB import DB
	storage = FileStorage(os.path.join(os.environ["HOME"], '.spazzino.db'))
	db = DB(storage)
	connection = db.open()
	pdb = connection.root()

class Lug(persistent.Persistent):
	def __init__(self, url_del_lug):
		self.url = url_del_lug
		self.email_errori = email_report()
		self.dominio = url_del_lug.split('/')[2]
		self.Termini_Precedenti = set()
		self.DNS_noti = set()
		self.numero_controlli = 0
		self.numero_errori = 0
		socket.setdefaulttimeout(35) # Timeout in secondi del fetching delle pagine (vedi urllib2)

	def controllo_dns(self):
		"""Controllo l'esistenza e la mappatura del dominio"""

		print "Controllo dominio",self.dominio
		self.numero_controlli += 1
		try:
			DNS_attuale = socket.getaddrinfo(self.dominio, 80, 0, 0, socket.SOL_TCP)[0][4][0]
		except:
			self.email_errori.aggiungi("      Errore: problema sul dominio (esistenza/mappatura)")
			self.numero_errori += 1
			return False

		if DNS_attuale not in self.DNS_noti:
			dettaglio = 'Attenzione: nuovo IP %s (%s)' % (DNS_attuale,' '.join(self.DNS_noti))
			self.DNS_noti.add(DNS_attuale)
			self.email_errori.aggiungi(dettaglio)
			self.numero_errori += 1
			return False
		return True

	def controllo_contenuto(self):
		"""Leggo lo URL e faccio una valutazione numerica. True/False di ritorno."""

		try: # pesco la pagina
			richiesta = urllib2.Request(self.url,None, {"User-Agent":"LugMap.it checker - lugmap@linux.it"})
			pagina_html = urllib2.urlopen(richiesta).read()
		except:
			self.email_errori.aggiungi('       Errore: impossibile leggere la pagina html.')
			self.numero_errori += 1
			return False

		self.Termini_Attuali = set(pagina_html.split()) # Estrapolo i termini presenti
		valore_magico = \
		  float(len(self.Termini_Precedenti.intersection(self.Termini_Attuali))*1.0/len(self.Termini_Precedenti.union(self.Termini_Attuali)))
		self.Termini_Precedenti = self.Termini_Attuali
		del self.Termini_Attuali

		if valore_magico <= 0.7:
			self.email_errori.aggiungi('      Errore: troppa differenza di contenuto:' +str(valore_magico))
			self.numero_errori += 1
			return False

	def invia_report(self):
		self.email_errori.subject = 'LugMap: '+self.url
		self.email_errori.invia()
		del self.email_errori
		self.email_errori = email_report()

class email_report():
	"""Prendo in pasto errori e li invio via SMTP"""

	def __init__(self):
		"""Definisco dettagli email"""

		self.mittente = 'lugmapcheck@gelma.net'
		self.destinatario = ['andrea.gelmini@gmail.com']
		self.righe = []
		self.subject = 'Lugmap:'

	def aggiungi(self,testo):
		"""ACcetto un argomento che metto nel corpo email"""

		if type(testo) == tuple or type(testo) == list:
			[self.righe.append(riga) for riga in testo]
		elif type(testo) == str:
			self.righe.append(testo)
		else:
			try:
				self.righe.append(str(testo))
			except:
				print "Il cast non ha funzionato"
				# Raise exception

	def invia(self):
		"""Effettuo l'invio vero e proprio"""

		if not self.righe: return # Se non ho alcun testo di errore, non proseguo

		self.msg = ("From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n" % (self.mittente, ", ".join(self.destinatario), self.subject))
		self.msg = self.msg + '\n'.join(self.righe) + '\n' + '\n'.join(riga)
		if False: # abilita/disabilita invio della mail, in luogo del report a video
			print 30*'-'+'\n'+self.msg+'\n'+30*'-'
		else:
			try:
				server = smtplib.SMTP('localhost')
				server.sendmail(self.mittente, self.destinatario, self.msg)
				server.quit()
			except:
				print "Non è stato possibile inviare la mail"

			syslog.syslog(syslog.LOG_ERR, 'Spazzino: '+self.subject+' '+'  '.join(self.righe))

if __name__ == "__main__":
	for filedb in glob.glob( os.path.join('./db/', '*.txt') ): # piglio ogni file db
		for riga in csv.reader(open(filedb, "r"), delimiter='|', quoting=csv.QUOTE_NONE): # e per ogni riga/Lug indicato
			url = riga[3]
			if pdb.has_key(url): # se è gia' presente nel DB
				lug = pdb[url] # la recupero
			else:
				lug = Lug(url) # diversamente creo la classe
				pdb[url] = lug # e la lego al DB
			if lug.controllo_dns():
				lug.controllo_contenuto()
			lug.invia_report()
transaction.commit()
db.pack()
db.close()
<commit_msg>Spazzino: reso piu chiaro commento (forse!)<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Per ogni Lug indicato nella LugMap effettuo un insieme di controlli di validità.
   Se qualcosa non torna, avverto chi di dovere"""

if True: # import dei moduli
	try:
		import csv, glob, os, socket, sys, smtplib, syslog, urllib2
	except:
		import sys
		print "Non sono disponibili tutti i moduli standard necessari."
		sys.exit(-1)
	try:
		import ZODB, persistent, transaction
	except:
		print "Non sono disponibili uno o più di questi moduli: ZODB, persistent, transaction"

if True: # attiva DB
	from ZODB.FileStorage import FileStorage
	from ZODB.DB import DB
	storage = FileStorage(os.path.join(os.environ["HOME"], '.spazzino.db'))
	db = DB(storage)
	connection = db.open()
	pdb = connection.root()

class Lug(persistent.Persistent):
	def __init__(self, url_del_lug):
		self.url = url_del_lug
		self.email_errori = email_report()
		self.dominio = url_del_lug.split('/')[2]
		self.Termini_Precedenti = set()
		self.DNS_noti = set()
		self.numero_controlli = 0
		self.numero_errori = 0
		socket.setdefaulttimeout(35) # Timeout in secondi del fetching delle pagine (vedi urllib2)

	def controllo_dns(self):
		"""Controllo l'esistenza e la mappatura del dominio"""

		print "Controllo dominio",self.dominio
		self.numero_controlli += 1
		try:
			DNS_attuale = socket.getaddrinfo(self.dominio, 80, 0, 0, socket.SOL_TCP)[0][4][0]
		except:
			self.email_errori.aggiungi("      Errore: problema sul dominio (esistenza/mappatura)")
			self.numero_errori += 1
			return False

		if DNS_attuale not in self.DNS_noti:
			dettaglio = 'Attenzione: nuovo IP %s (%s)' % (DNS_attuale,' '.join(self.DNS_noti))
			self.DNS_noti.add(DNS_attuale)
			self.email_errori.aggiungi(dettaglio)
			self.numero_errori += 1
			return False
		return True

	def controllo_contenuto(self):
		"""Leggo lo URL e faccio una valutazione numerica. True/False di ritorno."""

		try: # pesco la pagina
			richiesta = urllib2.Request(self.url,None, {"User-Agent":"LugMap.it checker - lugmap@linux.it"})
			pagina_html = urllib2.urlopen(richiesta).read()
		except:
			self.email_errori.aggiungi('       Errore: impossibile leggere la pagina html.')
			self.numero_errori += 1
			return False

		self.Termini_Attuali = set(pagina_html.split()) # Estrapolo le parole della pagina HTML
		valore_magico = \
		  float(len(self.Termini_Precedenti.intersection(self.Termini_Attuali))*1.0/len(self.Termini_Precedenti.union(self.Termini_Attuali)))
		self.Termini_Precedenti = self.Termini_Attuali
		del self.Termini_Attuali

		if valore_magico <= 0.7:
			self.email_errori.aggiungi('      Errore: troppa differenza di contenuto:' +str(valore_magico))
			self.numero_errori += 1
			return False

	def invia_report(self):
		self.email_errori.subject = 'LugMap: '+self.url
		self.email_errori.invia()
		del self.email_errori
		self.email_errori = email_report()

class email_report():
	"""Prendo in pasto errori e li invio via SMTP"""

	def __init__(self):
		"""Definisco dettagli email"""

		self.mittente = 'lugmapcheck@gelma.net'
		self.destinatario = ['andrea.gelmini@gmail.com']
		self.righe = []
		self.subject = 'Lugmap:'

	def aggiungi(self,testo):
		"""ACcetto un argomento che metto nel corpo email"""

		if type(testo) == tuple or type(testo) == list:
			[self.righe.append(riga) for riga in testo]
		elif type(testo) == str:
			self.righe.append(testo)
		else:
			try:
				self.righe.append(str(testo))
			except:
				print "Il cast non ha funzionato"
				# Raise exception

	def invia(self):
		"""Effettuo l'invio vero e proprio"""

		if not self.righe: return # Se non ho alcun testo di errore, non proseguo

		self.msg = ("From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n" % (self.mittente, ", ".join(self.destinatario), self.subject))
		self.msg = self.msg + '\n'.join(self.righe) + '\n' + '\n'.join(riga)
		if False: # abilita/disabilita invio della mail, in luogo del report a video
			print 30*'-'+'\n'+self.msg+'\n'+30*'-'
		else:
			try:
				server = smtplib.SMTP('localhost')
				server.sendmail(self.mittente, self.destinatario, self.msg)
				server.quit()
			except:
				print "Non è stato possibile inviare la mail"

			syslog.syslog(syslog.LOG_ERR, 'Spazzino: '+self.subject+' '+'  '.join(self.righe))

if __name__ == "__main__":
	for filedb in glob.glob( os.path.join('./db/', '*.txt') ): # piglio ogni file db
		for riga in csv.reader(open(filedb, "r"), delimiter='|', quoting=csv.QUOTE_NONE): # e per ogni riga/Lug indicato
			url = riga[3]
			if pdb.has_key(url): # se è gia' presente nel DB
				lug = pdb[url] # la recupero
			else:
				lug = Lug(url) # diversamente creo la classe
				pdb[url] = lug # e la lego al DB
			if lug.controllo_dns():
				lug.controllo_contenuto()
			lug.invia_report()
transaction.commit()
db.pack()
db.close()
<|endoftext|>


<commit_before>'''
'''

import sys
import os
import ctypes

class omp(object):
    LD_LIBRARY_PATHS = (
            "/usr/lib/x86_64-linux-gnu/",
            )

    def __init__(self):
        try:
            self.libomp = ctypes.CDLL("libgomp.so.1")
        except OSError:
            old_path = os.environ.get("LD_LIBRARY_PATH", "")
            os.environ["LD_LIBRARY_PATH"] = ":".join(omp.LD_LIBRARY_PATHS)
            self.libomp = ctypes.CDLL("libgomp.so.1")
            os.environ["LD_LIBRARY_PATH"] = old_path

    def __getattribute__(self, name):
        if name == 'libomp':
            return object.__getattribute__(self, 'libomp')
        else:
            return getattr(self.libomp, 'omp_' + name)

# see http://mail.python.org/pipermail/python-ideas/2012-May/014969.html
sys.modules[__name__] = omp()
<commit_msg>OSX portability: fix libgomp dynamic library loading<commit_after>'''
OpenMP wrapper using a (user provided) libgomp dynamically loaded library
'''

import sys
import glob
import ctypes

class omp(object):
    LD_LIBRARY_PATHS = [
        "/usr/lib/x86_64-linux-gnu/",
        # MacPorts install gcc in a "non standard" path on OSX
    ] + glob.glob("/opt/local/lib/gcc*/")

    def __init__(self):
        # Paths are "non-standard" place to lookup
        paths = omp.LD_LIBRARY_PATHS

        # Try to load find libgomp shared library using loader search dirs
        libgomp_path = ctypes.util.find_library("libgomp")

        # Try to use custom paths if lookup failed
        for path in paths:
            if libgomp_path:
                break
            libgomp_path = ctypes.util.find_library(path+"libgomp")

        if not libgomp_path:
            raise EnvironmentError("I can't find a shared library for libgomp,"
                                   " you may need to install it or adjust the "
                                   "LD_LIBRARY_PATH environment variable.")
        else:
            # Load the library (shouldn't fail with an absolute path right?)
            self.libomp = ctypes.CDLL(libgomp_path)

    def __getattribute__(self, name):
        if name == 'libomp':
            return object.__getattribute__(self, 'libomp')
        else:
            return getattr(self.libomp, 'omp_' + name)

# see http://mail.python.org/pipermail/python-ideas/2012-May/014969.html
sys.modules[__name__] = omp()
<|endoftext|>


<commit_before>import os, gc, urllib
import numpy as N

from neuroimaging.core.image.image import Image
from neuroimaging.algorithms.interpolation import ImageInterpolator
from neuroimaging.algorithms.onesample import ImageOneSample
from neuroimaging.core.reference.mapping import Affine
from neuroimaging.core.reference.grid import SamplingGrid

from fiac import FIACblock, FIACevent, FIACpath

def FIACfixed(contrast, which='block', subj=3, base='contrasts', clobber=False):
    keep = []

    pdir = {'block':FIACblock,
            'event':FIACevent}

    outdir = FIACpath('fixed/%s/%s/%s' % (which, base, contrast), run=-1, subj=subj)

    if not os.path.exists(outdir):
        os.makedirs(outdir)

    for run in range(1, 5):
        try:
            p = pdir[which](subj=subj, run=run)
            keep.append(run)
        except:
            pass

    input = []

    for run in keep:

        matfile = FIACpath('fsl/example_func2standard.mat', subj=subj, run=run)
        if os.path.exists(matfile):

            sdfile = FIACpath('fsl/fmristat_run/%s/%s/sd.img' % (base, contrast), subj=subj, run=run)
            outfile = os.path.join(outdir, 'sd_%d.img' % run)
            FIACresample(sdfile, outfile, subj=subj, run=run)
            sdimg = fix_origin(Image(outfile))

            efffile = FIACpath('fsl/fmristat_run/%s/%s/effect.img' % (base, contrast), subj=subj, run=run)
            outfile = os.path.join(outdir, 'effect_%d.hdr' % run)
            FIACresample(efffile, outfile, subj=subj, run=run)
            effimg = fix_origin(Image(outfile))
            input.append((effimg, sdimg))

    if input:
        fitter = ImageOneSample(input, path=outdir, clobber=clobber, all=True, use_scale=False, weight_type='sd')
        fitter.fit()

def fix_origin(img):

    atlas = Image('/home/stow/fsl/etc/standard/avg152T1_brain.img')
    img.image.origin = atlas.image.origin
    img.image.pixdim = atlas.image.pixdim

    hdrfile = file(img.image.hdrfilename(), 'w')
    img.image.writeheader(hdrfile)
    hdrfile.close()
    return Image(img.image.filename)

def FIACresample(infile, outfile, subj=3, run=3, **other):

    inimage = Image(infile, ignore_origin=True, abs_pixdim=True)
    fslmatstr = urllib.urlopen('%s' % FIACpath('fsl/example_func2standard.mat',
                                                       subj=subj, run=run)).read().split()

    fslmat =  N.array(map(float, fslmatstr))
    fslmat.shape = (4,4)

    M = [[0,0,1,0],
         [0,1,0,0],
         [1,0,0,0],
         [0,0,0,1]]

    fslmat = N.dot(M, N.dot(fslmat, M))

    standard = Image('http://kff.stanford.edu/FIAC/avg152T1_brain.img',
                                        ignore_origin=True,
                                        abs_pixdim=True)

    input_coords = inimage.grid.output_coords
    output_coords = standard.grid.output_coords

    fworld2sworld = Affine(fslmat)

    voxel2fworld = fworld2sworld.inverse() * standard.grid.mapping

    output_grid = SamplingGrid(standard.grid.shape, voxel2fworld, input_coords, output_coords)

    interp = ImageInterpolator(inimage)
    interp_data = interp(output_grid.range())

    new = Image(interp_data, grid=standard.grid)

    new.tofile(outfile, clobber=True)

    del(interp); del(interp_data); del(new) ; gc.collect()

def FIACrun(subj=3, clobber=True):
    for base in ['delays', 'contrasts']:
        for which in ['block', 'event']:
            for contrast in ['overall', 'speaker', 'sentence', 'interaction']:
                FIACfixed(contrast, base=base, which=which, subj=subj, clobber=True)
                print base, which, contrast, 'done'

if __name__ == '__main__':

    import sys
    subj = int(sys.argv[1])

    FIACrun(subj=subj)
<commit_msg>more explicit exception catching<commit_after>import os, gc, urllib
import numpy as N

from neuroimaging.core.image.image import Image
from neuroimaging.algorithms.interpolation import ImageInterpolator
from neuroimaging.algorithms.onesample import ImageOneSample
from neuroimaging.core.reference.mapping import Affine
from neuroimaging.core.reference.grid import SamplingGrid

from fiac import FIACblock, FIACevent, FIACpath

def FIACfixed(contrast, which='block', subj=3, base='contrasts', clobber=False):
    keep = []

    pdir = {'block':FIACblock,
            'event':FIACevent}

    outdir = FIACpath('fixed/%s/%s/%s' % (which, base, contrast), run=-1, subj=subj)

    if not os.path.exists(outdir):
        os.makedirs(outdir)

    for run in range(1, 5):
        try:
            p = pdir[which](subj=subj, run=run)
            keep.append(run)
        except IOError:
            pass

    input = []

    for run in keep:

        matfile = FIACpath('fsl/example_func2standard.mat', subj=subj, run=run)
        if os.path.exists(matfile):

            sdfile = FIACpath('fsl/fmristat_run/%s/%s/sd.img' % (base, contrast), subj=subj, run=run)
            outfile = os.path.join(outdir, 'sd_%d.img' % run)
            FIACresample(sdfile, outfile, subj=subj, run=run)
            sdimg = fix_origin(Image(outfile))

            efffile = FIACpath('fsl/fmristat_run/%s/%s/effect.img' % (base, contrast), subj=subj, run=run)
            outfile = os.path.join(outdir, 'effect_%d.hdr' % run)
            FIACresample(efffile, outfile, subj=subj, run=run)
            effimg = fix_origin(Image(outfile))
            input.append((effimg, sdimg))

    if input:
        fitter = ImageOneSample(input, path=outdir, clobber=clobber, all=True, use_scale=False, weight_type='sd')
        fitter.fit()

def fix_origin(img):

    atlas = Image('/home/stow/fsl/etc/standard/avg152T1_brain.img')
    img.image.origin = atlas.image.origin
    img.image.pixdim = atlas.image.pixdim

    hdrfile = file(img.image.hdrfilename(), 'w')
    img.image.writeheader(hdrfile)
    hdrfile.close()
    return Image(img.image.filename)

def FIACresample(infile, outfile, subj=3, run=3, **other):

    inimage = Image(infile, ignore_origin=True, abs_pixdim=True)
    fslmatstr = urllib.urlopen('%s' % FIACpath('fsl/example_func2standard.mat',
                                                       subj=subj, run=run)).read().split()

    fslmat =  N.array(map(float, fslmatstr))
    fslmat.shape = (4,4)

    M = [[0,0,1,0],
         [0,1,0,0],
         [1,0,0,0],
         [0,0,0,1]]

    fslmat = N.dot(M, N.dot(fslmat, M))

    standard = Image('http://kff.stanford.edu/FIAC/avg152T1_brain.img',
                                        ignore_origin=True,
                                        abs_pixdim=True)

    input_coords = inimage.grid.output_coords
    output_coords = standard.grid.output_coords

    fworld2sworld = Affine(fslmat)

    voxel2fworld = fworld2sworld.inverse() * standard.grid.mapping

    output_grid = SamplingGrid(standard.grid.shape, voxel2fworld, input_coords, output_coords)

    interp = ImageInterpolator(inimage)
    interp_data = interp(output_grid.range())

    new = Image(interp_data, grid=standard.grid)

    new.tofile(outfile, clobber=True)

    del(interp); del(interp_data); del(new) ; gc.collect()

def FIACrun(subj=3, clobber=True):
    for base in ['delays', 'contrasts']:
        for which in ['block', 'event']:
            for contrast in ['overall', 'speaker', 'sentence', 'interaction']:
                FIACfixed(contrast, base=base, which=which, subj=subj, clobber=True)
                print base, which, contrast, 'done'

if __name__ == '__main__':

    import sys
    subj = int(sys.argv[1])

    FIACrun(subj=subj)
<|endoftext|>


<commit_before><commit_msg>fix string formatting<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import click
import requests
import webbrowser
from lxml import etree

from . import writers

TRENDING_REPO_URL = 'http://github.com/trending'
TRENDING_DEV_URL = 'http://github.com/trending/developers'
HOME_PAGE = 'https://github.com'

requests.packages.urllib3.disable_warnings()

def replace_new_lines_and_strip(s):
    return s.strip()

def replace_new_lines_and_multiple_spaces(s):
    return ' '.join(s.replace('\n', '').split())

def read_page(url, timeout=5):

    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:11.0) Gecko/20100101 Firefox/11.0'}
    try:
        response = requests.get(url, timeout=timeout, headers=headers)
    except requests.exceptions.ConnectionError as e:  # noqa
        return(None, False)

    return(response, response.status_code)

def make_etree(url):
    response, status_code = read_page(url)
    if status_code == 200:
        response = etree.HTML(response.text)
    return (response, status_code)

def get_trending_repo_names(tree):
    repos = tree.xpath('//h3[@class="repo-list-name"]/a/@href')
    return repos

def get_trending_repo_description(tree):
    repo_desc = tree.xpath('//p[@class="repo-list-description"]')
    repo_desc = [" ".join([x for x in each.itertext()]) for each in repo_desc]
    repo_desc = [replace_new_lines_and_strip(each) for each in repo_desc]
    return repo_desc

def get_trending_repo_meta(tree):
    repo_meta = tree.xpath('//p[@class="repo-list-meta"]')
    return repo_meta

def get_trending_repo_stars_and_languages(repo_meta):
    dot = u'•'.encode('utf8')
    repo_stars_and_langauges = []
    for each in repo_meta:
        meta = each.text.strip().encode('utf8')
        stars, language = '', 'unknown'
        if dot in meta:
            temp = meta.split(dot)
            for each_option in temp:
                if b"stars" in each_option:
                    stars = replace_new_lines_and_strip(each_option).decode('utf8')
                elif b"built by" not in each_option.lower() and b"stars" not in each_option.lower():
                    language = replace_new_lines_and_strip(each_option).decode('utf8')
            repo_stars_and_langauges.append([stars, language])
    return repo_stars_and_langauges

def get_trending_repos(**kwargs):
    repos = []
    language = kwargs.get('language', None)
    timespan = kwargs.get('timespan', None)
    url = TRENDING_REPO_URL
    if language:
        url = url + '/' + language
    if timespan:
        url = url + '?since={}'.format(timespan)
    tree, status_code = make_etree(url)
    if status_code == 200:
        repo_names = get_trending_repo_names(tree)
        repo_desc = get_trending_repo_description(tree)
        repo_meta = get_trending_repo_meta(tree)
        repo_stars_and_languages = get_trending_repo_stars_and_languages(repo_meta)
        repos = list(zip(repo_names, repo_desc, repo_stars_and_languages))
        repos = [{'repo_name': repo_name,
                  'description': description,
                  'stars': stars,
                  'language': lang,
                  'url': HOME_PAGE + str(repo_name)}
                 for repo_name, description, [stars, lang] in repos]
    return repos

def get_trending_dev_names(tree):
    devs = tree.xpath('//h2[@class="user-leaderboard-list-name"]')
    devs = [" ".join([x for x in each.itertext()]) for each in devs]
    devs = [replace_new_lines_and_multiple_spaces(replace_new_lines_and_strip(each)) for each in devs]
    return devs

def get_trending_dev_repo_names(tree):
    dev_repo_names = tree.xpath('//span[@class="repo"]')
    dev_repo_names = [" ".join([x for x in each.itertext()]) for each in dev_repo_names]
    dev_repo_names = [replace_new_lines_and_strip(each) for each in dev_repo_names]
    return dev_repo_names

def get_trending_dev_repo_desc(tree):
    dev_repo_desc = tree.xpath('//span[@class="repo-snipit-description css-truncate-target"]')
    dev_repo_desc = [" ".join([x for x in each.itertext()]) for each in dev_repo_desc]
    dev_repo_desc = [replace_new_lines_and_strip(each) for each in dev_repo_desc]
    return dev_repo_desc

def get_trending_devs(**kwargs):
    devs = []
    language = kwargs.get('language', None)
    timespan = kwargs.get('timespan', None)
    url = TRENDING_DEV_URL
    if language:
        url = url + '/' + language
    if timespan:
        url = url + '?since={}'.format(timespan)
    tree, status_code = make_etree(url)
    if status_code == 200:
        dev_names = get_trending_dev_names(tree)
        dev_repo_names = get_trending_dev_repo_names(tree)
        dev_repo_desc = get_trending_dev_repo_desc(tree)
        devs = list(zip(dev_names, dev_repo_names, dev_repo_desc))
        devs = [{'dev_name': dev_name,
                 'repo_name': repo_name,
                 'description': description,
                 'url': HOME_PAGE + '/' + dev_name.split(' ')[0]}
                for dev_name, repo_name, description in devs]
    return devs

@click.command()
@click.option(
    '--repo', '-r', is_flag=True,
    help="Lists the trending repositories.")
@click.option(
    '--dev', '-d', is_flag=True,
    help="Lists the trending developers.")
@click.option('--lang', '-l', help='Specify the language')
@click.option('--week', 'timespan', flag_value='weekly')
@click.option('--month', 'timespan', flag_value='monthly')
@click.argument('goto', nargs=1, required=False, type=click.INT)
def main(repo, dev, lang, timespan, goto):
    '''
    A command line utility to see the trending repositories
    and developers on Github
    '''
    language = None
    if lang:
        language = str(lang)
    opts = {
        'language': language,
        'timespan': timespan,
    }
    try:
        if repo:
            repos = get_trending_repos(**opts)
            if goto:
                webbrowser.open(repos[goto-1]['url'], new=2)
                return
            else:
                writers.print_trending_repos(repos)
        if dev:
            devs = get_trending_devs(**opts)
            writers.print_trending_devs(devs)
        # if the user does not passes any argument then list the trending repo
        if not repo and not dev:
            repos = get_trending_repos(**opts)
            if goto:
                webbrowser.open(repos[goto-1]['url'], new=2)
                return
            else:
                writers.print_trending_repos(repos)
        return
    except Exception as e:
        click.secho(e.message, fg="red", bold=True)

if __name__ == '__main__':
    main()
<commit_msg>Trending.py: Add support to open dev in browser<commit_after># -*- coding: utf-8 -*-
import click
import requests
import webbrowser
from lxml import etree

from . import writers

TRENDING_REPO_URL = 'http://github.com/trending'
TRENDING_DEV_URL = 'http://github.com/trending/developers'
HOME_PAGE = 'https://github.com'

requests.packages.urllib3.disable_warnings()

def replace_new_lines_and_strip(s):
    return s.strip()

def replace_new_lines_and_multiple_spaces(s):
    return ' '.join(s.replace('\n', '').split())

def read_page(url, timeout=5):

    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:11.0) Gecko/20100101 Firefox/11.0'}
    try:
        response = requests.get(url, timeout=timeout, headers=headers)
    except requests.exceptions.ConnectionError as e:  # noqa
        return(None, False)

    return(response, response.status_code)

def make_etree(url):
    response, status_code = read_page(url)
    if status_code == 200:
        response = etree.HTML(response.text)
    return (response, status_code)

def get_trending_repo_names(tree):
    repos = tree.xpath('//h3[@class="repo-list-name"]/a/@href')
    return repos

def get_trending_repo_description(tree):
    repo_desc = tree.xpath('//p[@class="repo-list-description"]')
    repo_desc = [" ".join([x for x in each.itertext()]) for each in repo_desc]
    repo_desc = [replace_new_lines_and_strip(each) for each in repo_desc]
    return repo_desc

def get_trending_repo_meta(tree):
    repo_meta = tree.xpath('//p[@class="repo-list-meta"]')
    return repo_meta

def get_trending_repo_stars_and_languages(repo_meta):
    dot = u'•'.encode('utf8')
    repo_stars_and_langauges = []
    for each in repo_meta:
        meta = each.text.strip().encode('utf8')
        stars, language = '', 'unknown'
        if dot in meta:
            temp = meta.split(dot)
            for each_option in temp:
                if b"stars" in each_option:
                    stars = replace_new_lines_and_strip(each_option).decode('utf8')
                elif b"built by" not in each_option.lower() and b"stars" not in each_option.lower():
                    language = replace_new_lines_and_strip(each_option).decode('utf8')
            repo_stars_and_langauges.append([stars, language])
    return repo_stars_and_langauges

def get_trending_repos(**kwargs):
    repos = []
    language = kwargs.get('language', None)
    timespan = kwargs.get('timespan', None)
    url = TRENDING_REPO_URL
    if language:
        url = url + '/' + language
    if timespan:
        url = url + '?since={}'.format(timespan)
    tree, status_code = make_etree(url)
    if status_code == 200:
        repo_names = get_trending_repo_names(tree)
        repo_desc = get_trending_repo_description(tree)
        repo_meta = get_trending_repo_meta(tree)
        repo_stars_and_languages = get_trending_repo_stars_and_languages(repo_meta)
        repos = list(zip(repo_names, repo_desc, repo_stars_and_languages))
        repos = [{'repo_name': repo_name,
                  'description': description,
                  'stars': stars,
                  'language': lang,
                  'url': HOME_PAGE + str(repo_name)}
                 for repo_name, description, [stars, lang] in repos]
    return repos

def get_trending_dev_names(tree):
    devs = tree.xpath('//h2[@class="user-leaderboard-list-name"]')
    devs = [" ".join([x for x in each.itertext()]) for each in devs]
    devs = [replace_new_lines_and_multiple_spaces(replace_new_lines_and_strip(each)) for each in devs]
    return devs

def get_trending_dev_repo_names(tree):
    dev_repo_names = tree.xpath('//span[@class="repo"]')
    dev_repo_names = [" ".join([x for x in each.itertext()]) for each in dev_repo_names]
    dev_repo_names = [replace_new_lines_and_strip(each) for each in dev_repo_names]
    return dev_repo_names

def get_trending_dev_repo_desc(tree):
    dev_repo_desc = tree.xpath('//span[@class="repo-snipit-description css-truncate-target"]')
    dev_repo_desc = [" ".join([x for x in each.itertext()]) for each in dev_repo_desc]
    dev_repo_desc = [replace_new_lines_and_strip(each) for each in dev_repo_desc]
    return dev_repo_desc

def get_trending_devs(**kwargs):
    devs = []
    language = kwargs.get('language', None)
    timespan = kwargs.get('timespan', None)
    url = TRENDING_DEV_URL
    if language:
        url = url + '/' + language
    if timespan:
        url = url + '?since={}'.format(timespan)
    tree, status_code = make_etree(url)
    if status_code == 200:
        dev_names = get_trending_dev_names(tree)
        dev_repo_names = get_trending_dev_repo_names(tree)
        dev_repo_desc = get_trending_dev_repo_desc(tree)
        devs = list(zip(dev_names, dev_repo_names, dev_repo_desc))
        devs = [{'dev_name': dev_name,
                 'repo_name': repo_name,
                 'description': description,
                 'url': HOME_PAGE + '/' + dev_name.split(' ')[0]}
                for dev_name, repo_name, description in devs]
    return devs

@click.command()
@click.option(
    '--repo', '-r', is_flag=True,
    help="Lists the trending repositories.")
@click.option(
    '--dev', '-d', is_flag=True,
    help="Lists the trending developers.")
@click.option('--lang', '-l', help='Specify the language')
@click.option('--week', 'timespan', flag_value='weekly')
@click.option('--month', 'timespan', flag_value='monthly')
@click.argument('goto', nargs=1, required=False, type=click.INT)
def main(repo, dev, lang, timespan, goto):
    '''
    A command line utility to see the trending repositories
    and developers on Github
    '''
    language = None
    if lang:
        language = str(lang)
    opts = {
        'language': language,
        'timespan': timespan,
    }
    try:
        if repo:
            repos = get_trending_repos(**opts)
            if goto:
                webbrowser.open(repos[goto-1]['url'], new=2)
                return
            else:
                writers.print_trending_repos(repos)
        if dev:
            devs = get_trending_devs(**opts)
            if goto:
                webbrowser.open(devs[goto-1]['url'], new=2)
                return
            else:
                writers.print_trending_devs(devs)
        # if the user does not passes any argument then list the trending repo
        if not repo and not dev:
            repos = get_trending_repos(**opts)
            if goto:
                webbrowser.open(repos[goto-1]['url'], new=2)
                return
            else:
                writers.print_trending_repos(repos)
        return
    except Exception as e:
        click.secho(e.message, fg="red", bold=True)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from __future__ import division

from collections import deque

import pyglet
from pyglet import font
from pyglet.window import mouse, key
from pyglet.gl import *
from euclid import Vector2
from spring import Spring

DONE = object()
BACK = object()
SELECT = object()

LABEL_BACK = u'\u276e Back'
LABEL_REMOVE = u'\u2716 Remove'
LABEL_START = u'\u25b6 Start'
LABEL_STOP = u'\u25a0 Stop'

LOADED_FONTS = {}

class Font(object):
    name = None
    filename = None
    size = None

    def __init__(self):
        if not self.filename in LOADED_FONTS:
            pyglet.resource.add_font(self.filename)
            LOADED_FONTS[self.filename] = 1

class MainMenuFont(Font):
    name = "DejaVu Sans Light"
    filename = 'DejaVuSans-ExtraLight.ttf'
    size = 24
    height = 32

class GameMenuFont(Font):
    name = "DejaVu Sans"
    filename = 'DejaVuSans.ttf'
    size = 10
    height = 16

class PopupFont(Font):
    name = "DejaVu Sans"
    filename = 'DejaVuSans.ttf'
    size = 20
    height = 20

class Manager (object):
    ANIM_STRIDE = 2
    MARGIN = 20
    SPACING = 25

    colors = {
        'okay': (0x34, 0xB2, 0x7D, 0xFF),
        'title': (0xEE, 0xEE, 0xEE, 0xFF),
    }

    def __init__(self, window):
        self.window = window
        self.stack = []
        self.active = []
        self.buttons = []
        self.selected = None
        self.popup = None
        self.popup_queue = deque()

    def show_popup(self, text, color='okay'):
        color = self.colors[color]
        self.popup_queue.append((text, color))

    def replace(self, buttons):
        self.hide(True, True)
        self.stack = []
        self.set_active(buttons)

    def push(self, buttons, add_back=True):
        self.hide()
        self.stack.append(self.active)
        if add_back:
            buttons += [ Button(LABEL_BACK, GameMenuFont(), BACK) ]
        self.set_active(buttons)

    def set_active(self, buttons):
        self.active = list(buttons)
        self.buttons.extend(self.active)
        self.show(True)

    def pop(self):
        self.hide(True)
        self.selected = None
        self.active = self.stack.pop()
        self.show()

    def hide(self, delete=False, everything=False):
        buttons = self.buttons if everything else self.active

        for idx, btn in enumerate(buttons):
            btn.delay = idx * self.ANIM_STRIDE
            new_x = -btn.width - self.MARGIN if delete else -btn.width + self.MARGIN * (len(self.stack) + 1)
            new_y = btn.pos[1] - self.MARGIN
            btn.target = Vector2(new_x, new_y)
            btn.auto_delete = delete

    def show(self, reset_pos=False):
        offset = 0
        for idx, btn in enumerate(self.active):
            offset -= btn.height + self.SPACING
            btn.delay = idx * self.ANIM_STRIDE
            if reset_pos:
                btn.pos = Vector2(-btn.width - self.MARGIN, offset + self.MARGIN)
            btn.target = Vector2(self.MARGIN * (len(self.stack) + 1), offset)

    def tick(self, dt=None):
        self.buttons = [b for b in self.buttons if b.tick() is not DONE]

        if self.popup and self.popup.tick() is DONE:
            self.popup = None

        if not self.popup and self.popup_queue:
            text, color = self.popup_queue.popleft()
            self.popup = Popup(text, color)

    def draw(self):
        glPushMatrix()
        glTranslatef(0, self.window.height, 0)
        [btn.draw() for btn in self.buttons]
        glPopMatrix()

        if self.popup:
            self.popup.draw(self.window)

    def on_mouse_release(self, x, y, btn, mods):
        if not self.active:
            return
        y -= self.window.height
        if btn == mouse.LEFT:
            if y > self.active[-1].pos.y and x < self.MARGIN * len(self.stack):
                self.pop()
                return pyglet.event.EVENT_HANDLED

            for obj in self.active:
                if obj.contains(x, y):
                    return self.handle_click(obj)

    def handle_click(self, obj):
        old_selected = self.selected
        if self.selected:
            self.selected.selected(0)
            self.selected = None

        # Second click removes selection and that's it.
        if old_selected == obj:
            return pyglet.event.EVENT_HANDLED

        if callable(obj.callback):
            ret = obj.callback(self, obj.args)
        else:
            ret = obj.callback

        if ret is BACK:
            self.pop()

        elif ret is SELECT:
            obj.selected(self.MARGIN)
            self.selected = obj

        return pyglet.event.EVENT_HANDLED

    def on_key_press(self, sym, mod):
        if key._0 <= sym <= key._9:
            ofs = sym - key._0
            ofs = (ofs - 1) % 10

            if ofs < len(self.active):
                return self.handle_click(self.active[ofs])

        if sym == key.ESCAPE:
            if self.stack:
                self.pop()
            if self.selected:
                self.selected.selected(0)
                self.selected = None
            return pyglet.event.EVENT_HANDLED

class Button (object):
    THRESHOLD = 1
    SPEED = 0.4

    def __init__(self, text, font, callback, args=None, color=None):
        self.label = pyglet.text.Label(
            text,
            font_name = font.name,
            font_size = font.size,
            anchor_y = 'baseline'
        )
        self.callback = callback
        self.args = args

        w = self.label.content_width
        h = font.height

        self.width = w + 10
        self.height = h + 5

        self.delay = 0
        self.target = Vector2(0, 0)
        self.pos = Vector2(0, 0)
        self.offset = Vector2(0, 0)
        self.auto_delete = False

        self.vlist = pyglet.graphics.vertex_list(8,
            ('v2f', (
                -15, -15, w + 10, -15, w + 15, h + 9, -10, h + 9,
                -10, -10, w + 5, -10, w + 10, h + 4, -5, h + 4,
            )),
            ('c3f', (.1, .1, .1) * 4 + (.2, .2, .2) * 4)
        )

    def tick(self):
        if self.delay > 0:
            self.delay -= 1
            return

        target = self.target + self.offset
        delta = target - self.pos
        if abs(delta) < self.THRESHOLD:
            self.pos = target

            if self.auto_delete:
                self.delete()
                return DONE
        else:
            self.pos += delta * self.SPEED

    def draw(self):
        glPushMatrix(GL_MODELVIEW)
        x, y = self.pos
        glTranslatef(x, y, 0)
        self.vlist.draw(GL_QUADS)
        self.label.draw()
        glPopMatrix()

    def delete(self):
        self.label.delete()
        self.vlist.delete()

    def selected(self, offset):
        self.offset.x = offset
        color = (.4, .4, .2) if offset else (.2, .2, .2)
        self.vlist.colors[12:] = color * 4

    def contains(self, x, y):
        px, py = self.pos
        return -10 <= x - px <= self.width and -10 <= y - py <= self.height

class CountButton (Button):
    THRESHOLD = 1
    SPEED = 0.4

    def __init__(self, text, font, count, callback, args=None, color=None):
        self._text = text
        self._count = count
        super(CountButton, self).__init__(self.text_count, font, callback, args, color)

    @property
    def text(self):
        return self._text

    @text.setter
    def text(self, value):
        self._text = value
        self.label.text = self.text_count

    @property
    def count(self):
        return self._count

    @count.setter
    def count(self, value):
        self._count = value
        self.label.text = self.text_count

    @property
    def text_count(self):
        return u'%s \u00d7 %d' % (self._text, self._count)

class Submenu (object):
    def __init__(self, choices):
        self.choices = choices

    def __call__(self, manager, args):
        buttons = self.build()
        manager.push(buttons)

    def build(self):
        return [Button(label, font, callback, args) for label, font, callback, args in self.choices]

class Popup (object):
    SPEED = 0.2
    SNAP = 0.001
    SLEEP = 60
    POS_Y = 0.3

    def __init__(self, text, color):
        font = PopupFont()
        self.label = pyglet.text.Label(
            text,
            anchor_x = 'center',
            font_name = font.name,
            font_size = font.size,
            anchor_y = 'baseline',
            color = color
        )
        self.offset = Spring(Vector2(1.5, self.POS_Y), self.SPEED, self.SNAP)
        self.offset.next_value = Vector2(.5, self.POS_Y)
        self.sleep = self.SLEEP
        self.done = False

    def tick(self):
        self.offset.tick()
        if self.offset.static:
            if self.done:
                self.label.delete()
                return DONE

            if self.sleep > 0:
                self.sleep -= 1
            else:
                self.offset.next_value = Vector2(-.5, self.POS_Y)
                self.done = True

    def draw(self, window):
        x, y = self.offset.value.xy
        glPushMatrix()
        glTranslatef(x * window.width, y * window.height, 0)
        self.label.draw()
        glPopMatrix()

if __name__ == '__main__':
    win = pyglet.window.Window(width=640, height=480)
    man = Manager(win)
    txt = [
        ('Spam', (
            ('Spam', (('Eggs', SELECT), ('Bacon', SELECT), ('Spam', SELECT))),
            ('Bacon', SELECT)
        )),
        ('Welcome', lambda man, args: man.show_popup('Welcome')),
        ('Quit', DONE),
    ]

    def mk_handler(sub):
        if sub is DONE:
            return lambda man, args: pyglet.app.exit()
        elif sub is SELECT:
            return sub
        elif callable(sub):
            return sub
        elif sub:
            def handler(manager, args):
                manager.push([Button(label, mk_handler(sub2)) for label, sub2 in sub])
            return handler

    font = GameMenuFont()
    man.replace(CountButton(t, font, 15, mk_handler(sub)) for t, sub in txt)

    glClearColor(.3, .3, .3, 1)

    @win.event
    def on_draw():
        win.clear()
        man.draw()

    win.push_handlers(man)

    def update(dt):
        man.tick()
        man.active[0].count += 1

    pyglet.clock.schedule_interval_soft(update, 1/60)

    pyglet.app.run()
<commit_msg>Fix bug with lingering menu selection.<commit_after>from __future__ import division

from collections import deque

import pyglet
from pyglet import font
from pyglet.window import mouse, key
from pyglet.gl import *
from euclid import Vector2
from spring import Spring

DONE = object()
BACK = object()
SELECT = object()

LABEL_BACK = u'\u276e Back'
LABEL_REMOVE = u'\u2716 Remove'
LABEL_START = u'\u25b6 Start'
LABEL_STOP = u'\u25a0 Stop'

LOADED_FONTS = {}

class Font(object):
    name = None
    filename = None
    size = None

    def __init__(self):
        if not self.filename in LOADED_FONTS:
            pyglet.resource.add_font(self.filename)
            LOADED_FONTS[self.filename] = 1

class MainMenuFont(Font):
    name = "DejaVu Sans Light"
    filename = 'DejaVuSans-ExtraLight.ttf'
    size = 24
    height = 32

class GameMenuFont(Font):
    name = "DejaVu Sans"
    filename = 'DejaVuSans.ttf'
    size = 10
    height = 16

class PopupFont(Font):
    name = "DejaVu Sans"
    filename = 'DejaVuSans.ttf'
    size = 20
    height = 20

class Manager (object):
    ANIM_STRIDE = 2
    MARGIN = 20
    SPACING = 25

    colors = {
        'okay': (0x34, 0xB2, 0x7D, 0xFF),
        'title': (0xEE, 0xEE, 0xEE, 0xFF),
    }

    def __init__(self, window):
        self.window = window
        self.stack = []
        self.active = []
        self.buttons = []
        self.selected = None
        self.popup = None
        self.popup_queue = deque()

    def show_popup(self, text, color='okay'):
        color = self.colors[color]
        self.popup_queue.append((text, color))

    def replace(self, buttons):
        self.hide(True, True)
        self.stack = []
        self.set_active(buttons)
        self.reset_selection()

    def push(self, buttons, add_back=True):
        self.hide()
        self.stack.append(self.active)
        if add_back:
            buttons += [ Button(LABEL_BACK, GameMenuFont(), BACK) ]
        self.set_active(buttons)

    def set_active(self, buttons):
        self.active = list(buttons)
        self.buttons.extend(self.active)
        self.show(True)

    def pop(self):
        self.hide(True)
        self.selected = None
        self.active = self.stack.pop()
        self.show()

    def hide(self, delete=False, everything=False):
        buttons = self.buttons if everything else self.active

        for idx, btn in enumerate(buttons):
            btn.delay = idx * self.ANIM_STRIDE
            new_x = -btn.width - self.MARGIN if delete else -btn.width + self.MARGIN * (len(self.stack) + 1)
            new_y = btn.pos[1] - self.MARGIN
            btn.target = Vector2(new_x, new_y)
            btn.auto_delete = delete

    def show(self, reset_pos=False):
        offset = 0
        for idx, btn in enumerate(self.active):
            offset -= btn.height + self.SPACING
            btn.delay = idx * self.ANIM_STRIDE
            if reset_pos:
                btn.pos = Vector2(-btn.width - self.MARGIN, offset + self.MARGIN)
            btn.target = Vector2(self.MARGIN * (len(self.stack) + 1), offset)

    def tick(self, dt=None):
        self.buttons = [b for b in self.buttons if b.tick() is not DONE]

        if self.popup and self.popup.tick() is DONE:
            self.popup = None

        if not self.popup and self.popup_queue:
            text, color = self.popup_queue.popleft()
            self.popup = Popup(text, color)

    def draw(self):
        glPushMatrix()
        glTranslatef(0, self.window.height, 0)
        [btn.draw() for btn in self.buttons]
        glPopMatrix()

        if self.popup:
            self.popup.draw(self.window)

    def on_mouse_release(self, x, y, btn, mods):
        if not self.active:
            return
        y -= self.window.height
        if btn == mouse.LEFT:
            if y > self.active[-1].pos.y and x < self.MARGIN * len(self.stack):
                self.pop()
                return pyglet.event.EVENT_HANDLED

            for obj in self.active:
                if obj.contains(x, y):
                    return self.handle_click(obj)

    def reset_selection(self):
        if self.selected:
            self.selected.selected(0)
            self.selected = None

    def handle_click(self, obj):
        old_selected = self.selected
        self.reset_selection()

        # Second click removes selection and that's it.
        if old_selected == obj:
            return pyglet.event.EVENT_HANDLED

        if callable(obj.callback):
            ret = obj.callback(self, obj.args)
        else:
            ret = obj.callback

        if ret is BACK:
            self.pop()

        elif ret is SELECT:
            obj.selected(self.MARGIN)
            self.selected = obj

        return pyglet.event.EVENT_HANDLED

    def on_key_press(self, sym, mod):
        if key._0 <= sym <= key._9:
            ofs = sym - key._0
            ofs = (ofs - 1) % 10

            if ofs < len(self.active):
                return self.handle_click(self.active[ofs])

        if sym == key.ESCAPE:
            if self.stack:
                self.pop()
            self.reset_selection()
            return pyglet.event.EVENT_HANDLED

class Button (object):
    THRESHOLD = 1
    SPEED = 0.4

    def __init__(self, text, font, callback, args=None, color=None):
        self.label = pyglet.text.Label(
            text,
            font_name = font.name,
            font_size = font.size,
            anchor_y = 'baseline'
        )
        self.callback = callback
        self.args = args

        w = self.label.content_width
        h = font.height

        self.width = w + 10
        self.height = h + 5

        self.delay = 0
        self.target = Vector2(0, 0)
        self.pos = Vector2(0, 0)
        self.offset = Vector2(0, 0)
        self.auto_delete = False

        self.vlist = pyglet.graphics.vertex_list(8,
            ('v2f', (
                -15, -15, w + 10, -15, w + 15, h + 9, -10, h + 9,
                -10, -10, w + 5, -10, w + 10, h + 4, -5, h + 4,
            )),
            ('c3f', (.1, .1, .1) * 4 + (.2, .2, .2) * 4)
        )

    def tick(self):
        if self.delay > 0:
            self.delay -= 1
            return

        target = self.target + self.offset
        delta = target - self.pos
        if abs(delta) < self.THRESHOLD:
            self.pos = target

            if self.auto_delete:
                self.delete()
                return DONE
        else:
            self.pos += delta * self.SPEED

    def draw(self):
        glPushMatrix(GL_MODELVIEW)
        x, y = self.pos
        glTranslatef(x, y, 0)
        self.vlist.draw(GL_QUADS)
        self.label.draw()
        glPopMatrix()

    def delete(self):
        self.label.delete()
        self.vlist.delete()

    def selected(self, offset):
        self.offset.x = offset
        color = (.4, .4, .2) if offset else (.2, .2, .2)
        self.vlist.colors[12:] = color * 4

    def contains(self, x, y):
        px, py = self.pos
        return -10 <= x - px <= self.width and -10 <= y - py <= self.height

class CountButton (Button):
    THRESHOLD = 1
    SPEED = 0.4

    def __init__(self, text, font, count, callback, args=None, color=None):
        self._text = text
        self._count = count
        super(CountButton, self).__init__(self.text_count, font, callback, args, color)

    @property
    def text(self):
        return self._text

    @text.setter
    def text(self, value):
        self._text = value
        self.label.text = self.text_count

    @property
    def count(self):
        return self._count

    @count.setter
    def count(self, value):
        self._count = value
        self.label.text = self.text_count

    @property
    def text_count(self):
        return u'%s \u00d7 %d' % (self._text, self._count)

class Submenu (object):
    def __init__(self, choices):
        self.choices = choices

    def __call__(self, manager, args):
        buttons = self.build()
        manager.push(buttons)

    def build(self):
        return [Button(label, font, callback, args) for label, font, callback, args in self.choices]

class Popup (object):
    SPEED = 0.2
    SNAP = 0.001
    SLEEP = 60
    POS_Y = 0.3

    def __init__(self, text, color):
        font = PopupFont()
        self.label = pyglet.text.Label(
            text,
            anchor_x = 'center',
            font_name = font.name,
            font_size = font.size,
            anchor_y = 'baseline',
            color = color
        )
        self.offset = Spring(Vector2(1.5, self.POS_Y), self.SPEED, self.SNAP)
        self.offset.next_value = Vector2(.5, self.POS_Y)
        self.sleep = self.SLEEP
        self.done = False

    def tick(self):
        self.offset.tick()
        if self.offset.static:
            if self.done:
                self.label.delete()
                return DONE

            if self.sleep > 0:
                self.sleep -= 1
            else:
                self.offset.next_value = Vector2(-.5, self.POS_Y)
                self.done = True

    def draw(self, window):
        x, y = self.offset.value.xy
        glPushMatrix()
        glTranslatef(x * window.width, y * window.height, 0)
        self.label.draw()
        glPopMatrix()

if __name__ == '__main__':
    win = pyglet.window.Window(width=640, height=480)
    man = Manager(win)
    txt = [
        ('Spam', (
            ('Spam', (('Eggs', SELECT), ('Bacon', SELECT), ('Spam', SELECT))),
            ('Bacon', SELECT)
        )),
        ('Welcome', lambda man, args: man.show_popup('Welcome')),
        ('Quit', DONE),
    ]

    def mk_handler(sub):
        if sub is DONE:
            return lambda man, args: pyglet.app.exit()
        elif sub is SELECT:
            return sub
        elif callable(sub):
            return sub
        elif sub:
            def handler(manager, args):
                manager.push([Button(label, mk_handler(sub2)) for label, sub2 in sub])
            return handler

    font = GameMenuFont()
    man.replace(CountButton(t, font, 15, mk_handler(sub)) for t, sub in txt)

    glClearColor(.3, .3, .3, 1)

    @win.event
    def on_draw():
        win.clear()
        man.draw()

    win.push_handlers(man)

    def update(dt):
        man.tick()
        man.active[0].count += 1

    pyglet.clock.schedule_interval_soft(update, 1/60)

    pyglet.app.run()
<|endoftext|>


<commit_before># -*- coding: UTF-8
#
# oonicli
# -------
# In here we take care of running ooniprobe from the command
# line interface
#
# :authors: Arturo Filastò, Isis Lovecruft
# :license: see included LICENSE file

import sys
import os
import random
import time

from twisted.internet import defer, reactor
from twisted.application import app
from twisted.python import usage, failure
from twisted.python.util import spewer

from ooni import nettest, runner, reporter, config

from ooni.inputunit import InputUnitFactory

from ooni.utils import net
from ooni.utils import checkForRoot, PermissionsError
from ooni.utils import log

class Options(usage.Options, app.ReactorSelectionMixin):
    synopsis = """%s [options] [path to test].py
    """ % (os.path.basename(sys.argv[0]),)

    longdesc = ("ooniprobe loads and executes a suite or a set of suites of"
                " network tests. These are loaded from modules, packages and"
                " files listed on the command line")

    optFlags = [["help", "h"],
                ['debug-stacktraces', 'B',
                 'Report deferred creation and callback stack traces'],]

    optParameters = [
        ["reportfile", "o", None, "report file name"],
        ["collector", "c", None,
         "Address of the collector of test results. (example: http://127.0.0.1:8888)"],
        ["logfile", "l", None, "log file name"],
        ["pcapfile", "p", None, "pcap file name"]]

    compData = usage.Completions(
        extraActions=[usage.CompleteFiles(
                "*.py", descr="file | module | package | TestCase | testMethod",
                repeat=True)],)

    tracer = None

    def __init__(self):
        self['test'] = None
        usage.Options.__init__(self)

    def opt_asciilulz(self):
        from ooni.utils import logo
        print logo.getlogo()

    def opt_spew(self):
        """
        Print an insanely verbose log of everything that happens.  Useful
        when debugging freezes or locks in complex code.
        """
        sys.settrace(spewer)

    def parseArgs(self, *args):
        try:
            self['test'] = args[0]
            self['subArgs'] = args[1:]
        except:
            raise usage.UsageError("No test filename specified!")

def testsEnded(*arg, **kw):
    """You can place here all the post shutdown tasks."""
    log.debug("testsEnded: Finished running all tests")

def run():
    """Call me to begin testing from a file."""

    cmd_line_options = Options()
    if len(sys.argv) == 1:
        cmd_line_options.getUsage()
    try:
        cmd_line_options.parseOptions()
    except usage.UsageError, ue:
        raise SystemExit, "%s: %s" % (sys.argv[0], ue)

    if cmd_line_options['debug-stacktraces']:
        defer.setDebugging(True)

    log.start(cmd_line_options['logfile'])

    test_file_name = os.path.basename(cmd_line_options['test'])
    log.debug("Running script %s" % test_file_name)

    yamloo_filename, pcap_filename = config.oreport_filenames(test_file_name)

    if cmd_line_options['reportfile']:
        yamloo_filename = cmd_line_options['reportfile']
        pcap_filename = yamloo_filename+".pcap"

    if os.path.exists(yamloo_filename):
        log.msg("Report already exists with filename %s" % yamloo_filename)
        log.msg("Renaming it to %s" % yamloo_filename+'.old')
        os.rename(yamloo_filename, yamloo_filename+'.old')
    if os.path.exists(pcap_filename):
        log.msg("Report already exists with filename %s" % pcap_filename)
        log.msg("Renaming it to %s" % pcap_filename+'.old')
        os.rename(pcap_filename, pcap_filename+'.old')

    classes = runner.findTestClassesFromConfig(cmd_line_options)
    test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options)
    if config.privacy.includepcap:
        try:
            checkForRoot()
        except PermissionsError, pe:
            log.err(str("'includepcap' option requires administrator or root ",
                        "privileges to run. Run ooniprobe as root or disable ",
                        "the includepcap option in ooniprobe.conf "))
            sys.exit(1)
        log.debug("Starting sniffer")
        sniffer_d = net.capturePackets(pcap_filename)

    tests_d = runner.runTestCases(test_cases, options,
                                  cmd_line_options, yamloo_filename)
    tests_d.addBoth(testsEnded)

    reactor.run()

<commit_msg>Added warning suppression for scapy's logger to the beginning of oonicli.py.<commit_after># -*- coding: UTF-8
#
# oonicli
# -------
# In here we take care of running ooniprobe from the command
# line interface
#
# :authors: Arturo Filastò, Isis Lovecruft
# :license: see included LICENSE file

import sys
import os
import random
import time

from twisted.internet import defer, reactor
from twisted.application import app
from twisted.python import usage, failure
from twisted.python.util import spewer

## Get rid of the annoying "No route found for
## IPv6 destination warnings":
import logging as pylogging
pylogging.getLogger("scapy.runtime").setLevel(pylogging.ERROR)

from ooni import nettest, runner, reporter, config
from ooni.utils import net
from ooni.utils import checkForRoot, PermissionsError
from ooni.utils import log
from ooni.inputunit import InputUnitFactory

class Options(usage.Options, app.ReactorSelectionMixin):
    synopsis = """%s [options] [path to test].py
    """ % (os.path.basename(sys.argv[0]),)

    longdesc = ("ooniprobe loads and executes a suite or a set of suites of"
                " network tests. These are loaded from modules, packages and"
                " files listed on the command line")

    optFlags = [["help", "h"],
                ['debug-stacktraces', 'B',
                 'Report deferred creation and callback stack traces'],]

    optParameters = [
        ["reportfile", "o", None, "report file name"],
        ["collector", "c", None,
         "Address of the collector of test results. (example: http://127.0.0.1:8888)"],
        ["logfile", "l", None, "log file name"],
        ["pcapfile", "p", None, "pcap file name"]]

    compData = usage.Completions(
        extraActions=[usage.CompleteFiles(
                "*.py", descr="file | module | package | TestCase | testMethod",
                repeat=True)],)

    tracer = None

    def __init__(self):
        self['test'] = None
        usage.Options.__init__(self)

    def opt_asciilulz(self):
        from ooni.utils import logo
        print logo.getlogo()

    def opt_spew(self):
        """
        Print an insanely verbose log of everything that happens.  Useful
        when debugging freezes or locks in complex code.
        """
        sys.settrace(spewer)

    def parseArgs(self, *args):
        try:
            self['test'] = args[0]
            self['subArgs'] = args[1:]
        except:
            raise usage.UsageError("No test filename specified!")

def testsEnded(*arg, **kw):
    """You can place here all the post shutdown tasks."""
    log.debug("testsEnded: Finished running all tests")

def run():
    """Call me to begin testing from a file."""

    cmd_line_options = Options()
    if len(sys.argv) == 1:
        cmd_line_options.getUsage()
    try:
        cmd_line_options.parseOptions()
    except usage.UsageError, ue:
        raise SystemExit, "%s: %s" % (sys.argv[0], ue)

    if cmd_line_options['debug-stacktraces']:
        defer.setDebugging(True)

    log.start(cmd_line_options['logfile'])

    test_file_name = os.path.basename(cmd_line_options['test'])
    log.debug("Running script %s" % test_file_name)

    yamloo_filename, pcap_filename = config.oreport_filenames(test_file_name)

    if cmd_line_options['reportfile']:
        yamloo_filename = cmd_line_options['reportfile']
        pcap_filename = yamloo_filename+".pcap"

    if os.path.exists(yamloo_filename):
        log.msg("Report already exists with filename %s" % yamloo_filename)
        log.msg("Renaming it to %s" % yamloo_filename+'.old')
        os.rename(yamloo_filename, yamloo_filename+'.old')
    if os.path.exists(pcap_filename):
        log.msg("Report already exists with filename %s" % pcap_filename)
        log.msg("Renaming it to %s" % pcap_filename+'.old')
        os.rename(pcap_filename, pcap_filename+'.old')

    classes = runner.findTestClassesFromConfig(cmd_line_options)
    test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options)
    if config.privacy.includepcap:
        try:
            checkForRoot()
        except PermissionsError, pe:
            log.err(str("'includepcap' option requires administrator or root ",
                        "privileges to run. Run ooniprobe as root or disable ",
                        "the includepcap option in ooniprobe.conf "))
            sys.exit(1)
        log.debug("Starting sniffer")
        sniffer_d = net.capturePackets(pcap_filename)

    tests_d = runner.runTestCases(test_cases, options,
                                  cmd_line_options, yamloo_filename)
    tests_d.addBoth(testsEnded)

    reactor.run()

<|endoftext|>


<commit_before>'''
Example audio-to-MIDI alignment script.
Requires djitw https://github.com/craffel/djitw/
and librosa >= 0.4 https://github.com/bmcfee/librosa/
'''

import djitw
import librosa
import numpy as np
import sys
import argparse
import pretty_midi

# Default values for audio/CQT parameters
FS = 22050
HOP = 512
NOTE_START = 36
N_NOTES = 48

def extract_cqt(audio_data, fs, hop, note_start, n_notes):
    '''
    Compute a log-magnitude L2-normalized constant-Q-gram of some audio data.

    Parameters
    ----------
    audio_data : np.ndarray
        Audio data to compute CQT of
    fs : int
        Sampling rate of audio
    hop : int
        Hop length for CQT
    note_start : int
        Lowest MIDI note number for CQT
    n_notes : int
        Number of notes to include in the CQT

    Returns
    -------
    cqt : np.ndarray
        Log-magnitude L2-normalized CQT of the supplied audio data.
    frame_times : np.ndarray
        Times, in seconds, of each frame in the CQT
    '''
    # Compute CQT
    cqt = librosa.cqt(
        audio_data, sr=fs, hop_length=hop,
        fmin=librosa.midi_to_hz(note_start), n_bins=n_notes)
    # Transpose so that rows are spectra
    cqt = cqt.T
    # Compute log-amplitude
    cqt = librosa.logamplitude(cqt, ref_power=cqt.max())
    # L2 normalize the columns
    cqt = librosa.util.normalize(cqt, norm=2., axis=1)
    # Compute the time of each frame
    times = librosa.frames_to_time(np.arange(cqt.shape[0]), fs, hop)
    return cqt, times

def align(midi_object, audio_data, fs, hop, note_start, n_notes, penalty):
    '''
    Align a MIDI object in-place to some audio data.

    Parameters
    ----------
    midi_object : pretty_midi.PrettyMIDI
        A pretty_midi.PrettyMIDI class instance describing some MIDI content
    audio_data : np.ndarray
        Samples of some audio data
    fs : int
        audio_data's sampling rate, and the sampling rate to use when
        synthesizing MIDI
    hop : int
        Hop length for CQT
    note_start : int
        Lowest MIDI note number for CQT
    n_notes : int
        Number of notes to include in the CQT
    penalty : float
        DTW non-diagonal move penalty
    '''
    # Get synthesized MIDI audio
    midi_audio = midi_object.fluidsynth(fs=fs)
    # Compute CQ-grams for MIDI and audio
    midi_gram, midi_times = extract_cqt(
        midi_audio, fs, hop, note_start, n_notes)
    audio_gram, audio_times = extract_cqt(
        audio_data, fs, hop, note_start, n_notes)
    # Compute distance matrix; because the columns of the CQ-grams are
    # L2-normalized we can compute a cosine distance matrix via a dot product
    distance_matrix = 1 - np.dot(midi_gram, audio_gram.T)
    if penalty is None:
        penalty = distance_matrix.mean()
    # Compute lowest-cost path through distance matrix
    p, q, score = djitw.dtw(
        distance_matrix, gully=.98, additive_penalty=penalty)
    # Adjust the timing of the MIDI object according to the alignment
    midi_object.adjust_times(midi_times[p], audio_times[q])

if __name__ == '__main__':
    # Set up command-line argument parsing
    parser = argparse.ArgumentParser(
        description='Align a MIDI file to an audio file.',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('audio_file', action='store',
                        help='Path to the audio file to align to')
    parser.add_argument('midi_file', action='store',
                        help='Path to the MIDI file to align to the audio')
    parser.add_argument('output_file', action='store',
                        help='Path where the aligned MIDI will be written')
    parser.add_argument('--output_audio', default=None, type=str,
                        action='store',
                        help='Path where an audio file consisting of the '
                             'synthesized aligned MIDI in one channel and the '
                             'audio data in the other will be written')
    parser.add_argument('--fs', default=FS, type=int, action='store',
                        help='Global audio sampling rate to use')
    parser.add_argument('--hop', default=HOP, type=int, action='store',
                        help='CQT hop length')
    parser.add_argument('--note_start', default=NOTE_START, type=int,
                        action='store', help='Lowest CQT MIDI note')
    parser.add_argument('--n_notes', default=N_NOTES, type=int, action='store',
                        help='Number of notes in each CQT')
    parser.add_argument('--penalty', default=None, type=float,
                        action='store', help='DTW non-diagonal move penalty.  '
                        'By default, uses the mean of the distance matrix.')

    parameters = vars(parser.parse_args(sys.argv[1:]))
    print "Loading {} ...".format(parameters['audio_file'])
    audio_data, _ = librosa.load(parameters['audio_file'], sr=parameters['fs'])
    print "Loading {} ...".format(parameters['midi_file'])
    midi_object = pretty_midi.PrettyMIDI(parameters['midi_file'])
    print "Aligning {} to {} ...".format(parameters['audio_file'],
                                         parameters['midi_file'])
    align(midi_object, audio_data, parameters['fs'], parameters['hop'],
          parameters['note_start'], parameters['n_notes'],
          parameters['penalty'])
    print "Writing {} ...".format(parameters['output_file'])
    midi_object.write(parameters['output_file'])
    if parameters['output_audio']:
        print "Writing {} ...".format(parameters['output_audio'])
        # Re-synthesize the aligned mIDI
        midi_audio_aligned = midi_object.fluidsynth(fs=parameters['fs'])
        # Adjust to the same size as audio
        if midi_audio_aligned.shape[0] > audio_data.shape[0]:
            midi_audio_aligned = midi_audio_aligned[:audio_data.shape[0]]
        else:
            trim_amount = audio_data.shape[0] - midi_audio_aligned.shape[0]
            midi_audio_aligned = np.append(midi_audio_aligned,
                                           np.zeros(trim_amount))
        # Write out a .wav with aligned MIDI/audio in each channel
        librosa.output.write_wav(parameters['output_audio'],
                                 np.vstack([midi_audio_aligned, audio_data]),
                                 parameters['fs'])
<commit_msg>Update librosa calls in examples/align_midi.py<commit_after>'''
Example audio-to-MIDI alignment script.
Requires djitw https://github.com/craffel/djitw/
and librosa >= 0.4 https://github.com/bmcfee/librosa/
'''

import djitw
import librosa
import numpy as np
import sys
import argparse
import pretty_midi

# Default values for audio/CQT parameters
FS = 22050
HOP = 512
NOTE_START = 36
N_NOTES = 48

def extract_cqt(audio_data, fs, hop, note_start, n_notes):
    '''
    Compute a log-magnitude L2-normalized constant-Q-gram of some audio data.

    Parameters
    ----------
    audio_data : np.ndarray
        Audio data to compute CQT of
    fs : int
        Sampling rate of audio
    hop : int
        Hop length for CQT
    note_start : int
        Lowest MIDI note number for CQT
    n_notes : int
        Number of notes to include in the CQT

    Returns
    -------
    cqt : np.ndarray
        Log-magnitude L2-normalized CQT of the supplied audio data.
    frame_times : np.ndarray
        Times, in seconds, of each frame in the CQT
    '''
    # Compute CQT
    cqt = librosa.cqt(
        audio_data, sr=fs, hop_length=hop,
        fmin=librosa.midi_to_hz(note_start), n_bins=n_notes)
    # Transpose so that rows are spectra
    cqt = cqt.T
    # Compute log-amplitude
    cqt = librosa.amplitude_to_db(librosa.magphase(cqt)[0], ref=cqt.max())
    # L2 normalize the columns
    cqt = librosa.util.normalize(cqt, norm=2., axis=1)
    # Compute the time of each frame
    times = librosa.frames_to_time(np.arange(cqt.shape[0]), fs, hop)
    return cqt, times

def align(midi_object, audio_data, fs, hop, note_start, n_notes, penalty):
    '''
    Align a MIDI object in-place to some audio data.

    Parameters
    ----------
    midi_object : pretty_midi.PrettyMIDI
        A pretty_midi.PrettyMIDI class instance describing some MIDI content
    audio_data : np.ndarray
        Samples of some audio data
    fs : int
        audio_data's sampling rate, and the sampling rate to use when
        synthesizing MIDI
    hop : int
        Hop length for CQT
    note_start : int
        Lowest MIDI note number for CQT
    n_notes : int
        Number of notes to include in the CQT
    penalty : float
        DTW non-diagonal move penalty
    '''
    # Get synthesized MIDI audio
    midi_audio = midi_object.fluidsynth(fs=fs)
    # Compute CQ-grams for MIDI and audio
    midi_gram, midi_times = extract_cqt(
        midi_audio, fs, hop, note_start, n_notes)
    audio_gram, audio_times = extract_cqt(
        audio_data, fs, hop, note_start, n_notes)
    # Compute distance matrix; because the columns of the CQ-grams are
    # L2-normalized we can compute a cosine distance matrix via a dot product
    distance_matrix = 1 - np.dot(midi_gram, audio_gram.T)
    if penalty is None:
        penalty = distance_matrix.mean()
    # Compute lowest-cost path through distance matrix
    p, q, score = djitw.dtw(
        distance_matrix, gully=.98, additive_penalty=penalty)
    # Adjust the timing of the MIDI object according to the alignment
    midi_object.adjust_times(midi_times[p], audio_times[q])

if __name__ == '__main__':
    # Set up command-line argument parsing
    parser = argparse.ArgumentParser(
        description='Align a MIDI file to an audio file.',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('audio_file', action='store',
                        help='Path to the audio file to align to')
    parser.add_argument('midi_file', action='store',
                        help='Path to the MIDI file to align to the audio')
    parser.add_argument('output_file', action='store',
                        help='Path where the aligned MIDI will be written')
    parser.add_argument('--output_audio', default=None, type=str,
                        action='store',
                        help='Path where an audio file consisting of the '
                             'synthesized aligned MIDI in one channel and the '
                             'audio data in the other will be written')
    parser.add_argument('--fs', default=FS, type=int, action='store',
                        help='Global audio sampling rate to use')
    parser.add_argument('--hop', default=HOP, type=int, action='store',
                        help='CQT hop length')
    parser.add_argument('--note_start', default=NOTE_START, type=int,
                        action='store', help='Lowest CQT MIDI note')
    parser.add_argument('--n_notes', default=N_NOTES, type=int, action='store',
                        help='Number of notes in each CQT')
    parser.add_argument('--penalty', default=None, type=float,
                        action='store', help='DTW non-diagonal move penalty.  '
                        'By default, uses the mean of the distance matrix.')

    parameters = vars(parser.parse_args(sys.argv[1:]))
    print "Loading {} ...".format(parameters['audio_file'])
    audio_data, _ = librosa.load(parameters['audio_file'], sr=parameters['fs'])
    print "Loading {} ...".format(parameters['midi_file'])
    midi_object = pretty_midi.PrettyMIDI(parameters['midi_file'])
    print "Aligning {} to {} ...".format(parameters['audio_file'],
                                         parameters['midi_file'])
    align(midi_object, audio_data, parameters['fs'], parameters['hop'],
          parameters['note_start'], parameters['n_notes'],
          parameters['penalty'])
    print "Writing {} ...".format(parameters['output_file'])
    midi_object.write(parameters['output_file'])
    if parameters['output_audio']:
        print "Writing {} ...".format(parameters['output_audio'])
        # Re-synthesize the aligned mIDI
        midi_audio_aligned = midi_object.fluidsynth(fs=parameters['fs'])
        # Adjust to the same size as audio
        if midi_audio_aligned.shape[0] > audio_data.shape[0]:
            midi_audio_aligned = midi_audio_aligned[:audio_data.shape[0]]
        else:
            trim_amount = audio_data.shape[0] - midi_audio_aligned.shape[0]
            midi_audio_aligned = np.append(midi_audio_aligned,
                                           np.zeros(trim_amount))
        # Write out a .wav with aligned MIDI/audio in each channel
        librosa.output.write_wav(parameters['output_audio'],
                                 np.vstack([midi_audio_aligned, audio_data]),
                                 parameters['fs'])
<|endoftext|>


<commit_before>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import functools
import socket

from ovs.db import idl
from ovs import socket_util
from ovs import stream
from ovsdbapp.backend.ovs_idl import connection
from ovsdbapp.backend.ovs_idl import idlutils
from ovsdbapp.backend.ovs_idl import vlog
from ovsdbapp.schema.open_vswitch import impl_idl

from vif_plug_ovs.ovsdb import api

def idl_factory(config):
    conn = config.connection
    schema_name = 'Open_vSwitch'
    helper = idlutils.get_schema_helper(conn, schema_name)
    helper.register_all()
    return idl.Idl(conn, helper)

def api_factory(config):
    conn = connection.Connection(
        idl=idl_factory(config),
        timeout=config.timeout)
    return NeutronOvsdbIdl(conn)

class NeutronOvsdbIdl(impl_idl.OvsdbIdl, api.ImplAPI):
    """IDL interface for OVS database back-end

    This class provides an OVSDB IDL (Open vSwitch Database Interface
    Definition Language) interface to the OVS back-end.
    """
    def __init__(self, conn):
        vlog.use_python_logger()
        super(NeutronOvsdbIdl, self).__init__(conn)

    def _get_table_columns(self, table):
        return list(self.tables[table].columns)

    def has_table_column(self, table, column):
        return column in self._get_table_columns(table)

# this is derived form https://review.opendev.org/c/openstack/neutron/+/794892
def add_keepalives(fn):
    @functools.wraps(fn)
    def _open(*args, **kwargs):
        error, sock = fn(*args, **kwargs)
        try:
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
        except socket.error as e:
            sock.close()
            return socket_util.get_exception_errno(e), None
        return error, sock
    return _open

class NoProbesMixin:
    @staticmethod
    def needs_probes():
        # If we are using keepalives, we can force probe_interval=0
        return False

class TCPStream(stream.TCPStream, NoProbesMixin):
    @classmethod
    @add_keepalives
    def _open(cls, suffix, dscp):
        return super()._open(suffix, dscp)

class SSLStream(stream.SSLStream, NoProbesMixin):
    @classmethod
    @add_keepalives
    def _open(cls, suffix, dscp):
        return super()._open(suffix, dscp)

# Overwriting globals in a library is clearly a good idea
stream.Stream.register_method("tcp", TCPStream)
stream.Stream.register_method("ssl", SSLStream)
<commit_msg>only register tables used by os-vif<commit_after>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import functools
import socket

from ovs.db import idl
from ovs import socket_util
from ovs import stream
from ovsdbapp.backend.ovs_idl import connection
from ovsdbapp.backend.ovs_idl import idlutils
from ovsdbapp.backend.ovs_idl import vlog
from ovsdbapp.schema.open_vswitch import impl_idl

from vif_plug_ovs.ovsdb import api

REQUIRED_TABLES = ('Interface', 'Port', 'Bridge', 'Open_vSwitch')

def idl_factory(config):
    conn = config.connection
    schema_name = 'Open_vSwitch'
    helper = idlutils.get_schema_helper(conn, schema_name)
    for table in REQUIRED_TABLES:
        helper.register_table(table)
    return idl.Idl(conn, helper)

def api_factory(config):
    conn = connection.Connection(
        idl=idl_factory(config),
        timeout=config.timeout)
    return NeutronOvsdbIdl(conn)

class NeutronOvsdbIdl(impl_idl.OvsdbIdl, api.ImplAPI):
    """IDL interface for OVS database back-end

    This class provides an OVSDB IDL (Open vSwitch Database Interface
    Definition Language) interface to the OVS back-end.
    """
    def __init__(self, conn):
        vlog.use_python_logger()
        super(NeutronOvsdbIdl, self).__init__(conn)

    def _get_table_columns(self, table):
        return list(self.tables[table].columns)

    def has_table_column(self, table, column):
        return column in self._get_table_columns(table)

# this is derived form https://review.opendev.org/c/openstack/neutron/+/794892
def add_keepalives(fn):
    @functools.wraps(fn)
    def _open(*args, **kwargs):
        error, sock = fn(*args, **kwargs)
        try:
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
        except socket.error as e:
            sock.close()
            return socket_util.get_exception_errno(e), None
        return error, sock
    return _open

class NoProbesMixin:
    @staticmethod
    def needs_probes():
        # If we are using keepalives, we can force probe_interval=0
        return False

class TCPStream(stream.TCPStream, NoProbesMixin):
    @classmethod
    @add_keepalives
    def _open(cls, suffix, dscp):
        return super()._open(suffix, dscp)

class SSLStream(stream.SSLStream, NoProbesMixin):
    @classmethod
    @add_keepalives
    def _open(cls, suffix, dscp):
        return super()._open(suffix, dscp)

# Overwriting globals in a library is clearly a good idea
stream.Stream.register_method("tcp", TCPStream)
stream.Stream.register_method("ssl", SSLStream)
<|endoftext|>


<commit_before>from openml.apiconnector import APIConnector
from scipy.io.arff import loadarff
import os
import numpy as np
import matplotlib.pylab as plt
from sklearn.cross_validation import KFold, StratifiedShuffleSplit
from sklearn.tree import DecisionTreeClassifier
from scipy.stats import randint as sp_randint
from sklearn.grid_search import GridSearchCV, RandomizedSearchCV
from sklearn.metrics import zero_one_loss, accuracy_score, confusion_matrix
import itertools
import seaborn

home_dir = os.path.expanduser("~")
openml_dir = os.path.join(home_dir, ".openml")
cache_dir = os.path.join(openml_dir, "cache")

with open(os.path.join(openml_dir, "apikey.txt"), 'r') as fh:
    key = fh.readline().rstrip('\n')
fh.close()

## load dataset lists
openml = APIConnector(cache_directory = cache_dir, apikey = key)
# datasets = openml.get_dataset_list()
# data = pd.DataFrame(datasets)

dataset = openml.download_dataset(32)
# print('Data-set name: %s'%dataset.name)
# print(dataset.description)
data, meta = loadarff(dataset.data_file)
target_attribute = dataset.default_target_attribute
target_attribute_names = meta[target_attribute][1]
X, y, attribute_names = dataset.get_dataset(target = target_attribute, return_attribute_names = True)

sss = StratifiedShuffleSplit(y, 1, test_size = 0.2, random_state = 100)
for train_index, test_index in sss:
    X_train, X_test = X[train_index], X[test_index]
    y_train, y_test = y[train_index], y[test_index]
    # build a classifier
    clf_cart = DecisionTreeClassifier()
    ## random search with optimization with nested resampling
    # specify parameters and distributions to sample from
    param_distribution = {"max_depth": [3, None],
                  "max_features": sp_randint(1, len(attribute_names)),
                  "min_samples_split": sp_randint(1, 11),
                  "min_samples_leaf": sp_randint(1, 11),
                  "criterion": ["gini", "entropy"]}
    # run randomized search
    n_iter_search = 64
    random_search = RandomizedSearchCV(clf_cart, param_distributions = param_distribution,
                                       n_iter = n_iter_search)
    random_search.fit(X_train, y_train)
    print('Best estimator: %s'%random_search.best_estimator_)
    # print('Grid_scores: %s'%random_search.grid_scores_)
    print('Accuracy: %s'%accuracy_score(y_test, random_search.predict(X_test)))

    ## random search with optimization without nested resampling
    # TODO, maybe manully set parameters to find the best
    '''
    # run randomized search
    n_iter_search = 64
    random_search = RandomizedSearchCV(DecisionTreeClassifier(),
                        param_distributions = param_distribution, n_iter = n_iter_search)
    random_search.fit(X_train, y_train)
    print('Best estimator: %s'%random_search.best_estimator_)
    print('Grid_scores: %s'%random_search.grid_scores_)
    print('Accuracy: %s'%accuracy_score(y_test, random_search.predict(X_test)))
    '''
    # use a full grid over all parameters
    param_grid = {"max_depth": [3, None],
                  "max_features": [1, 3, 10],
                  "min_samples_split": [1, 3, 10],
                  "min_samples_leaf": [1, 3, 10],
                  "criterion": ["gini", "entropy"]}
    grid_search = GridSearchCV(DecisionTreeClassifier(), param_grid = param_grid)
    grid_search.fit(X_train, y_train)
    print('Best estimator: %s'%grid_search.best_estimator_)
    # print('Grid_scores: %s'%grid_search.grid_scores_)
    print('Accuracy: %s'%accuracy_score(y_test, grid_search.predict(X_test)))<commit_msg>Add optimization without cross validation.<commit_after>from openml.apiconnector import APIConnector
from scipy.io.arff import loadarff
import os
import numpy as np
import matplotlib.pylab as plt
from sklearn.base import clone
from sklearn.cross_validation import StratifiedShuffleSplit
from sklearn.tree import DecisionTreeClassifier
from scipy.stats import randint as sp_randint
from sklearn.grid_search import GridSearchCV, RandomizedSearchCV, ParameterSampler
from sklearn.metrics import zero_one_loss, accuracy_score, confusion_matrix
import itertools
import seaborn

home_dir = os.path.expanduser("~")
openml_dir = os.path.join(home_dir, ".openml")
cache_dir = os.path.join(openml_dir, "cache")

with open(os.path.join(openml_dir, "apikey.txt"), 'r') as fh:
    key = fh.readline().rstrip('\n')
fh.close()

## load dataset lists
openml = APIConnector(cache_directory = cache_dir, apikey = key)
# datasets = openml.get_dataset_list()
# data = pd.DataFrame(datasets)

dataset = openml.download_dataset(32)
# print('Data-set name: %s'%dataset.name)
# print(dataset.description)
data, meta = loadarff(dataset.data_file)
target_attribute = dataset.default_target_attribute
target_attribute_names = meta[target_attribute][1]
X, y, attribute_names = dataset.get_dataset(target = target_attribute, return_attribute_names = True)

sss = StratifiedShuffleSplit(y, 1, test_size = 0.2, random_state = 100)
for train_index, test_index in sss:
    X_train, X_test = X[train_index], X[test_index]
    y_train, y_test = y[train_index], y[test_index]
    # build a classifier
    clf_cart = DecisionTreeClassifier()
    ## random search with optimization with nested resampling
    # specify parameters and distributions to sample from
    param_distribution = {"max_depth": [3, None],
                  "max_features": sp_randint(1, len(attribute_names)),
                  "min_samples_split": sp_randint(1, 11),
                  "min_samples_leaf": sp_randint(1, 11),
                  "criterion": ["gini", "entropy"]}
    # run randomized search
    n_iter_search = 64
    random_search = RandomizedSearchCV(clf_cart, param_distributions = param_distribution,
                                       n_iter = n_iter_search)
    random_search.fit(X_train, y_train)
    print('Best estimator: %s'%random_search.best_estimator_)
    # print('Grid_scores: %s'%random_search.grid_scores_)
    print('Accuracy: %s'%accuracy_score(y_test, random_search.predict(X_test)))

    ## random search with optimization without nested resampling
    clf = DecisionTreeClassifier()
    param_list = ParameterSampler(param_distribution, n_iter = n_iter_search)
    max_accuracy, opt_paras_index = 0.0, 0
    opt_clf = None
    for index, params in enumerate(param_list):
        clf.set_params(**params)
        clf.fit(X_train, y_train)
        clf_accuracy = accuracy_score(y_train, clf.predict(X_train))
        if clf_accuracy > max_accuracy:
            max_accuracy = clf_accuracy
            optimal_paras_index = index
            opt_clf = clone(clf)

    print('Best estimator: %s'%opt_clf)
    print('Accuracy: %s'%accuracy_score(y_test, opt_clf.fit(X_train, y_train).predict(X_test)))

    ## grid search
    # use a full grid over all parameters
    param_grid = {"max_depth": [3, None],
                  "max_features": [1, 3, 6, 10],
                  "min_samples_split": [1, 3, 6, 10],
                  "min_samples_leaf": [1, 3, 6, 10],
                  "criterion": ["gini", "entropy"]}
    grid_search = GridSearchCV(DecisionTreeClassifier(), param_grid = param_grid)
    grid_search.fit(X_train, y_train)
    print('Best estimator: %s'%grid_search.best_estimator_)
    # print('Grid_scores: %s'%grid_search.grid_scores_)
    print('Accuracy: %s'%accuracy_score(y_test, grid_search.predict(X_test)))<|endoftext|>