<commit_before># coding: utf-8
import json
from flask import Flask, request
import sys
from musica import *
from collections import OrderedDict
import unicodedata

app = Flask(__name__)

ARTISTA_ID = 0
MUSICA_ID = 1
ARTISTA = 2
MUSICA = 3
GENERO = 4
POPULARIDADE = 5
TOM = 6
SEQ_FAMOSA = 7
CIFRA = 8
TAM_PAGINA = 100

def init():
    reload(sys)
    sys.setdefaultencoding('utf8')

    f = open('data/top/dataset_final.csv')
    f.readline()

    global generos
    global musicas
    global sequencias
    global acordes
    global genero_musicas

    genero_musicas = {}
    generos = set()
    acordes = set()
    sequencias = {'BmGDA' : 0,
            'CGAmF' : 1,
            'EmG' : 2,
            'CA7DmG7' : 3,
            'GmF' : 4,
            'CC7FFm' : 5}

    musicas_dict = {}
    for line in f:
        line = line.replace('"', '').replace('NA', '')[:-1]

        musica = line.split(',')
        musica[POPULARIDADE] = int(musica[POPULARIDADE].replace('.', ''))

        if musica[CIFRA] != '':
            musica[CIFRA] = limpa_cifra(musica[CIFRA].split(';'))
        else:
            musica[CIFRA] = []

        musica[SEQ_FAMOSA] = musica[SEQ_FAMOSA].split(";")

        # conjunto único de gêneros
        generos.add(musica[GENERO])

        # inclui música no dict de músicas
        musica_obj = Musica(musica[ARTISTA_ID], musica[ARTISTA],
                musica[MUSICA_ID], musica[MUSICA], musica[GENERO],
                int(musica[POPULARIDADE]), musica[SEQ_FAMOSA],
                musica[TOM], musica[CIFRA])

        musicas_dict[musica_obj.id_unico_musica] = musica_obj

        # conjunto único de acordes
        for acorde in musica_obj.acordes:
            acordes.add(acorde)

        # constrói dict mapeando gênero para músicas
        # deve ser usado para melhorar o desempenho das buscas
        if musica_obj.genero in genero_musicas:
            genero_musicas[musica_obj.genero].append(musica_obj)
        else:
            genero_musicas[musica_obj.genero] = [musica_obj]

    # dicionário de músicas cujos valores são ordenados por popularidade
    musicas = OrderedDict(sorted(musicas_dict.items(),
        key=lambda x: x[1].popularidade, reverse = True))

    # para trabalhar melhor com json
    generos = list(generos)

    # ordena genero_musicas por popularidade
    for k,v in genero_musicas.items():
        genero_musicas[k].sort(key = lambda x : x.popularidade, reverse = True)

    f.close()

def limpa_cifra(raw_cifra):
    cifra = []
    for m in raw_cifra:
        if m.strip() != '':
            # filtra tablaturas
            if '|' in m:
                acorde = m.split('|')[0].split()[0]
                cifra.append(acorde)
            # lida com acordes separados por espaço
            else:
                tokens = [token for token in m.split()]
                cifra += tokens
    return cifra

''' Busca por músicas que possuem no título ou no nome do artista o argumento passado por key.
    params: key e generos (opcional). Caso generos não sejam definidos, a busca não irá filtrar por gênero.
    exemplo 1: /search?key=no dia em que eu saí de casa
    exemplo 2: /search?key=no dia em que eu saí de casa&generos=Rock,Samba '''
@app.route('/search')
def busca():
    generos_tag = request.args.get('generos', [])
    pagina_tag = request.args.get('pagina','1')
    keys = request.args.get('key').lower()
    keys = remover_combinantes(keys).split(' ')

    generos_key = generos
    if generos_tag:
        generos_key = generos_tag.encode('utf-8').split(',')

    collection = apply_filtro(musicas.values, generos_key)
    out = []

    for musica in collection:
        text = '%s %s' % (musica.nome_artista.lower(), musica.nome_musica.lower())
        text_list = remover_combinantes(unicode(text)).split(' ')
        if all(key in text_list for key in keys):
            matches = {
                'id_unico_musica' : musica.id_unico_musica,
                'id_artista' : musica.id_artista,
                'id_musica' : musica.id_musica,
                'nome_artista' : musica.nome_artista,
                'nome_musica' : musica.nome_musica,
                'genero' : musica.genero,
                'url' : musica.url,
            }
            out.append(matches)
    return json.dumps(get_pagina(out, pagina_tag))

# cópia
def remover_combinantes(string):
    string = unicodedata.normalize('NFD', string)
    return u''.join(ch for ch in string if unicodedata.category(ch) != 'Mn')

''' Retorna as músicas armazenadas no sistema (ordenados por popularidade).
    O serviço é paginado. Cada página tem tamanho 100, por default.
    params: pagina. Caso não seja definida a página, o valor default é 1.
    exemplo 1: /musica?pagina=2
    exemplo 2: /musica'''
@app.route('/musicas')
def get_musicas():
    return json.dumps([v.__dict__ for v in musicas.values()])

@app.route('/generos')
def get_generos():
    return json.dumps(generos)

@app.route('/acordes')
def get_acordes():
    return json.dumps(list(acordes))

@app.route('/musica/<m_id>/')
def get_musica(m_id):
    return json.dumps(musicas[m_id].__dict__)

@app.route('/similares')
def get_similares():
    # tratando request
    acordes_tag = request.args.get('acordes')
    id_musica_tag = request.args.get('id_unico_musica')
    sequencia_tag = request.args.get('sequencia')
    pagina_tag = request.args.get('pagina','1')

    # se não existir, filtra por todos.
    generos_tag = request.args.get('generos')
    generos_key = generos
    if generos_tag:
        generos_key = generos_tag.encode('utf-8').split(',')

    acordes = []
    if acordes_tag:
        acordes = acordes_tag.encode('utf-8').split(',')
    elif id_musica_tag:
        musica = musicas[id_musica_tag]
        acordes = musica.acordes
    elif sequencia_tag:
        acordes = sequencia_tag.encode('utf-8').replace(',','')
        similares = []

        if acordes in sequencias:
            id_seq = sequencias[acordes]
            similares = get_pagina(get_similares_por_sequencia(id_seq, generos_key), pagina_tag)
        return json.dumps(similares)

    similares = get_similares(acordes, generos_key)

    return json.dumps(get_pagina(similares, pagina_tag))

def get_similares_por_sequencia(id_seq, generos_key):
    # filtra para melhor desempenho
    collection = apply_filtro(musicas.values(), generos_key)

    similares = []
    for musica in collection:
        if str(id_seq) in musica.seqs_famosas:
            similar = {
                'id_unico_musica' : musica.id_unico_musica,
                'id_artista' : musica.id_artista,
                'id_musica' : musica.id_musica,
                'nome_artista' : musica.nome_artista,
                'nome_musica' : musica.nome_musica,
                'popularidade' : musica.popularidade,
                'acordes' : musica.acordes,
                'genero' : musica.genero,
                'url' : musica.url,
            }
            similares.append(similar)

    return similares

def get_pagina(colecao, pagina_tag):
    sl = (int(pagina_tag) - 1)*TAM_PAGINA
    return colecao[sl:sl+TAM_PAGINA]

def get_similares(acordes, generos_key):

    # filtra para melhor desempenho
    collection = apply_filtro(musicas.values(), generos_key)

    similares = []
    for musica in collection:
        inter = set(acordes).intersection(set(musica.acordes))
        diff = set(musica.acordes) - set(acordes)

        # somente as que tiverem interseção e as que forem
        # dos generos solicitados.
        if len(inter) > 0:
            similar = {
                    'id_unico_musica' : musica.id_unico_musica,
                    'id_artista' : musica.id_artista,
                    'id_musica' : musica.id_musica,
                    'nome_artista' : musica.nome_artista,
                    'nome_musica' : musica.nome_musica,
                    'popularidade' : musica.popularidade,
                    'acordes' : musica.acordes,
                    'genero' : musica.genero,
                    'url' : musica.url,
                    'diferenca' : list(diff),
                    'intersecao' : list(inter)
            }

            similares.append(similar)

    # ordenados por menor diferença, maior interseção e maior popularidade.
    return sorted(similares, key=lambda x: (len(x['diferenca']), -len(x['intersecao'])))

## Filtra a coleção de músicas por gênero.
def apply_filtro(musicas, generos_key):
    # filtra para melhor desempenho
    collection = []
    for genero in generos_key:
        if genero in generos:
            collection += genero_musicas[genero]
    return collection

@app.after_request
def add_header(response):
    response.headers['Access-Control-Allow-Origin'] = '*'
    return response

if __name__ == '__main__':
    init()
    app.run(debug=True)
<commit_msg>heroku feelings<commit_after># coding: utf-8
import json
from flask import Flask, request
import sys
from musica import *
from collections import OrderedDict
import unicodedata

app = Flask(__name__)

ARTISTA_ID = 0
MUSICA_ID = 1
ARTISTA = 2
MUSICA = 3
GENERO = 4
POPULARIDADE = 5
TOM = 6
SEQ_FAMOSA = 7
CIFRA = 8
TAM_PAGINA = 100

def init():
    reload(sys)
    sys.setdefaultencoding('utf8')

    f = open('data/top/dataset_final.csv')
    f.readline()

    global generos
    global musicas
    global sequencias
    global acordes
    global genero_musicas

    genero_musicas = {}
    generos = set()
    acordes = set()
    sequencias = {'BmGDA' : 0,
            'CGAmF' : 1,
            'EmG' : 2,
            'CA7DmG7' : 3,
            'GmF' : 4,
            'CC7FFm' : 5}

    musicas_dict = {}
    for line in f:
        line = line.replace('"', '').replace('NA', '')[:-1]

        musica = line.split(',')
        musica[POPULARIDADE] = int(musica[POPULARIDADE].replace('.', ''))

        if musica[CIFRA] != '':
            musica[CIFRA] = limpa_cifra(musica[CIFRA].split(';'))
        else:
            musica[CIFRA] = []

        musica[SEQ_FAMOSA] = musica[SEQ_FAMOSA].split(";")

        # conjunto único de gêneros
        generos.add(musica[GENERO])

        # inclui música no dict de músicas
        musica_obj = Musica(musica[ARTISTA_ID], musica[ARTISTA],
                musica[MUSICA_ID], musica[MUSICA], musica[GENERO],
                int(musica[POPULARIDADE]), musica[SEQ_FAMOSA],
                musica[TOM], musica[CIFRA])

        musicas_dict[musica_obj.id_unico_musica] = musica_obj

        # conjunto único de acordes
        for acorde in musica_obj.acordes:
            acordes.add(acorde)

        # constrói dict mapeando gênero para músicas
        # deve ser usado para melhorar o desempenho das buscas
        if musica_obj.genero in genero_musicas:
            genero_musicas[musica_obj.genero].append(musica_obj)
        else:
            genero_musicas[musica_obj.genero] = [musica_obj]

    # dicionário de músicas cujos valores são ordenados por popularidade
    musicas = OrderedDict(sorted(musicas_dict.items(),
        key=lambda x: x[1].popularidade, reverse = True))

    # para trabalhar melhor com json
    generos = list(generos)

    # ordena genero_musicas por popularidade
    for k,v in genero_musicas.items():
        genero_musicas[k].sort(key = lambda x : x.popularidade, reverse = True)

    f.close()

def limpa_cifra(raw_cifra):
    cifra = []
    for m in raw_cifra:
        if m.strip() != '':
            # filtra tablaturas
            if '|' in m:
                acorde = m.split('|')[0].split()[0]
                cifra.append(acorde)
            # lida com acordes separados por espaço
            else:
                tokens = [token for token in m.split()]
                cifra += tokens
    return cifra

''' Busca por músicas que possuem no título ou no nome do artista o argumento passado por key.
    params: key e generos (opcional). Caso generos não sejam definidos, a busca não irá filtrar por gênero.
    exemplo 1: /search?key=no dia em que eu saí de casa
    exemplo 2: /search?key=no dia em que eu saí de casa&generos=Rock,Samba '''
@app.route('/search')
def busca():
    generos_tag = request.args.get('generos', [])
    pagina_tag = request.args.get('pagina','1')
    keys = request.args.get('key').lower()
    keys = remover_combinantes(keys).split(' ')

    generos_key = generos
    if generos_tag:
        generos_key = generos_tag.encode('utf-8').split(',')

    collection = apply_filtro(musicas.values, generos_key)
    out = []

    for musica in collection:
        text = '%s %s' % (musica.nome_artista.lower(), musica.nome_musica.lower())
        text_list = remover_combinantes(unicode(text)).split(' ')
        if all(key in text_list for key in keys):
            matches = {
                'id_unico_musica' : musica.id_unico_musica,
                'id_artista' : musica.id_artista,
                'id_musica' : musica.id_musica,
                'nome_artista' : musica.nome_artista,
                'nome_musica' : musica.nome_musica,
                'genero' : musica.genero,
                'url' : musica.url,
            }
            out.append(matches)
    return json.dumps(get_pagina(out, pagina_tag))

# cópia
def remover_combinantes(string):
    string = unicodedata.normalize('NFD', string)
    return u''.join(ch for ch in string if unicodedata.category(ch) != 'Mn')

''' Retorna as músicas armazenadas no sistema (ordenados por popularidade).
    O serviço é paginado. Cada página tem tamanho 100, por default.
    params: pagina. Caso não seja definida a página, o valor default é 1.
    exemplo 1: /musica?pagina=2
    exemplo 2: /musica'''
@app.route('/musicas')
def get_musicas():
    return json.dumps([v.__dict__ for v in musicas.values()])

@app.route('/generos')
def get_generos():
    return json.dumps(generos)

@app.route('/acordes')
def get_acordes():
    return json.dumps(list(acordes))

@app.route('/musica/<m_id>/')
def get_musica(m_id):
    return json.dumps(musicas[m_id].__dict__)

@app.route('/similares')
def get_similares():
    # tratando request
    acordes_tag = request.args.get('acordes')
    id_musica_tag = request.args.get('id_unico_musica')
    sequencia_tag = request.args.get('sequencia')
    pagina_tag = request.args.get('pagina','1')

    # se não existir, filtra por todos.
    generos_tag = request.args.get('generos')
    generos_key = generos
    if generos_tag:
        generos_key = generos_tag.encode('utf-8').split(',')

    acordes = []
    if acordes_tag:
        acordes = acordes_tag.encode('utf-8').split(',')
    elif id_musica_tag:
        musica = musicas[id_musica_tag]
        acordes = musica.acordes
    elif sequencia_tag:
        acordes = sequencia_tag.encode('utf-8').replace(',','')
        similares = []

        if acordes in sequencias:
            id_seq = sequencias[acordes]
            similares = get_pagina(get_similares_por_sequencia(id_seq, generos_key), pagina_tag)
        return json.dumps(similares)

    similares = get_similares(acordes, generos_key)

    return json.dumps(get_pagina(similares, pagina_tag))

def get_similares_por_sequencia(id_seq, generos_key):
    # filtra para melhor desempenho
    collection = apply_filtro(musicas.values(), generos_key)

    similares = []
    for musica in collection:
        if str(id_seq) in musica.seqs_famosas:
            similar = {
                'id_unico_musica' : musica.id_unico_musica,
                'id_artista' : musica.id_artista,
                'id_musica' : musica.id_musica,
                'nome_artista' : musica.nome_artista,
                'nome_musica' : musica.nome_musica,
                'popularidade' : musica.popularidade,
                'acordes' : musica.acordes,
                'genero' : musica.genero,
                'url' : musica.url,
            }
            similares.append(similar)

    return similares

def get_pagina(colecao, pagina_tag):
    sl = (int(pagina_tag) - 1)*TAM_PAGINA
    return colecao[sl:sl+TAM_PAGINA]

def get_similares(acordes, generos_key):

    # filtra para melhor desempenho
    collection = apply_filtro(musicas.values(), generos_key)

    similares = []
    for musica in collection:
        inter = set(acordes).intersection(set(musica.acordes))
        diff = set(musica.acordes) - set(acordes)

        # somente as que tiverem interseção e as que forem
        # dos generos solicitados.
        if len(inter) > 0:
            similar = {
                    'id_unico_musica' : musica.id_unico_musica,
                    'id_artista' : musica.id_artista,
                    'id_musica' : musica.id_musica,
                    'nome_artista' : musica.nome_artista,
                    'nome_musica' : musica.nome_musica,
                    'popularidade' : musica.popularidade,
                    'acordes' : musica.acordes,
                    'genero' : musica.genero,
                    'url' : musica.url,
                    'diferenca' : list(diff),
                    'intersecao' : list(inter)
            }

            similares.append(similar)

    # ordenados por menor diferença, maior interseção e maior popularidade.
    return sorted(similares, key=lambda x: (len(x['diferenca']), -len(x['intersecao'])))

## Filtra a coleção de músicas por gênero.
def apply_filtro(musicas, generos_key):
    # filtra para melhor desempenho
    collection = []
    for genero in generos_key:
        if genero in generos:
            collection += genero_musicas[genero]
    return collection

@app.after_request
def add_header(response):
    response.headers['Access-Control-Allow-Origin'] = '*'
    return response

if __name__ == '__main__':
    app.run(debug=True)
    init()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

import pyspeckit as p
import matplotlib.pyplot as plt, matplotlib.mlab as mlab
import numpy as np
from scipy.stats import norm
from os import path
import pdb

def equivalent_width(filename, xmin, xmax, exclude_min, exclude_max, n, fldr=None, name=None):
    """"
    This is the main function to be invoked by the user. This function gathers the spectrum and metadata, sets up the output PDF figure, and calls the measure_equivalent_width() function to perform the calculation.

    Args:
    ----------
    filename - String, spectrum file name with full path
    xmin,xmax - Integers, the specified interval in wavelength space, which defines the region of interest
    excludemin, excludemax - Integers, the specified interval in wavelength space of the spectral feature, which binds the edges of the spectral feature itself
    n - Integer, the number of times the EqW measurement is repeated in the MCMC step
    fldr - String, location where output figure is desired; if None, figure is saved in current folder
    name - String, if not None, it uses it to label the object

    Returns:
    -------
    - the mean and standard deviation of the equivalent width measured n times
    - A figure with a plot of the full spectrum; a plot of the spectral feature fit, with the Voigt profile line fit (blue), the pseudo-continuum (orange), and the approximated rectangle (green); and a histogram with the MCMC results of the EqW distribution
    """

    # Check that inputs are valid ---------------------------------------------
    # fldr
    if fldr is None:
        fldr = './'
    else:
        if not path.exists(fldr):
            fldr = './'

    # Invoke pyspeckit to perform equivalent width measurement ----------------
    mu, sigma = measure_equivalent_width(filename, xmin, xmax, exclude_min, exclude_max, n)

    # Save figure -------------------------------------------------------------
    fig = plt.gcf()
    plt.savefig(fldr + name + '_EWfit.pdf')

    return np.array([mu, sigma])

def measure_equivalent_width(filename, xmin, xmax, exclude_min, exclude_max, n):
    """Calculate the equivalent width of an absorption or emission line for a given spectrum using PySpecKit. By: Munazza Alam

    Args:
    =======
    xmin,xmax - the specified interval of the spectrum to plot
    excludemin, excludemax - the specified interval (in wavelength space) of the absorption feature
    n - the number of Monte Carlo iterations

    Returns:
    =======
    - the mean and standard deviation of the equivalent width measured n times
    - the spectrum plotted with the Voigt profile line fit (blue), the pseudo-continuum (yellow),
    and the approximated rectangle (green)
    - a histogram of the EqW distribution
    """
    sp = p.Spectrum(filename) # This has trouble loading flux error dimension
    sp.unit = 'Angstrom'
    sp.xarr.xtype = 'wavelength'

    # Get an estimate of flux error from noise in continuum
    icont = np.where(((sp.xarr.value >= xmin) & (sp.xarr.value < exclude_min)) |  \
                    ((sp.xarr.value > exclude_max) & (sp.xarr.value <= xmax)))[0]
    tmpcont = sp.flux[icont]
    tmperr = np.std(tmpcont)

    sp.plotter(xmin=xmin, xmax=xmax, ymin=0, errstyle='bars',
               color='grey')
    sp.baseline(xmin=xmin, xmax=xmax,
                exclude=[exclude_min,exclude_max],
                subtract=False, highlight_fitregion=False,
                selectregion=True, order=0)
    sp.specfit(plot=True, fittype='voigt', color='blue',
               guesses='moments', vheight=True)
    sp.specfit.EQW(plot=True, plotcolor='g', fitted=False, components=False,
                   annotate=True, loc='lower left', xmin=None, xmax=None)

    sp2 = sp.copy()
    EQWs = []

    for w in range(n):
        sp2.data = sp.data + np.random.randn(sp.data.size)*tmperr
        sp2.baseline(xmin=xmin, xmax=xmax, exclude=[exclude_min,exclude_max],
                     subtract=False, highlight_fitregion=False,
                     selectregion=True, order=0)
        sp2.specfit(fittype='voigt', guesses=sp.specfit.parinfo.values)
        dist = sp2.specfit.EQW(plotcolor='g', fitted=False, components=False,
                               annotate=True, loc='lower left', xmin=None, xmax=None)

        EQWs.append(dist)
    EQWs = np.array(EQWs)
    sp.specfit.EQW(plot=True, plotcolor='g', fitted=False,
                   components=False, annotate=True,
                   loc='lower left', xmin=None, xmax=None)

    # sp2 = sp.copy()
    # EQWs = np.zeros(n)

    # for w in range(n):
    #     sp2.data = sp.data + np.random.randn(sp.data.size)*sp.error
    #     sp2.baseline(xmin=xmin, xmax=xmax,
    #                  exclude=[exclude_min,exclude_max],
    #                  subtract=False, highlight_fitregion=False,
    #                  selectregion=True, order=0)
    #     sp2.specfit(fittype='voigt', guesses=sp.specfit.parinfo.values)
    #     dist = sp2.specfit.EQW(plotcolor='g', fitted=False,
    #                            components=False, annotate=True,
    #                            loc='lower left', xmin=None, xmax=None)

    fig = plt.figure()
    mu,sigma = norm.fit(EQWs)
    print(mu, sigma)
    ax = fig.add_subplot(111)

    num,bins,patches = ax.hist(EQWs, 10, facecolor='green', density=True, histtype='stepfilled')
    y = norm.pdf(bins,mu,sigma)
    ax.plot(bins,y,'r--',linewidth=2)
    ax.grid(True)
    ax.set_ylabel('Probability')
    ax.set_xlabel('EQW')
    plt.show()
    num_bins = np.floor(np.log10(n)) * 10
    pdb.set_trace()
    # Light blue histogram for contrast and for R/G colorblind folks
    n,bins,patches = plt.hist(EQWs, int(num_bins), density=True,
                              facecolor='lightblue',
                              histtype='stepfilled')
    y = norm.pdf(bins,mu,sigma)
    ax.plot(bins,y,'r--',linewidth=2)

    # Plot the median and 68th percentile values for comparison
    perc = np.percentile(EQWs, [16, 50, 84])
    ax = plt.gca()
    for p_value in perc:
        ax.axvline(p_value,linestyle=":",color="k",lw=1.5)

    plt.ylabel('Probability')
    plt.xlabel('EQW')
    plt.show()

    return mu, sigma

if __name__=="__main__":
    # xmin,xmax,exclude_min,exclude_max,n
    equivalent_width('2m1821_61_08jun11.txt',1.250, 1.255, 1.2514,
                     1.2538, 1000)

def final_plots(filename,xmin,xmax,exclude_min,exclude_max):
    vf = p.spectrum.models.inherited_voigtfitter.voigt_fitter()

    sp = p.Spectrum(filename)
    sp.xarr.units = 'micron'
    sp.xarr.xtype = 'wavelength'
    sp.plotter(xmin=xmin, xmax=xmax, ymin=0, errstyle='fill',color='grey')
    sp.baseline(xmin=xmin, xmax=xmax,exclude=[exclude_min,exclude_max],subtract=False,
                reset_selection=False,hightlight_fitregion=False,order=0)
    sp.specfit(plot=True, fittype='voigt', color='magenta', guesses='moments',
               vheight=True)
    fwhm = sp.specfit.measure_approximate_fwhm(threshold='error', emission=False,
                                               interpolate_factor=1024, plot=True,
                                               grow_threshold=1, color='magenta')
    sp.plotter.refresh()
    sp.specfit.EQW(plot=True, plotcolor='g', fitted=False, continuum=0.5, components=False, annotate=True, loc='lower left', xmin=None,
    xmax=None)
    sp.plotter.refresh()
    xarr_fit_units = 'microns'
    plt.ylabel('Normalized Flux')
    plt.xlabel('Wavelength ($\mu m$)')<commit_msg>Separated into a new function the routing of reading file spectrum with pyspeckit. Added the option for user to input the spectrum already loaded into an array instead of providing filename. Added some input quality checks. Other minor fixes. Code still assumes spectrum is in Angstrom. Figure output has not changed.<commit_after>#!/usr/bin/env python
# encoding: utf-8

import pyspeckit as p
import matplotlib.pyplot as plt, matplotlib.mlab as mlab
import numpy as np
from scipy.stats import norm
import astropy.io.fits as pf
from os import path
import pdb

def equivalent_width(spec, xmin, xmax, exclude_min, exclude_max, n, fldr=None, name=None):
    """"
    This is the main function to be invoked by the user. This function gathers the spectrum and metadata, sets up the output PDF figure, and calls the measure_equivalent_width() function to perform the calculation.

    Args:
    ----------
    spec - Numpy Array, List or String; if array or list, it should contain the spectrum with wavelength in Angstrom, flux, and optional flux errors as columns; if string, it should contain the filename with full path of the spectrum file
    xmin,xmax - Integers, the specified interval in wavelength space, which defines the region of interest
    excludemin, excludemax - Integers, the specified interval in wavelength space of the spectral feature, which binds the edges of the spectral feature itself
    n - Integer, the number of times the EqW measurement is repeated in the MCMC step
    fldr - String, location where output figure is desired; if None, figure is saved in current folder
    name - String, if not None, it uses it to label the object

    Returns:
    -------
    - the mean and standard deviation of the equivalent width measured n times
    - A figure with a plot of the full spectrum; a plot of the spectral feature fit, with the Voigt profile line fit (blue), the pseudo-continuum (orange), and the approximated rectangle (green); and a histogram with the MCMC results of the EqW distribution
    """

    # Check that inputs are valid ---------------------------------------------
    # Is spec a filename?
    if isinstance(spec, str):
        filename = spec
    else:
        filename = None
        # Is spec an array?
        specarr = None
        if isinstance(spec, (list, np.ndarray)):
            specarr = list(spec)
        else:
            # spec is neither a filename nor a list with spectrum
            if filename is None:
                print('ERROR: spec parameter has the wrong format.')
                return
        # Is the array an appropriate one?
        if len(specarr) not in (2,3):
            print('ERROR: spec parameter has the wrong format.')
            return

    # Is fldr included?
    if fldr is None:
        fldr = './'
    else:
        if not path.exists(fldr):
            fldr = './'

    sp = None

    # Attempt to load spectrum using input filename ---------------------------
    if filename is not None:
        sp = readspec(filename)
        if sp == 1:
            return

    # Get spectrum data from input array --------------------------------------
    # (Note: the empty header pf.Header() is added to avoid a pyspeckit warning message)
    if sp is None:
        if len(specarr) < 3:
            # Only wavelength and flux
            sp = p.Spectrum(data=specarr[1], xarr=specarr[0], xarrkwargs={'unit':'Angstrom'}, \
                            header=pf.Header())
        else:
            # Wavelength, flux, and flux errors
            sp = p.Spectrum(data=specarr[1], xarr=specarr[0], error=specarr[2], \
                            xarrkwargs={'unit':'Angstrom'}, header=pf.Header())
        sp.xarr.xtype = 'wavelength'

    # Invoke pyspeckit to perform equivalent width measurement ----------------
    result = measure_equivalent_width(sp, xmin, xmax, exclude_min, exclude_max, n)
    if result is None:
        return

    # Save figure -------------------------------------------------------------
    fig = plt.gcf()
    plt.savefig(fldr + name + '_EWfit.pdf')

    return result

def measure_equivalent_width(sp, xmin, xmax, exclude_min, exclude_max, n):
    """Calculate the equivalent width of an absorption or emission line for a given spectrum using PySpecKit. By: Munazza Alam

    Args:
    =======
    sp - pyspeckit instance of class Spectrum
    xmin,xmax - the specified interval of the spectrum to plot
    excludemin, excludemax - the specified interval (in wavelength space) of the absorption feature
    n - the number of Monte Carlo iterations

    Returns:
    =======
    - the mean and standard deviation of the equivalent width measured n times
    - the spectrum plotted with the Voigt profile line fit (blue), the pseudo-continuum (yellow),
    and the approximated rectangle (green)
    - a histogram of the EqW distribution
    """

    # Get an estimate of flux error from noise in continuum
    icont = np.where(((sp.xarr.value >= xmin) & (sp.xarr.value < exclude_min)) |  \
                    ((sp.xarr.value > exclude_max) & (sp.xarr.value <= xmax)))[0]
    tmpcont = sp.flux[icont]
    tmperr = np.std(tmpcont)

    sp.plotter(xmin=xmin, xmax=xmax, ymin=0, errstyle='bars',
               color='grey')
    sp.baseline(xmin=xmin, xmax=xmax,
                exclude=[exclude_min,exclude_max],
                subtract=False, highlight_fitregion=False,
                selectregion=True, order=0)
    sp.specfit(plot=True, fittype='voigt', color='blue',
               guesses='moments', vheight=True)
    sp.specfit.EQW(plot=True, plotcolor='g', fitted=False, components=False,
                   annotate=True, loc='lower left', xmin=None, xmax=None)

    sp2 = sp.copy()
    EQWs = []

    for w in range(n):
        sp2.data = sp.data + np.random.randn(sp.data.size)*tmperr
        sp2.baseline(xmin=xmin, xmax=xmax, exclude=[exclude_min,exclude_max],
                     subtract=False, highlight_fitregion=False,
                     selectregion=True, order=0)
        sp2.specfit(fittype='voigt', guesses=sp.specfit.parinfo.values)
        dist = sp2.specfit.EQW(plotcolor='g', fitted=False, components=False,
                               annotate=True, loc='lower left', xmin=None, xmax=None)

        EQWs.append(dist)
    EQWs = np.array(EQWs)
    sp.specfit.EQW(plot=True, plotcolor='g', fitted=False,
                   components=False, annotate=True,
                   loc='lower left', xmin=None, xmax=None)

    fig = plt.figure()
    mu,sigma = norm.fit(EQWs)
    print(mu, sigma)
    ax = fig.add_subplot(111)

    num,bins,patches = ax.hist(EQWs, 10, facecolor='green', density=True, histtype='stepfilled')
    y = norm.pdf(bins,mu,sigma)
    ax.plot(bins,y,'r--',linewidth=2)
    ax.grid(True)
    ax.set_ylabel('Probability')
    ax.set_xlabel('EW')
    plt.show()
    num_bins = np.floor(np.log10(n)) * 10

    # Light blue histogram for contrast and for R/G colorblind folks
    n,bins,patches = plt.hist(EQWs, int(num_bins), density=True,
                              facecolor='lightblue',
                              histtype='stepfilled')
    y = norm.pdf(bins,mu,sigma)
    ax.plot(bins,y,'r--',linewidth=2)

    # Plot the median and 68th percentile values for comparison
    perc = np.percentile(EQWs, [16, 50, 84])
    ax = plt.gca()
    for p_value in perc:
        ax.axvline(p_value,linestyle=":",color="k",lw=1.5)

    plt.ylabel('Probability')
    plt.xlabel('EW')
    plt.show()

    return [mu, sigma]

def readspec(fname):
    """ Uses pyspeckit to attempt to read fits file of spectrum and load it into an instance of class Spectrum."""

    try:
        sp = p.Spectrum(fname) # This has trouble loading flux error dimension sometimes
    except FileNotFoundError:
        print('ERROR: Spectrum file not found.')
        return 1
    except TypeError:
        print('ERROR: Spectrum file could not be loaded.')
        return 1

    sp.xarr.xtype = 'wavelength'
    if sp.xarr.unit != 'Angstrom':
        sp.xarr.convert_to_unit('Angstrom')

    return sp

if __name__=="__main__":
    # xmin,xmax,exclude_min,exclude_max,n
    equivalent_width('2m1821_61_08jun11.txt',1.250, 1.255, 1.2514,
                     1.2538, 1000)
                     <|endoftext|>


<commit_before>import argparse
import json
import logging
import logging.config
import os

import paho.mqtt.client as mqtt
import yaml

from services.data_service import DataService
from services.watering_service import WateringService
from services.config_service import ConfigService

def load_args():
    # setup commandline argument parser
    parser = argparse.ArgumentParser()
    parser.add_argument('--env')

    return parser.parse_args()

def setup_logging(default_level=logging.INFO):
    path = os.path.join(os.getcwd(), 'config', 'logging.yml')

    if os.path.exists(path):
        # load from config
        with open(path, 'rt') as f:
            config = yaml.safe_load(f.read())
        logging.config.dictConfig(config)
    else:
        logging.basicConfig(level=default_level)

def create_mqtt_client(config):
    client = mqtt.Client()
    client.on_connect = on_connect
    client.on_message = on_message

    user = config['user']

    if user is not None:
        # use authentication
        client.username_pw_set(user, config['password'])

    client.connect(config['host'], config['port'])

    return client

def watering(sensors_id, milliseconds):
    mqtt_client.publish(watering_config['topic'], milliseconds)
    data_service.save_watering(sensors_id, milliseconds)

def on_connect(client, userdata, flags_dict, rc):
    if rc != 0:
        logger.error('MQTT connection error: ' + str(rc))
        return

    logger.info('MQTT connected')

    # Subscribing in on_connect() means that if we lose the connection and
    # reconnect then subscriptions will be renewed.
    client.subscribe(mqtt_config['topic'])

def on_message(client, userdata, msg):
    logger.info('receive message "%s": %s', msg.topic, str(msg.payload))

    # transform payload to JSON
    sensor_values = json.loads(msg.payload.decode('utf-8'))

    temperature = sensor_values['Temperature']
    humidity = sensor_values['Humidity']
    soil_moisture = sensor_values['SoilMoisture']

    sensors_id = data_service.save_sensor_values(temperature, humidity, soil_moisture)

    if sensors_id is not None:
        watering_milliseconds = watering_service.calculate_milliseconds(soil_moisture)

        if watering_milliseconds > 200:
            watering(sensors_id, watering_milliseconds)

args = load_args()
setup_logging()
logger = logging.getLogger(__name__)

logger.info('starting MQTT client')

try:
    config_service = ConfigService(args.env)
    mqtt_config = config_service.get_section('mqtt')
    mysql_config = config_service.get_section('mysql')
    watering_config = config_service.get_section('watering')

    data_service = DataService(mysql_config)
    watering_service = WateringService(watering_config)

    mqtt_client = create_mqtt_client(mqtt_config)
    mqtt_client.loop_forever()
except Exception as error:
    logger.error('main error', exc_info=True)
<commit_msg>convert sensor values to int<commit_after>import argparse
import json
import logging
import logging.config
import os

import paho.mqtt.client as mqtt
import yaml

from services.data_service import DataService
from services.watering_service import WateringService
from services.config_service import ConfigService

def load_args():
    # setup commandline argument parser
    parser = argparse.ArgumentParser()
    parser.add_argument('--env')

    return parser.parse_args()

def setup_logging(default_level=logging.INFO):
    path = os.path.join(os.getcwd(), 'config', 'logging.yml')

    if os.path.exists(path):
        # load from config
        with open(path, 'rt') as f:
            config = yaml.safe_load(f.read())
        logging.config.dictConfig(config)
    else:
        logging.basicConfig(level=default_level)

def create_mqtt_client(config):
    client = mqtt.Client()
    client.on_connect = on_connect
    client.on_message = on_message

    user = config['user']
    if user is not None:
        # use authentication
        client.username_pw_set(user, config['password'])

    client.connect(config['host'], config['port'])

    return client

def watering(sensors_id, milliseconds):
    mqtt_client.publish(watering_config['topic'], milliseconds)
    data_service.save_watering(sensors_id, milliseconds)

def on_connect(client, userdata, flags_dict, rc):
    if rc != 0:
        logger.error('MQTT connection error: ' + str(rc))
        return

    logger.info('MQTT connected')

    # Subscribing in on_connect() means that if we lose the connection and
    # reconnect then subscriptions will be renewed.
    client.subscribe(mqtt_config['topic'])

def on_message(client, userdata, msg):
    logger.info('receive message "%s": %s', msg.topic, str(msg.payload))

    # transform payload to JSON
    sensor_values = json.loads(msg.payload.decode('utf-8'))

    temperature = int(sensor_values['Temperature'])
    humidity = int(sensor_values['Humidity'])
    soil_moisture = int(sensor_values['SoilMoisture'])

    sensors_id = data_service.save_sensor_values(temperature, humidity, soil_moisture)

    if sensors_id is not None:
        watering_milliseconds = watering_service.calculate_milliseconds(soil_moisture)

        if watering_milliseconds > 200:
            watering(sensors_id, watering_milliseconds)

args = load_args()
setup_logging()
logger = logging.getLogger(__name__)

logger.info('starting MQTT client')

try:
    config_service = ConfigService(args.env)
    mqtt_config = config_service.get_section('mqtt')
    mysql_config = config_service.get_section('mysql')
    watering_config = config_service.get_section('watering')

    data_service = DataService(mysql_config)
    watering_service = WateringService(watering_config)

    mqtt_client = create_mqtt_client(mqtt_config)
    mqtt_client.loop_forever()
except Exception as error:
    logger.error('main error', exc_info=True)
<|endoftext|>


<commit_before>import kaptan
from sh import tmux, cut
from pprint import pprint

class Session(object):

    def __init__(self, **kwargs):
        if 'session_name' not in kwargs:
            raise ValueError('Session requires session_name')
        else:
            for (k, v) in kwargs.items():
                setattr(self, k, v)

    @property
    def windows(self):
        """
            ``tmux list-windows`` outputs 1 session per line ``\n``.

            -F (FORMATS) allows returning custom sessings, we delimit with
            a tab ``\t``.

            we then use dict+zip to align the format variable with the
            output.

            the ``Window`` object accepts the returned properties  and
            ``Session`` (``self``) object.
        """

        if hasattr(self, 'session_name'):
            formats = WINDOW_FORMATS
            tmux_formats = ['#{%s}\t' % format for format in formats]

            windows = cut(
                tmux(
                    'list-windows',                 # ``tmux list-windows``
                    '-t%s' % self.session_name,     # target (session name)
                    '-F%s' % ''.join(tmux_formats)  # output
                ), '-f1', '-d:'
            )

            # combine format keys with values returned from ``tmux list-windows``
            windows = [dict(zip(formats, window.split('\t'))) for window in windows]

            # clear up empty dict
            windows = [
                dict((k, v) for k, v in window.iteritems() if v) for window in windows
            ]

            windows = [Window(session=self, **window) for window in windows]

            return windows
        else:
            return None  # session is not bound to a current session, the user
                         # is using Session imported config file to launch a
                         # new session

    def __repr__(self):
        # todo test without session_name
        return "%s(%s)" % (self.__class__, self.session_name)

class Window(object):
    """
        todo
            - @has_session property, or throw an Error
            - Pane
            - __cmd__
    """

    """
     Each window displayed by tmux may be split into one or more panes; each pane takes up a certain area of the
     display and is a separate terminal.  A window may be split into panes using the split-window command.  Windows
     may be split horizontally (with the -h flag) or vertically.  Panes may be resized with the resize-pane command
     (bound to 'C-up', 'C-down' 'C-left' and 'C-right' by default), the current pane may be changed with the
     select-pane command and the rotate-window and swap-pane commands may be used to swap panes without changing
     their position.  Panes are numbered beginning from zero in the order they are created.

     A number of preset layouts are available.  These may be selected with the select-layout command or cycled with
     next-layout (bound to 'Space' by default); once a layout is chosen, panes within it may be moved and resized as
     normal.

     The following layouts are supported:

     even-horizontal
             Panes are spread out evenly from left to right across the window.

     even-vertical
             Panes are spread evenly from top to bottom.

     main-horizontal
             A large (main) pane is shown at the top of the window and the remaining panes are spread from left to
             right in the leftover space at the bottom.  Use the main-pane-height window option to specify the
             height of the top pane.

     main-vertical
             Similar to main-horizontal but the large pane is placed on the left and the others spread from top to
             bottom along the right.  See the main-pane-width window option.

     tiled   Panes are spread out as evenly as possible over the window in both rows and columns.

     In addition, select-layout may be used to apply a previously used layout - the list-windows command displays
     the layout of each window in a form suitable for use with select-layout.  For example:

           $ tmux list-windows
           0: ksh [159x48]
               layout: bb62,159x48,0,0{79x48,0,0,79x48,80,0}
           $ tmux select-layout bb62,159x48,0,0{79x48,0,0,79x48,80,0}

    """
    def __init__(self, **kwargs):

        if 'session' in kwargs:
            if isinstance(kwargs['session'], Session):
                self._session = kwargs['session']
            else:
                raise TypeError('session must be a Session object')

        [setattr(self, k, v) for (k, v) in kwargs.items() if k is not 'session']

    def __repr__(self):
        # todo test without session_name
        return "%s(%s)" % (self.__class__, self.__dict__)

    __LAYOUTS__ = [
        'even-horizontal',  # Panes are spread out evenly from left to right across the window.
        'even-vertical',  # Panes are spread evenly from top to bottom.
        'main-horizontal',
        'main-vertical',
        'tiled'
    ]

    def from_tmux(self):
        """ parse into normal properties """
        pass

    def to_cmd(self):
        """ parse into tmux cmd to recreate workspace panes """
        pass

    # __dict__ should return properties that would export into a config

    @property
    def __cmd__(self):
        for pane in self.panes:
            pprint(pane.__dict__)

    @property
    def session(self):
        # remove this?
        return self._session if self._session else None

    @property
    def panes(self):
        """
            ``tmux list-panes`` outputs 1 session per line ``\n``.

            -F (FORMATS) picks settings return, we delimit with tab ``\t``.

            we then use dict+zip to align the format variable with the
            output.

            the ``Pane`` object accepts the returned properties, ``Session``
            object and ``Window`` (``self``) object.
        """

        if hasattr(self.session, 'session_name'):
            formats = PANE_FORMATS
            tmux_formats = ['#{%s}\t' % format for format in formats]

            panes = cut(
                tmux(
                    'list-panes',
                    '-s',                                # for sessions
                    '-t%s' % self.session.session_name,  # target (name of session)
                    '-F%s' % ''.join(tmux_formats)       # output
                ), '-f1', '-d:'
            )

            # `tmux list-panes` outputs a session per-line,
            # separate every line from `tmux list-panes` into a pane
            panes = str(panes).split('\n')

            # zip and map the results into the dict of formats used above
            panes = [dict(zip(formats, pane.split('\t'))) for pane in panes]

            # clear up empty dict
            panes = [
                dict((k, v) for k, v in pane.iteritems() if v) for pane in panes
            ]

            panes = [Pane.from_tmux(session=self.session, window=self, **pane) for pane in panes]

            return panes

class Pane(object):

    @classmethod
    def from_tmux(cls, **kwargs):
        """ return a Pane object
            arguments are results passed from tmux
        """
        pane = cls()

        for (k, v) in kwargs.items():
            setattr(pane, k, v)

        # verify Session object
        if 'session' in kwargs:
            if isinstance(kwargs['session'], Session):
                pane._session = kwargs['session']
            else:
                raise TypeError('session must be a Session object')

        # verify Window object
        if 'window' in kwargs:
            if isinstance(kwargs['window'], Window):
                pane._window = kwargs['window']
            else:
                raise TypeError('window must be a Window object')

        return pane

    def __init__(self, **kwargs):
        pass

    def __repr__(self):
        # todo test without session_name
        return "%s(%s)" % (self.__class__, self.__dict__)

def get_sessions():
    formats = SESSION_FORMATS
    tmux_formats = ['#{%s}\t' % format for format in formats]

    sessions = cut(
        tmux(
            'list-sessions',                 # ``tmux list-windows``
            '-F%s' % ''.join(tmux_formats)  # output
        ), '-f1', '-d:'
    )

    # combine format keys with values returned from ``tmux list-windows``
    sessions = [dict(zip(formats, session.split('\t'))) for session in sessions]

    # clear up empty dict
    sessions = [
        dict((k, v) for k, v in session.iteritems() if v) for session in sessions
    ]

    sessions = [Session(**session) for session in sessions]

    for session in sessions:
        yield session

"""
    experiment, ALL_FORMATS parsing for session, window, pane object and
    remove blank values

    http://sourceforge.net/p/tmux/tmux-code/ci/master/tree/format.c
"""

SESSION_FORMATS = [
    'session_group',
    'session_name',
    'session_windows',
    'session_width',
    'session_height',
    'session_id',
    'session_grouped',
    'session_group',
    'session_created',
    'session_created_string',
    'session_attached',
]

CLIENT_FORMATS = [
    'client_cwd',
    'client_height',
    'client_width',
    'client_tty',
    'client_termname',
    'client_created',
    'client_created_string',
    'client_activity',
    'client_activity_string',
    'client_prefix',
    'client_utf8',
    'client_readonly',
    'client_session',
    'client_last_session',
]

WINDOW_FORMATS = [
    # format_window()
    'window_id',
    'window_name',
    'window_width',
    'window_height',
    'window_layout',
    'window_panes',
    # format_winlink()
    'window_index',
    'window_flags',
    'window_active',
    'window_bell_flag',
    'window_activity_flag',
    'window_silence_flag',
]

PANE_FORMATS = [
    'history_size',
    'history_limit',
    'history_bytes',
    'pane_index',
    'pane_width',
    'pane_height',
    'pane_title',
    'pane_id',
    'pane_active',
    'pane_dead',
    'pane_in_mode',
    'pane_synchronized',
    'pane_tty',
    'pane_pid',
    'pane_start_command',
    'pane_start_path',
    'pane_current_path',
    'pane_current_command',

    'cursor_x',
    'cursor_y',
    'scroll_region_upper',
    'scroll_region_lower',
    'saved_cursor_x',
    'saved_cursor_y',

    'alternate_on',
    'alternate_saved_x',
    'alternate_saved_y',

    'cursor_flag',
    'insert_flag',
    'keypad_cursor_flag',
    'keypad_flag',
    'wrap_flag',

    'mouse_standard_flag',
    'mouse_button_flag',
    'mouse_any_flag',
    'mouse_utf8_flag',
]

config = kaptan.Kaptan(handler="yaml")
config.import_config("""
    windows:
        - editor:
            layout: main-vertical
            panes:
                - vim
                - cowsay "hey"
        - server: htop
        - logs: tail -f logs/development.log
    """)

print config
pprint(config.__dict__)
""" expand inline config
    dict({'session_name': { dict })

    to

    dict({ name='session_name', **dict})
"""
windows = list()
for window in config.get('windows'):

    if len(window) == int(1):
        name = window.iterkeys().next()  # get window name

        """expand
            window[name] = 'command'

            to

            window[name] = {
                panes=['command']
            }
        """
        if isinstance(window[name], basestring):
            windowoptions = dict(
                panes=[window[name]]
            )
        else:
            windowoptions = window[name]

        window = dict(name=name, **windowoptions)
        if len(window['panes']) > int(1):
            pprint('omg multiple panes')

    windows.append(window)

for session in get_sessions():
    #pprint(session)
    for window in session.windows:
        #pprint(window)
        pprint(window.__cmd__)

        for pane in window.panes:
            #pprint(pane.__dict__)
            pass
<commit_msg>.from_tmux to pull config options<commit_after>import kaptan
from sh import tmux, cut
from pprint import pprint

class Session(object):

    def __init__(self, **kwargs):
        if 'session_name' not in kwargs:
            raise ValueError('Session requires session_name')
        else:
            for (k, v) in kwargs.items():
                setattr(self, k, v)

    @classmethod
    def from_tmux(cls, **kwargs):
        if 'session_name' not in kwargs:
            raise ValueError('Session requires session_name')

        session = cls(session_name=kwargs['session_name'])
        session.__TMUX__ = dict()
        for (k, v) in kwargs.items():
            session.__TMUX__[k] = v

        """
            ``tmux list-windows`` outputs 1 session per line ``\n``.

            -F (FORMATS) allows returning custom sessings, we delimit with
            a tab ``\t``.

            we then use dict+zip to align the format variable with the
            output.

            the ``Window`` object accepts the returned properties  and
            ``Session`` (``self``) object.
        """

        formats = WINDOW_FORMATS
        tmux_formats = ['#{%s}\t' % format for format in formats]

        windows = cut(
            tmux(
                'list-windows',                 # ``tmux list-windows``
                '-t%s' % session.session_name,     # target (session name)
                '-F%s' % ''.join(tmux_formats)  # output
            ), '-f1', '-d:'
        )

        # combine format keys with values returned from ``tmux list-windows``
        windows = [dict(zip(formats, window.split('\t'))) for window in windows]

        # clear up empty dict
        windows = [
            dict((k, v) for k, v in window.iteritems() if v) for window in windows
        ]

        session._windows = [Window(session=session, **window) for window in windows]

        return session

    @property
    def windows(self):
        return self._windows

    def windows_from_tmux(self):
            return self._windows

    def __repr__(self):
        # todo test without session_name
        return "%s(%s)" % (self.__class__, self.session_name)

class Window(object):
    """
        todo
            - @has_session property, or throw an Error
            - Pane
            - __cmd__
    """

    """
     Each window displayed by tmux may be split into one or more panes; each pane takes up a certain area of the
     display and is a separate terminal.  A window may be split into panes using the split-window command.  Windows
     may be split horizontally (with the -h flag) or vertically.  Panes may be resized with the resize-pane command
     (bound to 'C-up', 'C-down' 'C-left' and 'C-right' by default), the current pane may be changed with the
     select-pane command and the rotate-window and swap-pane commands may be used to swap panes without changing
     their position.  Panes are numbered beginning from zero in the order they are created.

     A number of preset layouts are available.  These may be selected with the select-layout command or cycled with
     next-layout (bound to 'Space' by default); once a layout is chosen, panes within it may be moved and resized as
     normal.

     The following layouts are supported:

     even-horizontal
             Panes are spread out evenly from left to right across the window.

     even-vertical
             Panes are spread evenly from top to bottom.

     main-horizontal
             A large (main) pane is shown at the top of the window and the remaining panes are spread from left to
             right in the leftover space at the bottom.  Use the main-pane-height window option to specify the
             height of the top pane.

     main-vertical
             Similar to main-horizontal but the large pane is placed on the left and the others spread from top to
             bottom along the right.  See the main-pane-width window option.

     tiled   Panes are spread out as evenly as possible over the window in both rows and columns.

     In addition, select-layout may be used to apply a previously used layout - the list-windows command displays
     the layout of each window in a form suitable for use with select-layout.  For example:

           $ tmux list-windows
           0: ksh [159x48]
               layout: bb62,159x48,0,0{79x48,0,0,79x48,80,0}
           $ tmux select-layout bb62,159x48,0,0{79x48,0,0,79x48,80,0}

    """
    def __init__(self, **kwargs):

        if 'session' in kwargs:
            if isinstance(kwargs['session'], Session):
                self._session = kwargs['session']
            else:
                raise TypeError('session must be a Session object')

        [setattr(self, k, v) for (k, v) in kwargs.items() if k is not 'session']

    def __repr__(self):
        # todo test without session_name
        return "%s(%s)" % (self.__class__, self.__dict__)

    __LAYOUTS__ = [
        'even-horizontal',  # Panes are spread out evenly from left to right across the window.
        'even-vertical',  # Panes are spread evenly from top to bottom.
        'main-horizontal',
        'main-vertical',
        'tiled'
    ]

    def from_tmux(self):
        """ parse into normal properties """
        pass

    def to_cmd(self):
        """ parse into tmux cmd to recreate workspace panes """
        pass

    # __dict__ should return properties that would export into a config

    @property
    def __cmd__(self):
        for pane in self.panes:
            pprint(pane.__dict__)

    @property
    def session(self):
        # remove this?
        return self._session if self._session else None

    @property
    def panes(self):
        if self.session:
            return self.panes_from_tmux

    @property
    def panes_from_tmux(self):
        """
            ``tmux list-panes`` outputs 1 session per line ``\n``.

            -F (FORMATS) picks settings return, we delimit with tab ``\t``.

            we then use dict+zip to align the format variable with the
            output.

            the ``Pane`` object accepts the returned properties, ``Session``
            object and ``Window`` (``self``) object.
        """

        if hasattr(self.session, 'session_name'):
            formats = PANE_FORMATS
            tmux_formats = ['#{%s}\t' % format for format in formats]

            panes = cut(
                tmux(
                    'list-panes',
                    '-s',                                # for sessions
                    '-t%s' % self.session.session_name,  # target (name of session)
                    '-F%s' % ''.join(tmux_formats)       # output
                ), '-f1', '-d:'
            )

            # `tmux list-panes` outputs a session per-line,
            # separate every line from `tmux list-panes` into a pane
            panes = str(panes).split('\n')

            # zip and map the results into the dict of formats used above
            panes = [dict(zip(formats, pane.split('\t'))) for pane in panes]

            # clear up empty dict
            panes = [
                dict((k, v) for k, v in pane.iteritems() if v) for pane in panes
            ]

            panes = [Pane.from_tmux(session=self.session, window=self, **pane) for pane in panes]

            return panes

class Pane(object):

    @classmethod
    def from_tmux(cls, **kwargs):
        """ return a Pane object
            arguments are results passed from tmux
        """
        pane = cls()

        for (k, v) in kwargs.items():
            setattr(pane, k, v)

        # verify Session object
        if 'session' in kwargs:
            if isinstance(kwargs['session'], Session):
                pane._session = kwargs['session']
            else:
                raise TypeError('session must be a Session object')

        # verify Window object
        if 'window' in kwargs:
            if isinstance(kwargs['window'], Window):
                pane._window = kwargs['window']
            else:
                raise TypeError('window must be a Window object')

        return pane

    def __init__(self, **kwargs):
        pass

    def __repr__(self):
        # todo test without session_name
        return "%s(%s)" % (self.__class__, self.__dict__)

def get_sessions():
    formats = SESSION_FORMATS
    tmux_formats = ['#{%s}\t' % format for format in formats]

    sessions = cut(
        tmux(
            'list-sessions',                 # ``tmux list-windows``
            '-F%s' % ''.join(tmux_formats)  # output
        ), '-f1', '-d:'
    )

    # combine format keys with values returned from ``tmux list-windows``
    sessions = [dict(zip(formats, session.split('\t'))) for session in sessions]

    # clear up empty dict
    sessions = [
        dict((k, v) for k, v in session.iteritems() if v) for session in sessions
    ]

    sessions = [Session.from_tmux(**session) for session in sessions]

    for session in sessions:
        yield session

"""
    experiment, ALL_FORMATS parsing for session, window, pane object and
    remove blank values

    http://sourceforge.net/p/tmux/tmux-code/ci/master/tree/format.c
"""

SESSION_FORMATS = [
    'session_group',
    'session_name',
    'session_windows',
    'session_width',
    'session_height',
    'session_id',
    'session_grouped',
    'session_group',
    'session_created',
    'session_created_string',
    'session_attached',
]

CLIENT_FORMATS = [
    'client_cwd',
    'client_height',
    'client_width',
    'client_tty',
    'client_termname',
    'client_created',
    'client_created_string',
    'client_activity',
    'client_activity_string',
    'client_prefix',
    'client_utf8',
    'client_readonly',
    'client_session',
    'client_last_session',
]

WINDOW_FORMATS = [
    # format_window()
    'window_id',
    'window_name',
    'window_width',
    'window_height',
    'window_layout',
    'window_panes',
    # format_winlink()
    'window_index',
    'window_flags',
    'window_active',
    'window_bell_flag',
    'window_activity_flag',
    'window_silence_flag',
]

PANE_FORMATS = [
    'history_size',
    'history_limit',
    'history_bytes',
    'pane_index',
    'pane_width',
    'pane_height',
    'pane_title',
    'pane_id',
    'pane_active',
    'pane_dead',
    'pane_in_mode',
    'pane_synchronized',
    'pane_tty',
    'pane_pid',
    'pane_start_command',
    'pane_start_path',
    'pane_current_path',
    'pane_current_command',

    'cursor_x',
    'cursor_y',
    'scroll_region_upper',
    'scroll_region_lower',
    'saved_cursor_x',
    'saved_cursor_y',

    'alternate_on',
    'alternate_saved_x',
    'alternate_saved_y',

    'cursor_flag',
    'insert_flag',
    'keypad_cursor_flag',
    'keypad_flag',
    'wrap_flag',

    'mouse_standard_flag',
    'mouse_button_flag',
    'mouse_any_flag',
    'mouse_utf8_flag',
]

config = kaptan.Kaptan(handler="yaml")
config.import_config("""
    windows:
        - editor:
            layout: main-vertical
            panes:
                - vim
                - cowsay "hey"
        - server: htop
        - logs: tail -f logs/development.log
    """)

print config
pprint(config.__dict__)
""" expand inline config
    dict({'session_name': { dict })

    to

    dict({ name='session_name', **dict})
"""
windows = list()
for window in config.get('windows'):

    if len(window) == int(1):
        name = window.iterkeys().next()  # get window name

        """expand
            window[name] = 'command'

            to

            window[name] = {
                panes=['command']
            }
        """
        if isinstance(window[name], basestring):
            windowoptions = dict(
                panes=[window[name]]
            )
        else:
            windowoptions = window[name]

        window = dict(name=name, **windowoptions)
        if len(window['panes']) > int(1):
            pprint('omg multiple panes')

    windows.append(window)

for session in get_sessions():
    #pprint(session)
    for window in session.windows:
        #pprint(window)
        pprint(window.__cmd__)

        for pane in window.panes:
            #pprint(pane.__dict__)
            pass
<|endoftext|>


<commit_before>#!/usr/local/bin/python3
# -*- coding: utf-8 -*-
from tkinter import *
from tkinter import ttk
from PIL import ImageTk, Image
import json, os

# Called once to load in the JSON.
elements = {}
with open('traits_list.json') as f:
	firsttraits = json.loads(f.read())
	traits = []
	for item in firsttraits:
		traits.append({"Name":item["Name"], "Personality":item["Personality"]})
		for i, key in enumerate(('Type', 'Element', 'DisplayName', 'ShortQuote', 'Quality1',\
			'Quality2', 'Quality3', 'LongQuote', 'Superficial', 'Deep', 'AllConsuming')):
			traits[-1][key] = item["Data"][i]

# Starts up the core tkinter stuff
root = Tk()
root.title("Mystic Empyrean Traits")
s = ttk.Style()
s.configure('white.TFrame', background='#FFFFFF')
mainFrame = ttk.Frame(root, padding="5", style='white.TFrame')
mainFrame.grid(column=0, row=0, sticky=(N,S,E,W))
mainFrame.config()
summaryFrame = ttk.Frame(root, padding="5", style='white.TFrame')
summaryFrame.grid(column=0, row=1, sticky=(N,S,E,W))
summaryFrame.config()
detailFrame = ttk.Frame(root, padding="5", style='white.TFrame')
detailFrame.grid(column=0, row=3, sticky=(N,S,E,W))
detailFrame.grid_remove()
detailFrame.config()

# Functions to change the information displayed in the UI depending on what trait or personality has been selected.
def setTrait(selectedTraitIndex):
	for key in ('Name', 'Element', 'Type', 'Personality', 'DisplayName', 'ShortQuote',\
		'Quality1', 'Quality2', 'Quality3', 'LongQuote', 'Superficial', 'Deep', 'AllConsuming'):
		SelectedTraits[key].set(traits[selectedTraitIndex][key])
	SelectedTraits["DisplayName"].set("An eidolon displays " + traits[selectedTraitIndex]["DisplayName"] + " if:")
	currentImages['Type'] = ImageTk.PhotoImage(Image.open(os.path.join('assets', SelectedTraits['Type'].get()+'-tag.png')))
	currentImages['Element'] = ImageTk.PhotoImage(Image.open(os.path.join('assets', SelectedTraits['Element'].get()+'-tag.png')).convert("RGB"))
	elements['Type'].configure(image=currentImages['Type'])
	elements['Element'].configure(image=currentImages['Element'])
	return (SelectedTraits['Name'].get(), SelectedTraits['Personality'].get())

def chooseTrait(*args):
	personality = setTrait(traitNames[selectedTraitName.get()])[1]
	selectedPersonalityName.set(personality)
def choosePersonality(*args):
	trait = setTrait(personalities[selectedPersonalityName.get()])[0]
	selectedTraitName.set(trait)

# Functions to open and close the detail view
def openDetails(*args):
	detailFrame.grid()
	elements['OpenDetail'].grid_remove()
def closeDetails(*args):
	detailFrame.grid_remove()
	elements['OpenDetail'].grid()

# Sets defaults for the dropdown lists at the top, and the images.
traitNames, personalities= {}, {}
for index, traitobj in enumerate(traits):
	traitNames[traitobj["Name"]] = index
	personalities[traitobj["Personality"]] = index
traitNamesList, personalitiesList = sorted(traitNames.keys()), sorted(personalities.keys())
selectedTraitName, selectedPersonalityName = StringVar(), StringVar()
selectedTraitName.set("League Sprinter")
currentImages, SelectedTraits = {'Type':None, 'Element':None}, {}

# Creates variables that are aware of what is active, and sets defaults for them
for key in traits[0].keys():
	SelectedTraits[key] = StringVar()
	if key in ('Type', 'Element'):
		elements[key] = ttk.Label(summaryFrame, image=currentImages[key])
	elif key in ('Superficial', 'Deep', 'AllConsuming'):
		elements[key] = Message(detailFrame, textvariable=SelectedTraits[key], width=710)
	else:
		elements[key] = ttk.Label(summaryFrame, textvariable=SelectedTraits[key])
elements['LongQuote'] = Message(summaryFrame, textvariable=SelectedTraits['LongQuote'], width=710)
chooseTrait()

# Creates the dropdown lists and buttons at the top, and grids them
OptionMenu(mainFrame, selectedTraitName, *traitNamesList, command=chooseTrait).grid(column=0, row=0, sticky=(N,S))
OptionMenu(mainFrame, selectedPersonalityName, *personalitiesList, command=choosePersonality).grid(column=1, row=0, sticky=(N,S))
# Creates the buttons to open and close the detail view
elements['OpenDetail'] = ttk.Button(root, command=openDetails, text='+')
elements['CloseDetail'] = ttk.Button(detailFrame, command=closeDetails, text='-')

# summaryFrame stuff
elements['Name'].grid(column=0, row=0, columnspan=2, sticky=(N,S,E,W))
elements['Personality'].grid(column=2, row=0, sticky=(N,S,E,W))
elements['Type'].grid(column=0, row=1, columnspan=2, sticky=W)
elements['Element'].grid(column=2, row=1)
elements['ShortQuote'].grid(column=0, row=2, columnspan=3, sticky=W)
elements['DisplayName'].grid(column=0, row=3, columnspan=3, sticky=W)
# Qualities, which need a bullet point just before them
ttk.Label(summaryFrame, text="•").grid(column=0, row=4, sticky=E)
elements['Quality1'].grid(column=1, row=4, columnspan=2, sticky=W)
ttk.Label(summaryFrame, text="•").grid(column=0, row=5, sticky=E)
elements['Quality2'].grid(column=1, row=5, columnspan=2, sticky=W)
ttk.Label(summaryFrame, text="•").grid(column=0, row=6, sticky=E)
elements['Quality3'].grid(column=1, row=6, columnspan=2, sticky=W)
# End of the summaryFrame stuff
elements['LongQuote'].grid(column=0, row=7, columnspan=3, sticky=W)
elements['OpenDetail'].grid(column=0, row=2, sticky=W)

# detailFrame stuff.
elements['Superficial'].grid(column=0, row=0, sticky=W)
elements['Deep'].grid(column=0, row=1, sticky=W)
elements['AllConsuming'].grid(column=0, row=2, sticky=W)
elements['CloseDetail'].grid(column=0, row=3, sticky=E)

root.mainloop()<commit_msg>Added a search box which doesn't yet do anything beyond tracking what is being searched for.<commit_after>#!/usr/local/bin/python3
# -*- coding: utf-8 -*-
from tkinter import *
from tkinter import ttk
from PIL import ImageTk, Image
import json, os

# Called once to load in the JSON.
elements = {}
with open('traits_list.json') as f:
	firsttraits = json.loads(f.read())
	traits = []
	for item in firsttraits:
		traits.append({"Name":item["Name"], "Personality":item["Personality"]})
		for i, key in enumerate(('Type', 'Element', 'DisplayName', 'ShortQuote', 'Quality1',\
			'Quality2', 'Quality3', 'LongQuote', 'Superficial', 'Deep', 'AllConsuming')):
			traits[-1][key] = item["Data"][i]

# Starts up the core tkinter stuff
root = Tk()
root.title("Mystic Empyrean Traits")
s = ttk.Style()
s.configure('white.TFrame', background='#FFFFFF')
mainFrame = ttk.Frame(root, padding="5", style='white.TFrame')
mainFrame.grid(column=0, row=0, sticky=(N,S,E,W))
mainFrame.config()
summaryFrame = ttk.Frame(root, padding="5", style='white.TFrame')
summaryFrame.grid(column=0, row=1, sticky=(N,S,E,W))
summaryFrame.config()
detailFrame = ttk.Frame(root, padding="5", style='white.TFrame')
detailFrame.grid(column=0, row=3, sticky=(N,S,E,W))
detailFrame.grid_remove()
detailFrame.config()

# Functions to change the information displayed in the UI depending on what trait or personality has been selected.
def setTrait(selectedTraitIndex):
	for key in ('Name', 'Element', 'Type', 'Personality', 'DisplayName', 'ShortQuote',\
		'Quality1', 'Quality2', 'Quality3', 'LongQuote', 'Superficial', 'Deep', 'AllConsuming'):
		SelectedTraits[key].set(traits[selectedTraitIndex][key])
	SelectedTraits["DisplayName"].set("An eidolon displays " + traits[selectedTraitIndex]["DisplayName"] + " if:")
	currentImages['Type'] = ImageTk.PhotoImage(Image.open(os.path.join('assets', SelectedTraits['Type'].get()+'-tag.png')))
	currentImages['Element'] = ImageTk.PhotoImage(Image.open(os.path.join('assets', SelectedTraits['Element'].get()+'-tag.png')).convert("RGB"))
	elements['Type'].configure(image=currentImages['Type'])
	elements['Element'].configure(image=currentImages['Element'])
	return (SelectedTraits['Name'].get(), SelectedTraits['Personality'].get())

def chooseTrait(*args):
	personality = setTrait(traitNames[selectedTraitName.get()])[1]
	selectedPersonalityName.set(personality)
def choosePersonality(*args):
	trait = setTrait(personalities[selectedPersonalityName.get()])[0]
	selectedTraitName.set(trait)

# Functions to open and close the detail view
def openDetails(*args):
	detailFrame.grid()
	elements['OpenDetail'].grid_remove()
def closeDetails(*args):
	detailFrame.grid_remove()
	elements['OpenDetail'].grid()

# Function to update the drop-down lists.
def updateLists(*args):
	print("current string is " + searchTerm.get())

# Sets defaults for the dropdown lists at the top, and the images.
traitNames, personalities= {}, {}
for index, traitobj in enumerate(traits):
	traitNames[traitobj["Name"]] = index
	personalities[traitobj["Personality"]] = index
traitNamesList, personalitiesList = sorted(traitNames.keys()), sorted(personalities.keys())
selectedTraitName, selectedPersonalityName, searchTerm = StringVar(), StringVar(), StringVar()
selectedTraitName.set("League Sprinter")
searchTerm.set("")
searchTerm.trace("w", updateLists)
currentImages, SelectedTraits = {'Type':None, 'Element':None}, {}

# Creates variables that are aware of what is active, and sets defaults for them
for key in traits[0].keys():
	SelectedTraits[key] = StringVar()
	if key in ('Type', 'Element'):
		elements[key] = ttk.Label(summaryFrame, image=currentImages[key])
	elif key in ('Superficial', 'Deep', 'AllConsuming'):
		elements[key] = Message(detailFrame, textvariable=SelectedTraits[key], width=710)
	else:
		elements[key] = ttk.Label(summaryFrame, textvariable=SelectedTraits[key])
elements['LongQuote'] = Message(summaryFrame, textvariable=SelectedTraits['LongQuote'], width=710)
chooseTrait()

# Creates the dropdown lists and buttons at the top, and grids them
Entry(mainFrame, textvariable=searchTerm, width=20).grid(column=0, row=0, sticky=(N,S))
OptionMenu(mainFrame, selectedTraitName, *traitNamesList, command=chooseTrait).grid(column=1, row=0, sticky=(N,S))
OptionMenu(mainFrame, selectedPersonalityName, *personalitiesList, command=choosePersonality).grid(column=2, row=0, sticky=(N,S))
# Creates the buttons to open and close the detail view
elements['OpenDetail'] = ttk.Button(root, command=openDetails, text='+')
elements['CloseDetail'] = ttk.Button(detailFrame, command=closeDetails, text='-')

# summaryFrame stuff
elements['Name'].grid(column=0, row=0, columnspan=2, sticky=(N,S,E,W))
elements['Personality'].grid(column=2, row=0, sticky=(N,S,E,W))
elements['Type'].grid(column=0, row=1, columnspan=2, sticky=W)
elements['Element'].grid(column=2, row=1)
elements['ShortQuote'].grid(column=0, row=2, columnspan=3, sticky=W)
elements['DisplayName'].grid(column=0, row=3, columnspan=3, sticky=W)
# Qualities, which need a bullet point just before them
ttk.Label(summaryFrame, text="•").grid(column=0, row=4, sticky=E)
elements['Quality1'].grid(column=1, row=4, columnspan=2, sticky=W)
ttk.Label(summaryFrame, text="•").grid(column=0, row=5, sticky=E)
elements['Quality2'].grid(column=1, row=5, columnspan=2, sticky=W)
ttk.Label(summaryFrame, text="•").grid(column=0, row=6, sticky=E)
elements['Quality3'].grid(column=1, row=6, columnspan=2, sticky=W)
# End of the summaryFrame stuff
elements['LongQuote'].grid(column=0, row=7, columnspan=3, sticky=W)
elements['OpenDetail'].grid(column=0, row=2, sticky=W)

# detailFrame stuff.
elements['Superficial'].grid(column=0, row=0, sticky=W)
elements['Deep'].grid(column=0, row=1, sticky=W)
elements['AllConsuming'].grid(column=0, row=2, sticky=W)
elements['CloseDetail'].grid(column=0, row=3, sticky=E)

root.mainloop()<|endoftext|>


<commit_before>import cv2
import sys

def main():
    if not len(sys.argv) == 2:
        print("usage: {} [cascade filepath]".format(sys.argv[0]))

    cap = cv2.VideoCapture(0) # reads from first webcam
    casc_path = sys.argv[1] # cascade filepath
    face_casc = cv2.CascadeClassifier(casc_path)

    while True:
        ret, frame = cap.read() # read a frame from cap
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # create a grayscal version of the frame
        faces = face_casc.detectMultiScale(
            gray,
            scaleFactor=1.1,
            minNeighbours=5,
            minSize=(30, 30),
            flags = cv2.cv.CV_HAAR_SCALE_IMAGE
        )
        print("Found {} faces".format(len(faces)))

        for (x, y, w, h) in faces:
            # print coordinates of centre of rectangle
            print("centre of face: ({}, {})".format((2*x + w) / 2, (2*y + h) / 2))

    # cleanup
    cap.release()
    cv2.destroyAllWindows()

if __name__ == "__main__":
    main()
<commit_msg>changes to account for new version of opencv<commit_after>import cv2
import sys

def main():
    if not len(sys.argv) == 2:
        print("usage: {} [cascade filepath]".format(sys.argv[0]))

    cap = cv2.VideoCapture(0) # reads from first webcam
    casc_path = sys.argv[1] # cascade filepath
    face_casc = cv2.CascadeClassifier(casc_path)

    while True:
        ret, frame = cap.read() # read a frame from cap
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # create a grayscal version of the frame
        faces = face_casc.detectMultiScale(
            gray,
            scaleFactor=1.1,
            minNeighbors=5,
            minSize=(30, 30),
            flags = cv2.CASCADE_SCALE_IMAGE
        )
        print("Found {} faces".format(len(faces)))

        for (x, y, w, h) in faces:
            # print coordinates of centre of rectangle
            print("centre of face: ({}, {})".format((2*x + w) / 2, (2*y + h) / 2))

    # cleanup
    cap.release()
    cv2.destroyAllWindows()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import logging
import sys
import time

# noinspection PyUnresolvedReferences
import better_exceptions
import redis
from praw import Reddit

from helpers import _
from helpers import flair
from helpers import flair_post
from helpers import get_wiki_page
from helpers import is_valid
from helpers import log_header
from helpers import update_wiki_page
from processing import process_claim
from processing import process_done
from processing import process_mention
from processing import process_post
from strings import id_already_handled_in_db

# This program is dedicated to Aramanthe and Icon For Hire, whose music
# served as the soundtrack for much of its development.

class Context(object):
    """
    Support object for the bot -- holds data that doesn't have
    anywhere else to go.
    """
    video_domains = []
    audio_domains = []
    image_domains = []

    video_formatting = ''
    audio_formatting = ''
    image_formatting = ''
    header = ''

    subreddits_to_check = []
    # subreddits that we're only getting 100 posts at a time from
    # instead of jump-starting it with 500
    subreddit_members = []
    tor_mods = []

    perform_header_check = False
    debug_sleep = False

def populate_header():
    Context.header = ''

    result = get_wiki_page('format/header', tor=tor)
    result = result.split('\r\n')
    temp = []
    for part in result:
        part = part.lstrip().rstrip()
        if part == '':
            continue
        if part == '---':
            continue
        temp.append(part)
    Context.header = ' '.join(temp)

def populate_formatting():
    """
    Grabs the contents of the three wiki pages that contain the
    formatting examples and stores them in the Context object.

    :return: None.
    """
    # zero out everything so we can reinitialize later
    Context.audio_formatting = ''
    Context.video_formatting = ''
    Context.image_formatting = ''

    Context.audio_formatting = get_wiki_page('format/audio', tor=tor)
    Context.video_formatting = get_wiki_page('format/video', tor=tor)
    Context.image_formatting = get_wiki_page('format/images', tor=tor)

def populate_domain_lists():
    """
    Loads the approved content domains into the Context object from the
    wiki page.

    :return: None.
    """

    Context.video_domains = []
    Context.image_domains = []
    Context.audio_domains = []

    domains = get_wiki_page('domains', tor=tor)
    domains = ''.join(domains.splitlines()).split('---')

    for domainset in domains:
        domain_list = domainset[domainset.index('['):].strip('[]').split(', ')
        current_domain_list = []
        if domainset.startswith('video'):
            current_domain_list = Context.video_domains
        elif domainset.startswith('audio'):
            current_domain_list = Context.audio_domains
        elif domainset.startswith('images'):
            current_domain_list = Context.image_domains
        [current_domain_list.append(x) for x in domain_list]
        logging.debug('Domain list populated: {}'.format(current_domain_list))

def populate_moderators(tor):
    # Praw doesn't cache this information, so it requests it every damn time
    # we ask about the moderators. Let's cache this so we can drastically cut
    # down on the number of calls for the mod list.

    # nuke the existing list
    Context.tor_mods = []

    # this call returns a full list rather than a generator. Praw is weird.
    Context.tor_mods = tor.moderator()

def populate_subreddit_lists():
    """
    Gets the list of subreddits to monitor and loads it into memory.

    :return: None.
    """

    Context.subreddits_to_check = []
    Context.subreddit_members = []

    Context.subreddits_to_check = get_wiki_page('subreddits', tor=tor).split('\r\n')
    logging.debug(
        'Created list of subreddits from wiki: {}'.format(
            Context.subreddits_to_check
        )
    )
    Context.subreddit_members = get_wiki_page('subreddits/members', tor=tor).split('\r\n')
    logging.debug(
        'Created list of subreddits from wiki: {}'.format(
            Context.subreddit_members
        )
    )

def configure_logging():
    logging.basicConfig(
        level=logging.INFO,
        format='[%(asctime)s] - [%(levelname)s] - [%(funcName)s] - %(message)s',
        datefmt='%m/%d/%Y %I:%M:%S %p',
        filename='transcribersofreddit.log'
    )
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    formatter = logging.Formatter('[%(asctime)s] - [%(funcName)s] - %(message)s')
    # tell the handler to use this format
    console.setFormatter(formatter)
    # add the handler to the root logger
    logging.getLogger('').addHandler(console)

    log_header('Starting!')

def respond_to_thanks(mention):
    """
    An easter egg; it posts a reply to anything that includes the word
    'thank'. It's very rudimentary but should be a little nugget of fun.

    :param mention: The Comment object.
    :return: None.
    """
    logging.info(
        'Responding to a Thank You comment, ID {}'.format(mention)
    )
    mention.reply(_('You\'re very welcome! I\'m just doing my job!'))

def check_inbox():
    """
    Goes through all the unread messages in the inbox. It has two
    loops within this section, each one dealing with a different type
    of mail. Also deliberately leaves mail which does not fit into
    either category so that it can be read manually at a later point.

    The first loop handles username mentions.
    The second loop sorts out and handles comments that include 'claim'
        and 'done'.
    :return: None.
    """
    # first we do mentions, then comment replies
    mentions = []
    # grab all of our messages and filter
    for item in r.inbox.unread(limit=None):
        if item.subject == 'username mention':
            mentions.append(item)
            item.mark_read()

    # sort them and create posts where necessary
    for mention in mentions:
        logging.info('Received mention! ID {}'.format(mention))

        if not is_valid(mention.parent_id, redis_server):
            # Do our check here to make sure we can actually work on this one and
            # that we haven't already posted about it. We use the full ID here
            # instead of the cleaned one, just in case.
            logging.info(id_already_handled_in_db.format(mention.parent_id))
            continue

        process_mention(mention, r, tor, redis_server, Context)

    # comment replies
    replies = []
    for item in r.inbox.unread(limit=None):
        if item.subject == 'comment reply':
            replies.append(item)
            item.mark_read()

    for reply in replies:
        # if 'thank' in reply.body.lower():
        #     respond_to_thanks(reply)
        #     continue
        if 'claim' in reply.body.lower():
            process_claim(reply, r)
        if 'done' in reply.body.lower():
            process_done(reply, r, tor, Context)

def check_submissions(subreddit, Context):
    """
    Loops through all of the subreddits that have opted in and pulls
    the 100 newest submissions. It checks the domain of the submission
    against the domain lists and hands off the post to process_post()
    for formatting and posting on ToR.

    :param subreddit: String. A valid subreddit name.
    :param Context: the Context object.
    :return: None.
    """
    if subreddit in Context.subreddit_members:
        sr = r.subreddit(subreddit).new(limit=10)
    else:
        sr = r.subreddit(subreddit).new(limit=50)
        Context.subreddit_members.append(subreddit)
        update_wiki_page(
            'subreddits/members',
            '\r\n'.join(Context.subreddit_members),
            tor)

    for post in sr:
        if (
            post.domain in Context.image_domains or
            post.domain in Context.audio_domains or
            post.domain in Context.video_domains
        ):
            process_post(post, tor, redis_server, Context)

def set_meta_flair_on_other_posts(transcribersofreddit, Context):
    """
    Loops through the 25 newest posts on ToR and sets the flair to
    'Meta' for any post that is not authored by the bot or any of
    the moderators.

    :param transcribersofreddit: The Subreddit object for ToR.
    :param Context: the active context object.
    :return: None.
    """
    for post in transcribersofreddit.new(limit=10):

        if (
            post.author != r.redditor('transcribersofreddit') and
            post.author not in Context.tor_mods
        ):
            logging.info(
                'Flairing post {} by author {} with Meta.'.format(
                    post.fullname, post.author
                )
            )
            flair_post(post, flair.meta)

def initialize(tor):
    populate_domain_lists()
    logging.info('Domains loaded.')
    populate_subreddit_lists()
    logging.info('Subreddits loaded.')
    populate_formatting()
    logging.info('Formatting loaded.')
    populate_header()
    logging.info('Header loaded.')
    populate_moderators(tor)
    logging.info('Mod list loaded.')

    logging.info('Initialization complete.')

if __name__ == '__main__':
    r = Reddit('bot')  # load from config file
    configure_logging()
    try:
        redis_server = redis.StrictRedis(host='localhost', port=6379, db=0)
        redis_server.ping()
    except redis.exceptions.ConnectionError:
        logging.error("Redis server is not running! Exiting!")
        sys.exit(1)
    tor = r.subreddit('transcribersofreddit')
    initialize(tor)

    try:
        number_of_loops = 0
        # primary loop
        while True:
            check_inbox()
            for sub in Context.subreddits_to_check:
                check_submissions(sub, Context)
            set_meta_flair_on_other_posts(tor, Context)
            logging.debug('Looping!')
            number_of_loops += 1
            if number_of_loops > 9:
                # reload configuration every ten loops
                initialize(tor)
                number_of_loops = 0
            if Context.debug_sleep:
                time.sleep(60)

    except KeyboardInterrupt:
        logging.error('User triggered shutdown. Shutting down.')
<commit_msg>bugfix for endlessly setting the meta flair on a post that already has it<commit_after>import logging
import sys
import time

# noinspection PyUnresolvedReferences
import better_exceptions
import redis
from praw import Reddit

from helpers import _
from helpers import flair
from helpers import flair_post
from helpers import get_wiki_page
from helpers import is_valid
from helpers import log_header
from helpers import update_wiki_page
from processing import process_claim
from processing import process_done
from processing import process_mention
from processing import process_post
from strings import id_already_handled_in_db

# This program is dedicated to Aramanthe and Icon For Hire, whose music
# served as the soundtrack for much of its development.

class Context(object):
    """
    Support object for the bot -- holds data that doesn't have
    anywhere else to go.
    """
    video_domains = []
    audio_domains = []
    image_domains = []

    video_formatting = ''
    audio_formatting = ''
    image_formatting = ''
    header = ''

    subreddits_to_check = []
    # subreddits that we're only getting 100 posts at a time from
    # instead of jump-starting it with 500
    subreddit_members = []
    tor_mods = []

    perform_header_check = False
    debug_sleep = False

def populate_header():
    Context.header = ''

    result = get_wiki_page('format/header', tor=tor)
    result = result.split('\r\n')
    temp = []
    for part in result:
        part = part.lstrip().rstrip()
        if part == '':
            continue
        if part == '---':
            continue
        temp.append(part)
    Context.header = ' '.join(temp)

def populate_formatting():
    """
    Grabs the contents of the three wiki pages that contain the
    formatting examples and stores them in the Context object.

    :return: None.
    """
    # zero out everything so we can reinitialize later
    Context.audio_formatting = ''
    Context.video_formatting = ''
    Context.image_formatting = ''

    Context.audio_formatting = get_wiki_page('format/audio', tor=tor)
    Context.video_formatting = get_wiki_page('format/video', tor=tor)
    Context.image_formatting = get_wiki_page('format/images', tor=tor)

def populate_domain_lists():
    """
    Loads the approved content domains into the Context object from the
    wiki page.

    :return: None.
    """

    Context.video_domains = []
    Context.image_domains = []
    Context.audio_domains = []

    domains = get_wiki_page('domains', tor=tor)
    domains = ''.join(domains.splitlines()).split('---')

    for domainset in domains:
        domain_list = domainset[domainset.index('['):].strip('[]').split(', ')
        current_domain_list = []
        if domainset.startswith('video'):
            current_domain_list = Context.video_domains
        elif domainset.startswith('audio'):
            current_domain_list = Context.audio_domains
        elif domainset.startswith('images'):
            current_domain_list = Context.image_domains
        [current_domain_list.append(x) for x in domain_list]
        logging.debug('Domain list populated: {}'.format(current_domain_list))

def populate_moderators(tor):
    # Praw doesn't cache this information, so it requests it every damn time
    # we ask about the moderators. Let's cache this so we can drastically cut
    # down on the number of calls for the mod list.

    # nuke the existing list
    Context.tor_mods = []

    # this call returns a full list rather than a generator. Praw is weird.
    Context.tor_mods = tor.moderator()

def populate_subreddit_lists():
    """
    Gets the list of subreddits to monitor and loads it into memory.

    :return: None.
    """

    Context.subreddits_to_check = []
    Context.subreddit_members = []

    Context.subreddits_to_check = get_wiki_page('subreddits', tor=tor).split('\r\n')
    logging.debug(
        'Created list of subreddits from wiki: {}'.format(
            Context.subreddits_to_check
        )
    )
    Context.subreddit_members = get_wiki_page('subreddits/members', tor=tor).split('\r\n')
    logging.debug(
        'Created list of subreddits from wiki: {}'.format(
            Context.subreddit_members
        )
    )

def configure_logging():
    logging.basicConfig(
        level=logging.INFO,
        format='[%(asctime)s] - [%(levelname)s] - [%(funcName)s] - %(message)s',
        datefmt='%m/%d/%Y %I:%M:%S %p',
        filename='transcribersofreddit.log'
    )
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    formatter = logging.Formatter('[%(asctime)s] - [%(funcName)s] - %(message)s')
    # tell the handler to use this format
    console.setFormatter(formatter)
    # add the handler to the root logger
    logging.getLogger('').addHandler(console)

    log_header('Starting!')

def respond_to_thanks(mention):
    """
    An easter egg; it posts a reply to anything that includes the word
    'thank'. It's very rudimentary but should be a little nugget of fun.

    :param mention: The Comment object.
    :return: None.
    """
    logging.info(
        'Responding to a Thank You comment, ID {}'.format(mention)
    )
    mention.reply(_('You\'re very welcome! I\'m just doing my job!'))

def check_inbox():
    """
    Goes through all the unread messages in the inbox. It has two
    loops within this section, each one dealing with a different type
    of mail. Also deliberately leaves mail which does not fit into
    either category so that it can be read manually at a later point.

    The first loop handles username mentions.
    The second loop sorts out and handles comments that include 'claim'
        and 'done'.
    :return: None.
    """
    # first we do mentions, then comment replies
    mentions = []
    # grab all of our messages and filter
    for item in r.inbox.unread(limit=None):
        if item.subject == 'username mention':
            mentions.append(item)
            item.mark_read()

    # sort them and create posts where necessary
    for mention in mentions:
        logging.info('Received mention! ID {}'.format(mention))

        if not is_valid(mention.parent_id, redis_server):
            # Do our check here to make sure we can actually work on this one and
            # that we haven't already posted about it. We use the full ID here
            # instead of the cleaned one, just in case.
            logging.info(id_already_handled_in_db.format(mention.parent_id))
            continue

        process_mention(mention, r, tor, redis_server, Context)

    # comment replies
    replies = []
    for item in r.inbox.unread(limit=None):
        if item.subject == 'comment reply':
            replies.append(item)
            item.mark_read()

    for reply in replies:
        # if 'thank' in reply.body.lower():
        #     respond_to_thanks(reply)
        #     continue
        if 'claim' in reply.body.lower():
            process_claim(reply, r)
        if 'done' in reply.body.lower():
            process_done(reply, r, tor, Context)

def check_submissions(subreddit, Context):
    """
    Loops through all of the subreddits that have opted in and pulls
    the 100 newest submissions. It checks the domain of the submission
    against the domain lists and hands off the post to process_post()
    for formatting and posting on ToR.

    :param subreddit: String. A valid subreddit name.
    :param Context: the Context object.
    :return: None.
    """
    if subreddit in Context.subreddit_members:
        sr = r.subreddit(subreddit).new(limit=10)
    else:
        sr = r.subreddit(subreddit).new(limit=50)
        Context.subreddit_members.append(subreddit)
        update_wiki_page(
            'subreddits/members',
            '\r\n'.join(Context.subreddit_members),
            tor)

    for post in sr:
        if (
            post.domain in Context.image_domains or
            post.domain in Context.audio_domains or
            post.domain in Context.video_domains
        ):
            process_post(post, tor, redis_server, Context)

def set_meta_flair_on_other_posts(transcribersofreddit, Context):
    """
    Loops through the 25 newest posts on ToR and sets the flair to
    'Meta' for any post that is not authored by the bot or any of
    the moderators.

    :param transcribersofreddit: The Subreddit object for ToR.
    :param Context: the active context object.
    :return: None.
    """
    for post in transcribersofreddit.new(limit=10):

        if (
            post.author != r.redditor('transcribersofreddit') and
            post.author not in Context.tor_mods and
            post.link_flair_text != flair.meta
        ):
            logging.info(
                'Flairing post {} by author {} with Meta.'.format(
                    post.fullname, post.author
                )
            )
            flair_post(post, flair.meta)

def initialize(tor):
    populate_domain_lists()
    logging.info('Domains loaded.')
    populate_subreddit_lists()
    logging.info('Subreddits loaded.')
    populate_formatting()
    logging.info('Formatting loaded.')
    populate_header()
    logging.info('Header loaded.')
    populate_moderators(tor)
    logging.info('Mod list loaded.')

    logging.info('Initialization complete.')

if __name__ == '__main__':
    r = Reddit('bot')  # load from config file
    configure_logging()
    try:
        redis_server = redis.StrictRedis(host='localhost', port=6379, db=0)
        redis_server.ping()
    except redis.exceptions.ConnectionError:
        logging.error("Redis server is not running! Exiting!")
        sys.exit(1)
    tor = r.subreddit('transcribersofreddit')
    initialize(tor)

    try:
        number_of_loops = 0
        # primary loop
        while True:
            check_inbox()
            for sub in Context.subreddits_to_check:
                check_submissions(sub, Context)
            set_meta_flair_on_other_posts(tor, Context)
            logging.debug('Looping!')
            number_of_loops += 1
            if number_of_loops > 9:
                # reload configuration every ten loops
                initialize(tor)
                number_of_loops = 0
            if Context.debug_sleep:
                time.sleep(60)

    except KeyboardInterrupt:
        logging.error('User triggered shutdown. Shutting down.')
<|endoftext|>