<commit_before>from __future__ import absolute_import
from __future__ import division
#from __future__ import print_function

import tensorflow as tf
import numpy as np
import argparse
import os
from os import path
import json
import glob
import random
import collections
import math
import time
from data_io import read_kaldi_ark_from_scp

data_base_dir = "/data/data2/scratch/bagchid/specGAN"
def read_mats(uid, offset, batch_size, file_name):
    #Read a buffer containing 10*batch_size+offset
    #Returns a line number of the scp file
    scp_fn = path.join(data_base_dir, file_name)
    ark_dict,uid = read_kaldi_ark_from_scp(uid, offset, batch_size, scp_fn, data_base_dir)
    return ark_dict,uid

def create_batch(ark_dict, batch_number, batch_size):
    def create_permutations(ids):
        return np.random.permutation(ids)

    #idnums = [i for (i,j) in enumerate(ids)]

    rperm = create_permutations(idnums)
    sub_ids = [ids[rperm[i]] for i in xrange(batch_number*batch_size, (batch_number+1)*batch_size)]
    mats = [ark_dict[i].T for i in sub_ids]
    return mats

def create_model():
    def create_generator(generator_inputs):
        layers = []
        with tf.variable_scope("gen_layer_1"):
            output = dense(generator_inputs)
            layers.append(output)
            for _ in range(nlayers-1):
                with tf.variable_scope("gen_layer_%d" % (len(layers) + 1)):
                    rectified = dense(layers[-1])
                    output = batchnorm(convolved)
                    layers.append(output)
        return layers[-1]

    def create_discriminator(discrim_inputs, discrim_targets):
        n_layers = 6
        layers = []
        with tf.variable_scope("disc_layer_1"):
            output = dense(generator_inputs)
            layers.append(output)
            for _ in range(nlayers-1):
                with tf.variable_scope("disc_layer_%d" % (len(layers) + 1)):
                    rectified = dense(layers[-1])
                    output = batchnorm(convolved)
                    layers.append(output)
        return layers[-1]

    with tf.variable_scope("generator") as scope:
        out_channels = int(targets.get_shape()[-1])
        outputs = create_generator(inputs, out_channels)

    with tf.name_scope("discriminator_real"):
        with tf.variable_scope("discriminator"):
            predict_real = create_discriminator(inputs, targets)

    with tf.name_scope("discriminator_fake"):
        with tf.variable_scope("discriminator", reuse=True):
            predict_fake = create_discriminator(inputs, outputs)
    with tf.name_scope("discriminator_loss"):
        discrim_loss = tf.reduce_mean(-(tf.log(predict_real + EPS) + tf.log(1 - predict_fake + EPS)))

    with tf.name_scope("generator_loss"):
        gen_loss_GAN = tf.reduce_mean(-tf.log(predict_fake + EPS))
        gen_loss_L1 = tf.reduce_mean(tf.abs(targets - outputs))
        gen_loss = gen_loss_GAN * a.gan_weight + gen_loss_L1 * a.l1_weight

def next_batch(config):
    batch_index = config['batch_index']
    batch_size = config['batch_size']
    offset_frames_noisy = config['offset_frames_noisy']
    offset_frames_clean = config['offset_frames_clean']

    def create_buffer(uid, offset):
        ark_dict_noisy,uid_new= read_mats(uid,offset,batch_size,"data-spectrogram/train_si84_noisy/feats.scp")
        ark_dict_clean,uid_new = read_mats(uid,offset,batch_size,"data-spectrogram/train_si84_clean/feats.scp")

        ids_noisy = sorted(ark_dict_noisy.keys())
        mats_noisy = [ark_dict_noisy[i] for i in ids_noisy]
        mats2_noisy = np.vstack(mats_noisy)
        nonlocal offset_frames_noisy
        mats2_noisy = np.concatenate((offset_frames_noisy,mats2_noisy),axis=0)

        ids_clean = sorted(ark_dict_clean.keys())
        mats_clean = [ark_dict_clean[i] for i in ids_clean]
        mats2_clean = np.vstack(mats_clean)
        nonlocal offset_frames_clean
        mats2_clean = np.concatenate((offset_frames_clean,mats2_clean),axis=0)

        if mats2_noisy.shape[0]>=(batch_size*10):
            offset_frames_noisy = mats2_noisy[batch_size*10:]
            mats2_noisy = mats2_noisy[:batch_size*10]
            offset_frames_clean = mats2_clean[batch_size*10:]
            mats2_clean = mats2_clean[:batch_size*10]
            offset = offset_frames_noisy.shape[0]
        return mats2_noisy, mats2_clean, uid_new, offset

    if batch_index==0:
        frame_buffer_noisy, frame_buffer_clean, uid_new, offset = create_buffer(config['uid'], config['offset'])
    else:
        frame_buffer_noisy = config['frame_buffer_noisy']
        frame_buffer_clean = config['frame_buffer_clean']
        uid_new = config['uid']
        offset = config['offset']

    start = batch_index*batch_size
    end = min((batch_index+1)*batch_size,frame_buffer_noisy.shape[0])
    config = {'batch_size':batch_size, 'batch_index':(batch_index+1)%10, 'uid':uid_new, 'offset':offset, 'offset_frames_noisy':offset_frames_noisy, 'offset_frames_clean':offset_frames_clean, 'frame_buffer_noisy':frame_buffer_noisy, 'frame_buffer_clean':frame_buffer_clean}

    return (frame_buffer_noisy[start:end], frame_buffer_clean[start:end], config)

def placeholder_inputs():
    noisy_placeholder = tf.placeholder(tf.float32, shape=(batch_size,257))
    clean_placeholder = tf.placeholder(tf.float32, shape=(batch_size,257))
    return noisy_placeholder, clean_placeholder

def fill_feed_dict():
    batch_size=100000
    batch_index = 0
    offset_frames_noisy = np.array([], dtype=np.float32).reshape(0,257)
    offset_frames_clean = np.array([], dtype=np.float32).reshape(0,257)
    frame_buffer_clean = np.array([], dtype=np.float32)
    frame_buffer_noisy = np.array([], dtype=np.float32)

    config = {'batch_size':batch_size, 'batch_index':0, 'uid':0, 'offset':0, 'offset_frames_noisy':offset_frames_noisy, 'offset_frames_clean':offset_frames_clean, 'frame_buffer_clean':frame_buffer_clean, 'frame_buffer_noisy':frame_buffer_noisy}

    while(True):
        noisy_feed, clean_feed, config = next_batch(config)
        if noisy_feed.shape[0]<batch_size:
            break

    #feed_dict = {noisy_pl:noisy_feed, clean_pl:clean_feed,}
    return True

def main():
    fill_feed_dict()

if __name__=='__main__':
    main()

<commit_msg>shuffled the buffer contents<commit_after>from __future__ import absolute_import
from __future__ import division
#from __future__ import print_function

import tensorflow as tf
import numpy as np
import argparse
import os
from os import path
import json
import glob
import random
import collections
import math
import time
from data_io import read_kaldi_ark_from_scp

data_base_dir = "/data/data2/scratch/bagchid/specGAN"
def read_mats(uid, offset, batch_size, file_name):
    #Read a buffer containing 10*batch_size+offset
    #Returns a line number of the scp file
    scp_fn = path.join(data_base_dir, file_name)
    ark_dict,uid = read_kaldi_ark_from_scp(uid, offset, batch_size, scp_fn, data_base_dir)
    return ark_dict,uid

def create_batch(ark_dict, batch_number, batch_size):
    def create_permutations(ids):
        return np.random.permutation(ids)

    #idnums = [i for (i,j) in enumerate(ids)]

    rperm = create_permutations(idnums)
    sub_ids = [ids[rperm[i]] for i in xrange(batch_number*batch_size, (batch_number+1)*batch_size)]
    mats = [ark_dict[i].T for i in sub_ids]
    return mats

def create_model():
    def create_generator(generator_inputs):
        layers = []
        with tf.variable_scope("gen_layer_1"):
            output = dense(generator_inputs)
            layers.append(output)
            for _ in range(nlayers-1):
                with tf.variable_scope("gen_layer_%d" % (len(layers) + 1)):
                    rectified = dense(layers[-1])
                    output = batchnorm(convolved)
                    layers.append(output)
        return layers[-1]

    def create_discriminator(discrim_inputs, discrim_targets):
        n_layers = 6
        layers = []
        with tf.variable_scope("disc_layer_1"):
            output = dense(generator_inputs)
            layers.append(output)
            for _ in range(nlayers-1):
                with tf.variable_scope("disc_layer_%d" % (len(layers) + 1)):
                    rectified = dense(layers[-1])
                    output = batchnorm(convolved)
                    layers.append(output)
        return layers[-1]

    with tf.variable_scope("generator") as scope:
        out_channels = int(targets.get_shape()[-1])
        outputs = create_generator(inputs, out_channels)

    with tf.name_scope("discriminator_real"):
        with tf.variable_scope("discriminator"):
            predict_real = create_discriminator(inputs, targets)

    with tf.name_scope("discriminator_fake"):
        with tf.variable_scope("discriminator", reuse=True):
            predict_fake = create_discriminator(inputs, outputs)
    with tf.name_scope("discriminator_loss"):
        discrim_loss = tf.reduce_mean(-(tf.log(predict_real + EPS) + tf.log(1 - predict_fake + EPS)))

    with tf.name_scope("generator_loss"):
        gen_loss_GAN = tf.reduce_mean(-tf.log(predict_fake + EPS))
        gen_loss_L1 = tf.reduce_mean(tf.abs(targets - outputs))
        gen_loss = gen_loss_GAN * a.gan_weight + gen_loss_L1 * a.l1_weight

def next_batch(config):
    batch_index = config['batch_index']
    batch_size = config['batch_size']
    offset_frames_noisy = config['offset_frames_noisy']
    offset_frames_clean = config['offset_frames_clean']

    def create_buffer(uid, offset):
        ark_dict_noisy,uid_new= read_mats(uid,offset,batch_size,"data-spectrogram/train_si84_noisy/feats.scp")
        ark_dict_clean,uid_new = read_mats(uid,offset,batch_size,"data-spectrogram/train_si84_clean/feats.scp")

        ids_noisy = sorted(ark_dict_noisy.keys())
        mats_noisy = [ark_dict_noisy[i] for i in ids_noisy]
        mats2_noisy = np.vstack(mats_noisy)
        nonlocal offset_frames_noisy
        mats2_noisy = np.concatenate((offset_frames_noisy,mats2_noisy),axis=0)

        ids_clean = sorted(ark_dict_clean.keys())
        mats_clean = [ark_dict_clean[i] for i in ids_clean]
        mats2_clean = np.vstack(mats_clean)
        nonlocal offset_frames_clean
        mats2_clean = np.concatenate((offset_frames_clean,mats2_clean),axis=0)

        if mats2_noisy.shape[0]>=(batch_size*10):
            offset_frames_noisy = mats2_noisy[batch_size*10:]
            mats2_noisy = mats2_noisy[:batch_size*10]
            offset_frames_clean = mats2_clean[batch_size*10:]
            mats2_clean = mats2_clean[:batch_size*10]
            offset = offset_frames_noisy.shape[0]
        return mats2_noisy, mats2_clean, uid_new, offset

    if batch_index==0:
        frame_buffer_noisy, frame_buffer_clean, uid_new, offset = create_buffer(config['uid'], config['offset'])
    else:
        frame_buffer_noisy = config['frame_buffer_noisy']
        frame_buffer_clean = config['frame_buffer_clean']
        uid_new = config['uid']
        offset = config['offset']

    start = batch_index*batch_size
    end = min((batch_index+1)*batch_size,frame_buffer_noisy.shape[0])
    config = {'batch_size':batch_size, 'batch_index':(batch_index+1)%10, 'uid':uid_new, 'offset':offset, 'offset_frames_noisy':offset_frames_noisy, 'offset_frames_clean':offset_frames_clean, 'frame_buffer_noisy':frame_buffer_noisy, 'frame_buffer_clean':frame_buffer_clean}

    return (frame_buffer_noisy[start:end], frame_buffer_clean[start:end], config)

def placeholder_inputs():
    noisy_placeholder = tf.placeholder(tf.float32, shape=(batch_size,257))
    clean_placeholder = tf.placeholder(tf.float32, shape=(batch_size,257))
    return noisy_placeholder, clean_placeholder

def fill_feed_dict():
    batch_size=100000
    batch_index = 0
    offset_frames_noisy = np.array([], dtype=np.float32).reshape(0,257)
    offset_frames_clean = np.array([], dtype=np.float32).reshape(0,257)
    frame_buffer_clean = np.array([], dtype=np.float32)
    frame_buffer_noisy = np.array([], dtype=np.float32)

    config = {'batch_size':batch_size, 'batch_index':0, 'uid':0, 'offset':0, 'offset_frames_noisy':offset_frames_noisy, 'offset_frames_clean':offset_frames_clean, 'frame_buffer_clean':frame_buffer_clean, 'frame_buffer_noisy':frame_buffer_noisy}

    while(True):
        noisy_feed, clean_feed, config = next_batch(config)
        A = np.random.permutation(batch_size*10)
        noisy_feed = noisy_feed[A]
        clean_feed = clean_feed[A]
        if noisy_feed.shape[0]<batch_size:
            break

    #feed_dict = {noisy_pl:noisy_feed, clean_pl:clean_feed,}
    return True

def main():
    fill_feed_dict()

if __name__=='__main__':
    main()

<|endoftext|>


<commit_before>from . import ChadoTestCase, ci

class LoadTest(ChadoTestCase):

    def test_add_blast(self):

        # Setup testing data
        org = self._create_fake_org()
        an = self._create_fake_an()
        blast_file_path = "./test-data/blast.xml"
        an_blast = self._create_fake_an('BLAST')
        an_blast_id = an_blast['analysis_id']
        self.ci.feature.load_fasta(fasta="./test-data/genome.fa", analysis_id=an['analysis_id'], organism_id=org['organism_id'], sequence_type='supercontig')
        self.ci.feature.load_gff(gff="./test-data/annot.gff", analysis_id=an['analysis_id'], organism_id=org['organism_id'], no_seq_compute=True)
        self.ci.load.blast(an_blast_id, blast_file_path, blastdb="swissprot:display", search_keywords=True, query_type="mRNA")

        feats = self.ci.feature.get_features(organism_id=org['organism_id'], uniquename='PAC:18136217', analysis_id=an['analysis_id'])
        assert feats != [], "Feature PAC:18136217 was not created"
        feat_id = feats[0]['feature_id']

        res = self.ci.session.query(self.ci.model.analysisfeatureprop) \
            .join(self.ci.model.analysisfeature, self.ci.model.analysisfeature.analysisfeature_id == self.ci.model.analysisfeatureprop.analysisfeature_id) \
            .filter(self.ci.model.analysisfeature.feature_id == feat_id, self.ci.model.analysisfeature.analysis_id == an_blast_id)
        assert res.count(), "No result in analysisfeatureprop table for this feature and analysis"
        assert res.count() == 1, "More than one result in analysisfeatureprop table for this feature and analysis"

        cvterm_id = self.ci.get_cvterm_id("analysis_blast_output_iteration_hits", "tripal")
        res = res.filter(self.ci.model.analysisfeatureprop.type_id == cvterm_id)
        assert res.count(), "Cvterm is not matching analysis_blast_output_iteration_hits"

    def test_add_interpro(self):

        # Setup testing data
        org = self._create_fake_org()
        an = self._create_fake_an()
        interpro_file_path = "./test-data/interproscan.xml"
        an_interpro = self._create_fake_an('INTERPRO')
        an_interpro_id = an_interpro['analysis_id']
        self.ci.feature.load_fasta(fasta="./test-data/genome.fa", analysis_id=an['analysis_id'], organism_id=org['organism_id'], sequence_type='supercontig')
        self.ci.feature.load_gff(gff="./test-data/annot.gff", analysis_id=an['analysis_id'], organism_id=org['organism_id'], no_seq_compute=True)
        feats = self.ci.feature.get_features(organism_id=org['organism_id'], uniquename='PAC:18136217', analysis_id=an['analysis_id'])
        assert len(feats) != 0, "Feature PAC:18136217 was not created"
        feat_id = feats[0]['feature_id']

        cv_terms = self.ci.feature.get_feature_cvterm(feat_id)
        assert cv_terms == [], "CV term list is not empty before adding terms!"

        self.ci.load.interpro(an_interpro_id, interpro_file_path, parse_go=True, query_type='mRNA')

        cv_terms = self.ci.feature.get_feature_cvterm(feat_id)
        assert len(cv_terms) == 8, "Number of loaded CV term is not 8!"
        interpro_terms = [d for d in cv_terms if d['db_name'] == "INTERPRO"]
        assert len(interpro_terms) == 5, "Number of INTERPRO cvterms is not 5!"
        go_terms = [d for d in cv_terms if d['db_name'] == "GO"]
        assert len(go_terms) == 3, "Number of GO cvterms is not 3!"

        test_terms = [d for d in cv_terms if d['cvterm_name'] == "Kinase-like_dom_sf"]
        assert len(test_terms) == 1, "Term Kinase-like_dom_sf is not loaded!"
        test_term = test_terms[0]
        assert test_term['dbxref_accession'] == "IPR011009", "Wrong DBXREF accession number"
        assert test_term['db_name'] == "INTERPRO", "Wrong DB name"
        assert test_term['cvterm_definition'] == "Protein kinase-like domain superfamily", "Wrong cvterm definition"

        res = self.ci.ession.query(self.ci.model.analysisfeatureprop) \
            .join(self.ci.model.analysisfeature, self.ci.model.analysisfeature.analysisfeature_id == self.ci.model.analysisfeatureprop.analysisfeature_id) \
            .filter(self.ci.model.analysisfeature.feature_id == feat_id, self.ci.model.analysisfeature.analysis_id == an_interpro_id)
        assert res.count(), "No result in analysisfeatureprop table for this feature and analysis"
        assert res.count() == 9, "Number of results not matching in analysisfeatureprop table for this feature and analysis : " + res.count()

        cvterm_id = self.ci.get_cvterm_id("analysis_interpro_xmloutput_hit", "tripal")
        res = res.filter(self.ci.model.analysisfeatureprop.type_id == cvterm_id)
        assert res.count(), "Cvterm analysis_interpro_xmloutput_hit not found in table"

    def setUp(self):

        self.ci = ci
        ci.feature.delete_features()
        ci.organism.delete_organisms()
        ci.analysis.delete_analyses()

        ci.session.commit()

    def tearDown(self):
        ci.feature.delete_features()
        ci.organism.delete_organisms()
        ci.analysis.delete_analyses()

        ci.session.commit()
<commit_msg>tests<commit_after>from . import ChadoTestCase, ci

class LoadTest(ChadoTestCase):

    def test_add_blast(self):

        # Setup testing data
        org = self._create_fake_org()
        an = self._create_fake_an()
        blast_file_path = "./test-data/blast.xml"
        an_blast = self._create_fake_an('BLAST')
        an_blast_id = an_blast['analysis_id']
        self.ci.feature.load_fasta(fasta="./test-data/genome.fa", analysis_id=an['analysis_id'], organism_id=org['organism_id'], sequence_type='supercontig')
        self.ci.feature.load_gff(gff="./test-data/annot.gff", analysis_id=an['analysis_id'], organism_id=org['organism_id'], no_seq_compute=True)
        self.ci.load.blast(an_blast_id, blast_file_path, blastdb="swissprot:display", search_keywords=True, query_type="mRNA")

        feats = self.ci.feature.get_features(organism_id=org['organism_id'], uniquename='PAC:18136217', analysis_id=an['analysis_id'])
        assert feats != [], "Feature PAC:18136217 was not created"
        feat_id = feats[0]['feature_id']

        res = self.ci.session.query(self.ci.model.analysisfeatureprop) \
            .join(self.ci.model.analysisfeature, self.ci.model.analysisfeature.analysisfeature_id == self.ci.model.analysisfeatureprop.analysisfeature_id)
        print(res.count())
        print(feat_id)
        print(an_blast_id)
        res = res.filter(self.ci.model.analysisfeature.feature_id == feat_id, self.ci.model.analysisfeature.analysis_id == an_blast_id)
        print(res.count())
        assert res.count(), "No result in analysisfeatureprop table for this feature and analysis"
        assert res.count() == 1, "More than one result in analysisfeatureprop table for this feature and analysis"

        cvterm_id = self.ci.get_cvterm_id("analysis_blast_output_iteration_hits", "tripal")
        res = res.filter(self.ci.model.analysisfeatureprop.type_id == cvterm_id)
        assert res.count(), "Cvterm is not matching analysis_blast_output_iteration_hits"

    def test_add_interpro(self):

        # Setup testing data
        org = self._create_fake_org()
        an = self._create_fake_an()
        interpro_file_path = "./test-data/interproscan.xml"
        an_interpro = self._create_fake_an('INTERPRO')
        an_interpro_id = an_interpro['analysis_id']
        self.ci.feature.load_fasta(fasta="./test-data/genome.fa", analysis_id=an['analysis_id'], organism_id=org['organism_id'], sequence_type='supercontig')
        self.ci.feature.load_gff(gff="./test-data/annot.gff", analysis_id=an['analysis_id'], organism_id=org['organism_id'], no_seq_compute=True)
        feats = self.ci.feature.get_features(organism_id=org['organism_id'], uniquename='PAC:18136217', analysis_id=an['analysis_id'])
        assert len(feats) != 0, "Feature PAC:18136217 was not created"
        feat_id = feats[0]['feature_id']

        cv_terms = self.ci.feature.get_feature_cvterm(feat_id)
        assert cv_terms == [], "CV term list is not empty before adding terms!"

        self.ci.load.interpro(an_interpro_id, interpro_file_path, parse_go=True, query_type='mRNA')

        cv_terms = self.ci.feature.get_feature_cvterm(feat_id)
        assert len(cv_terms) == 8, "Number of loaded CV term is not 8!"
        interpro_terms = [d for d in cv_terms if d['db_name'] == "INTERPRO"]
        assert len(interpro_terms) == 5, "Number of INTERPRO cvterms is not 5!"
        go_terms = [d for d in cv_terms if d['db_name'] == "GO"]
        assert len(go_terms) == 3, "Number of GO cvterms is not 3!"

        test_terms = [d for d in cv_terms if d['cvterm_name'] == "Kinase-like_dom_sf"]
        assert len(test_terms) == 1, "Term Kinase-like_dom_sf is not loaded!"
        test_term = test_terms[0]
        assert test_term['dbxref_accession'] == "IPR011009", "Wrong DBXREF accession number"
        assert test_term['db_name'] == "INTERPRO", "Wrong DB name"
        assert test_term['cvterm_definition'] == "Protein kinase-like domain superfamily", "Wrong cvterm definition"

        res = self.ci.ession.query(self.ci.model.analysisfeatureprop) \
            .join(self.ci.model.analysisfeature, self.ci.model.analysisfeature.analysisfeature_id == self.ci.model.analysisfeatureprop.analysisfeature_id) \
            .filter(self.ci.model.analysisfeature.feature_id == feat_id, self.ci.model.analysisfeature.analysis_id == an_interpro_id)
        assert res.count(), "No result in analysisfeatureprop table for this feature and analysis"
        assert res.count() == 9, "Number of results not matching in analysisfeatureprop table for this feature and analysis : " + res.count()

        cvterm_id = self.ci.get_cvterm_id("analysis_interpro_xmloutput_hit", "tripal")
        res = res.filter(self.ci.model.analysisfeatureprop.type_id == cvterm_id)
        assert res.count(), "Cvterm analysis_interpro_xmloutput_hit not found in table"

    def setUp(self):

        self.ci = ci
        ci.feature.delete_features()
        ci.organism.delete_organisms()
        ci.analysis.delete_analyses()

        ci.session.commit()

    def tearDown(self):
        ci.feature.delete_features()
        ci.organism.delete_organisms()
        ci.analysis.delete_analyses()

        ci.session.commit()
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import with_statement

from errno import EACCES
from os.path import realpath
from sys import argv, exit
from threading import Lock
from pprint import pprint
from json import dumps as serialize
from json import loads as unserialize
from sys import exit
from time import sleep

import os

from fusepy.fuse import FUSE, FuseOSError, Operations

DEFAULT_CHUNK_SIZE = '2M'

class LoggingMixIn:
    def __call__(self, op, path, *args):
        if op in ['access', 'getattr', 'statfs', 'getxattr']:
            return getattr(self, op)(path, *args)
        print '->', op, path, repr(args)
        ret = '[Unhandled Exception]'
        try:
            ret = getattr(self, op)(path, *args)
            return ret
        except OSError, e:
            ret = str(e)
            raise
        finally:
            print '<-', op, repr(ret)

def parseSize(text):
    prefixes = {
        'b': 1,
        'k': 1024, 'ki': 1024, 'kb': 1024,
        'm': 1024 * 1024, 'mb': 1024 * 1024,
        'g': 1024 * 1024 * 1024, 'gb': 1024 * 1024 * 1024
        }
    num = ""
    text = str(text).strip()
    while text and text[0:1].isdigit() or text[0:1] == '.':
        num += text[0]
        text = text[1:]
    num = float(num)
    letter = text.strip().lower()
    return int(num * (prefixes[letter] if letter in prefixes else 1))

def saveManifest(path, sf):
    fh = os.open(path, os.O_WRONLY | os.O_CREAT, 0777)
    os.write(fh, serialize(sf) + '\n')
    os.close(fh)

def getManifest(path):
    if not os.path.isfile(path):
        return None
    fh = open(path)
    data = fh.read()
    fh.close()
    return unserialize(data)

def relativeFileOffset(manifest, offset = 0, size = None, chunk_size = None):
    if chunk_size is None:
        chunk_size = manifest.get('chunk_size')
    start_file, start_offset = int(offset / chunk_size), int(offset % chunk_size)
    if size is None:
        return start_file, start_offset
    end_file, end_offset = int((offset + size) / chunk_size), int((offset + size) % chunk_size)
    #print "offset", offset, "size", size, "end_file", end_file, "end_offset", end_offset
    return start_file, start_offset, end_file, end_offset

def chunkPath(path, nth):
    return path + '.sf-%d' % nth

class SplitFS(LoggingMixIn, Operations):
    def __init__(self, root, sleep = None, chunk_size = parseSize(DEFAULT_CHUNK_SIZE)):
        self.root = realpath(root)
        self.rwlock = Lock()
        self.sleep = sleep
        self.manifests = {}
        self.chunk_size = chunk_size

    def __del__(self):
        pass

    def __call__(self, op, path, *args):
        if self.sleep:
            sleep(self.sleep)
        return super(SplitFS, self).__call__(op, self.root + path, *args)

    chmod = os.chmod
    chown = os.chown

    def create(self, path, mode):
        saveManifest(path, {'mode': mode, 'path': path, 'chunk_size': self.chunk_size})
        return 0

    def getattr(self, path, fh=None):
        st = os.lstat(path)
        ret = dict((key, getattr(st, key)) for key in ('st_atime', 'st_ctime', 'st_gid', 'st_mode', 'st_mtime', 'st_nlink', 'st_size', 'st_uid'))
        manifest = getManifest(path)
        print manifest
        if manifest and manifest.get('size', None) is not None:
            ret['st_size'] = manifest.get('size')
        return ret

    def read(self, path, size, offset, fh = None):
        with self.rwlock:
            manifest = getManifest(path)
            chunk_size = manifest.get('chunk_size')
            start_file, start_offset, end_file, end_offset = relativeFileOffset(manifest, offset = offset, size = size)
            #print "READ: path=%s,size=%d,offset=%d,fh=%d, start=%s, end=%s, chunk_size: %d\n" % (path, size, offset, fh, (start_file, start_offset,), (end_file, end_offset,), chunk_size)
            buff = ""
            for nth in xrange(start_file, end_file + 1):
                chunk = open(chunkPath(path, nth))
                read_size = chunk_size
                if start_file == nth and start_offset:
                    chunk.seek(start_offset)
                    read_size -= start_offset
                if end_file == nth:
                    read_size -= (chunk_size - end_offset)
                buff += chunk.read(read_size)
                chunk.close()
            return buff

    def readdir(self, path, fh):
        return ['.', '..'] + list(filter(lambda x: x.find('.sf-') == -1, os.listdir(path)))

    def write(self, path, data, offset, fh):
        with self.rwlock:
            manifest = getManifest(path)
            chunk_size = manifest.get('chunk_size')
            start_file, start_offset, end_file, end_offset = relativeFileOffset(manifest, offset = offset, size = len(data))
            wrote_size = 0
            for nth in xrange(start_file, end_file + 1):
                chunk = os.open(chunkPath(path, nth), os.O_WRONLY | os.O_CREAT, 0777)
                write_size = chunk_size
                if start_file == nth and start_offset:
                    os.lseek(chunk, start_offset, os.SEEK_SET)
                    write_size -= start_offset
                if end_file == nth:
                    write_size -= (chunk_size - end_offset)
                print "____________"
                #print data
                print chunk
                print wrote_size
                print write_size
                print wrote_size + write_size
                print "____________"
                wrote_size += os.write(chunk, data[wrote_size:(wrote_size + write_size)])
                os.close(chunk)
                manifest['size'] = max(manifest.get('size'), nth * chunk_size + end_offset)
            saveManifest(path, manifest)
            return wrote_size

    def truncate(self, path, length, fh=None):
        pass

if __name__ == "__main__":
    if len(argv) != 3:
        print 'usage: %s <root> <mountpoint>' % argv[0]
        exit(1)
    fuse = FUSE(SplitFS(argv[1], sleep = 0, chunk_size = parseSize(DEFAULT_CHUNK_SIZE)), argv[2], foreground=True, raw_fi = False)
<commit_msg>enhanced debug<commit_after>#!/usr/bin/env python

from __future__ import with_statement

from errno import EACCES
from os.path import realpath
from sys import argv, exit
from threading import Lock
from pprint import pprint
from json import dumps as serialize
from json import loads as unserialize
from sys import exit
from time import sleep
from textwrap import wrap

import os

from fusepy.fuse import FUSE, FuseOSError, Operations

DEFAULT_CHUNK_SIZE = '20M'
FULL_DEBUG = True

class LoggingMixIn:
    def __call__(self, op, path, *args):
        if op in ['access', 'getattr', 'statfs', 'getxattr']:
            return getattr(self, op)(path, *args)
        if FULL_DEBUG:
            print '->', op, path, wrap(repr(args), 100)[0]
        else:
            print '->', op, path
        ret = '[Unhandled Exception]'
        try:
            ret = getattr(self, op)(path, *args)
            return ret
        except OSError, e:
            ret = str(e)
            raise
        finally:
            if FULL_DEBUG:
                print '<-', op, wrap(repr(ret), 100)[0]
            else:
                print '<-', op

def parseSize(text):
    prefixes = {
        'b': 1,
        'k': 1024, 'ki': 1024, 'kb': 1024,
        'm': 1024 * 1024, 'mb': 1024 * 1024,
        'g': 1024 * 1024 * 1024, 'gb': 1024 * 1024 * 1024
        }
    num = ""
    text = str(text).strip()
    while text and text[0:1].isdigit() or text[0:1] == '.':
        num += text[0]
        text = text[1:]
    num = float(num)
    letter = text.strip().lower()
    return int(num * (prefixes[letter] if letter in prefixes else 1))

def saveManifest(path, sf):
    fh = os.open(path, os.O_WRONLY | os.O_CREAT, 0777)
    os.write(fh, serialize(sf) + '\n')
    os.close(fh)

def getManifest(path):
    if not os.path.isfile(path):
        return None
    fh = open(path)
    data = fh.read()
    fh.close()
    return unserialize(data)

def relativeFileOffset(manifest, offset = 0, size = None, chunk_size = None):
    if chunk_size is None:
        chunk_size = manifest.get('chunk_size')
    start_file, start_offset = int(offset / chunk_size), int(offset % chunk_size)
    if size is None:
        return start_file, start_offset
    end_file, end_offset = int((offset + size) / chunk_size), int((offset + size) % chunk_size)
    #print "offset", offset, "size", size, "end_file", end_file, "end_offset", end_offset
    return start_file, start_offset, end_file, end_offset

def chunkPath(path, nth):
    return path + '.sf-%d' % nth

class SplitFS(LoggingMixIn, Operations):
    def __init__(self, root, sleep = None, chunk_size = parseSize(DEFAULT_CHUNK_SIZE)):
        self.root = realpath(root)
        self.rwlock = Lock()
        self.sleep = sleep
        self.manifests = {}
        self.chunk_size = chunk_size

    def __del__(self):
        pass

    def __call__(self, op, path, *args):
        if self.sleep:
            sleep(self.sleep)
        return super(SplitFS, self).__call__(op, self.root + path, *args)

    chmod = os.chmod
    chown = os.chown

    def create(self, path, mode):
        saveManifest(path, {'mode': mode, 'path': path, 'chunk_size': self.chunk_size})
        return 0

    def getattr(self, path, fh=None):
        st = os.lstat(path)
        ret = dict((key, getattr(st, key)) for key in ('st_atime', 'st_ctime', 'st_gid', 'st_mode', 'st_mtime', 'st_nlink', 'st_size', 'st_uid'))
        manifest = getManifest(path)
        #print manifest
        if manifest and manifest.get('size', None) is not None:
            ret['st_size'] = manifest.get('size')
        return ret

    def read(self, path, size, offset, fh = None):
        with self.rwlock:
            manifest = getManifest(path)
            chunk_size = manifest.get('chunk_size')
            start_file, start_offset, end_file, end_offset = relativeFileOffset(manifest, offset = offset, size = size)
            print "READ: path=%s,size=%d,offset=%d,fh=%d, start=%s, end=%s, chunk_size: %d\n" % (path, size, offset, fh, (start_file, start_offset,), (end_file, end_offset,), chunk_size)
            buff = ""
            for nth in xrange(start_file, end_file + 1):
                chunk = open(chunkPath(path, nth))
                read_size = chunk_size
                if start_file == nth and start_offset:
                    chunk.seek(start_offset)
                    read_size -= start_offset
                if end_file == nth:
                    read_size -= (chunk_size - end_offset)
                buff += chunk.read(read_size)
                chunk.close()
            return buff

    def readdir(self, path, fh):
        return ['.', '..'] + list(filter(lambda x: x.find('.sf-') == -1, os.listdir(path)))

    def write(self, path, data, offset, fh):
        with self.rwlock:
            manifest = getManifest(path)
            chunk_size = manifest.get('chunk_size')
            start_file, start_offset, end_file, end_offset = relativeFileOffset(manifest, offset = offset, size = len(data))
            wrote_size = 0
            for nth in xrange(start_file, end_file + 1):
                chunk = os.open(chunkPath(path, nth), os.O_WRONLY | os.O_CREAT, 0777)
                write_size = chunk_size
                if start_file == nth and start_offset:
                    os.lseek(chunk, start_offset, os.SEEK_SET)
                    write_size -= start_offset
                if end_file == nth:
                    write_size -= (chunk_size - end_offset)
                #print "____________"
                ##print data
                #print chunk
                #print wrote_size
                #print write_size
                #print wrote_size + write_size
                #print "____________"
                wrote_size += os.write(chunk, data[wrote_size:(wrote_size + write_size)])
                os.close(chunk)
                manifest['size'] = max(manifest.get('size'), nth * chunk_size + end_offset)
            saveManifest(path, manifest)
            return wrote_size

    def truncate(self, path, length, fh=None):
        pass

if __name__ == "__main__":
    if len(argv) != 3:
        print 'usage: %s <root> <mountpoint>' % argv[0]
        exit(1)
    fuse = FUSE(SplitFS(argv[1], sleep = 0, chunk_size = parseSize(DEFAULT_CHUNK_SIZE)), argv[2], foreground=True, raw_fi = False)
<|endoftext|>


<commit_before>#
# Use the Spotify Web Connect API using credentials auth code and tokens.
#

import json
import requests
import time
import urlparse
import sys
from myopenhab import openhab
from myopenhab import mapValues
from myopenhab import getJSONValue

#   API Gateway
ACCOUNT_URL = 'https://accounts.spotify.com/api/token'
API_ROOT_URL = 'https://api.spotify.com/v1/me/player/'
REDIRECT_URI = 'http://openhabianpi.local:8080/static/spotify-auth.html'

class spotify(object):
    """

    A wrapper for the Spotify Web Connect API

    https://developer.spotify.com/web-api/web-api-connect-endpoint-reference/

    """
    def __init__(self):

        self.debug = False
        self.oh = openhab()

        self.client_id = self.oh.getState('spotify_client_id')
        self.client_secret = self.oh.getState('spotify_client_secret')

        self.access_token = self.oh.getState('spotify_access_token')
        self.refresh_token = self.oh.getState('spotify_refresh_token')
        self.token_issued = self.oh.getState('spotify_token_issued')
        self.token_expiry = self.oh.getState('spotify_token_expiry')

        if(self.token_expiry == "NULL"):
            self.refreshCredentials()
        if (self.access_token == "NULL"):
            self.generateCredentials()
        else:
            if (time.time() > float(self.token_expiry)):
                self.refreshCredentials()

    def generateCredentials(self):
        """
        Generate auth and refresh token for the very first time.
        """

        #   Send OAuth payload to get access_token
        payload = { 'code':self.oh.getState('spotify_auth_code'), 'client_id':self.client_id, 'client_secret':self.client_secret, 'redirect_uri':REDIRECT_URI, 'grant_type':'authorization_code' }

        print "-- Calling Token Service for the first time"

        try:
            r = requests.post(ACCOUNT_URL, data=payload, allow_redirects=False)

            if (self.debug): print r.headers
            if (self.debug): print r.json()
            resp = r.json()

            if(r.status_code == 200):
                access_token = resp['access_token']
                refresh_token = resp['refresh_token']
                expires_in = resp['expires_in']

                #   Set and Save the access token
                self.access_token = access_token
                self.refresh_token = refresh_token
                self.token_expiry = time.time() + float(expires_in)
                self.token_issued = time.strftime("%Y-%m-%dT%H:%M:%S")
                self.saveCredentials()

        except:
            print " -> Error getting token:" + str(sys.exc_info()[1])

    def refreshCredentials(self):
        """
        If previous auth token expired, get a new one with refresh token.
        """

        #   Send OAuth payload to get access_token
        payload = { 'refresh_token':self.refresh_token, 'client_id':self.client_id, 'client_secret':self.client_secret, 'redirect_uri':REDIRECT_URI, 'grant_type':'refresh_token' }

        print "-- Calling Token Refresh Service"

        try:
            r = requests.post(ACCOUNT_URL, data=payload, allow_redirects=False)

            if (self.debug): print r.headers
            if (self.debug): print r.json()
            resp = r.json()

            if(r.status_code == 200):
                access_token = resp['access_token']
                expires_in = resp['expires_in']
                if('refresh_token' in resp):
                    refresh_token = resp['refresh_token']
                    self.refresh_token = refresh_token

                #   Set and Save the access token
                self.access_token = access_token
                self.token_expiry = time.time() + float(expires_in)
                self.token_issued = time.strftime("%Y-%m-%dT%H:%M:%S")
                self.saveCredentials()

        except:
            print " -> Error refreshing token:" + str(sys.exc_info()[1])

    def saveCredentials(self):
        """
        Save current tokens to the openhab.
        """

        self.oh.sendCommand('spotify_access_token',self.access_token)
        self.oh.sendCommand('spotify_refresh_token',self.refresh_token)
        self.oh.sendCommand('spotify_token_expiry',self.token_expiry)
        self.oh.sendCommand('spotify_token_issued',self.token_issued)

    def call(self, path, mode=None, payload=None):
        """
        Call the API at the given path.
        """

        if (time.time() > self.token_expiry):
            self.refreshCredentials()
        headers = {"Authorization": "Bearer " + self.access_token}
        if mode == "POST":
            r = requests.post(API_ROOT_URL + path,  headers=headers, data=payload)
            return r.status_code
        elif mode == "PUT":
            r = requests.put(API_ROOT_URL + path,  headers=headers, data=payload)
            return r.status_code
        else:
            r = requests.get(API_ROOT_URL + path,  headers=headers)
            return r.json()

    def update(self):
        """
        Get a current player state.
        """
        print "-- Calling Service: Update"
        try:
            resp = self.call("")
            if (self.debug): print resp
            if ('item' in resp):

                self.oh.sendCommand('spotify_current_track', getJSONValue(resp, ['item','name']))
                self.oh.sendCommand('spotify_current_artist', getJSONValue(resp, ['item', 'artists', 0, 'name']))
                self.oh.sendCommand('spotify_current_cover', getJSONValue(resp, ['item', 'album', 'images', 1, 'url']))
                self.oh.sendCommand('spotify_current_duration', getJSONValue(resp, ['item', 'duration_ms']))
                self.oh.sendCommand('spotify_current_progress', getJSONValue(resp, ['progress_ms']))
                self.oh.sendCommand('spotify_current_progress', getJSONValue(resp, ['progress_ms']))
                self.oh.sendCommand('spotify_current_playing', mapValues(getJSONValue(resp, ['is_playing']), { 'True': 'ON', 'False': 'OFF' }))
                self.oh.sendCommand('spotify_current_device', getJSONValue(resp, ['device', 'name']))
                self.oh.sendCommand('spotify_current_volume', getJSONValue(resp, ['device', 'volume_percent']))
                self.oh.sendCommand('spotify_current_device_id', getJSONValue(resp, ['device', 'id']))

                print " -> Success"
            else:
                print " -> Item node missing from response :("
        except:
            print " -> Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def volumeUp(self):
        """
        Volume up by 10%
        """
        print "-- Calling Service: Volume Up"
        try:
            vol = int(self.oh.getState('spotify_current_volume'))
            vol = int(round(vol/10)*10 + 10)
            if(vol>100):
                vol = 100
            print " -> Volume To:" + str(vol)
            resp = self.call("volume?volume_percent=" + str(vol),"PUT" )
            self.oh.sendCommand('spotify_current_volume',vol)
            if (self.debug): print resp
        except:
            print " -> VolumeUp Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def volumeDown(self):
        """
        Volume down by 10%
        """
        print "-- Calling Service: Volume Down"
        try:
            vol = int(self.oh.getState('spotify_current_volume'))
            vol = int(round(vol/10)*10 - 10)
            if(vol<0):
                vol = 0
            print "Volume To:" + str(vol)
            resp = self.call("volume?volume_percent=" + str(vol),"PUT" )
            self.oh.sendCommand('spotify_current_volume',vol)
            if (self.debug): print resp
        except:
            print " -> VolumeDown Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def pause(self):
        """
        Pause player
        """
        print "-- Calling Service: Pause"
        try:
            resp = self.call("pause","PUT")
            self.oh.sendCommand('spotify_current_playing',"OFF")
            if (self.debug): print resp
        except:
            print " -> Pause Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def play(self):
        """
        Resume player
        """
        print "-- Calling Service: Play"
        try:
            resp = self.call("play","PUT")
            self.oh.sendCommand('spotify_current_playing',"ON")
            if (self.debug): print resp
        except:
            print " -> Play Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def previous(self):
        """
        Skip to previous track
        """
        print "-- Calling Service: Previous"
        try:
            resp = self.call("previous","POST")
            if (self.debug): print resp
            self.refresh()
        except:
            print " -> Previous Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def next(self):
        """
        Skip to next track
        """
        print "-- Calling Service: Next"
        try:
            resp = self.call("next","POST")
            if (self.debug): print resp
            self.refresh()
        except:
            print " -> Next Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def updateConnectionDateTime(self):
        self.oh.sendCommand('spotify_lastConnectionDateTime',time.strftime("%Y-%m-%dT%H:%M:%S+0000",time.gmtime(time.time())))

def main():

    t1 = time.time()

    c = spotify()

    args = sys.argv

    if(len(args) == 1):
        c.update()
    else:

        if(args[1] == "volume_up"):
            c.volumeUp()
        if(args[1] == "volume_down"):
            c.volumeDown()
        if(args[1] == "play"):
            c.play()
        if(args[1] == "pause"):
            c.pause()
        if(args[1] == "previous"):
            c.previous()
        if(args[1] == "next"):
            c.next()

    c.updateConnectionDateTime()

    t2 = time.time()
    print "Done in " + str(t2-t1) + " seconds"

if __name__ == '__main__':
    main()
<commit_msg>Update spotify.py<commit_after>#
# Use the Spotify Web Connect API using credentials auth code and tokens.
#

import json
import requests
import time
import urlparse
import sys
from myopenhab import openhab
from myopenhab import mapValues
from myopenhab import getJSONValue

#   API Gateway
ACCOUNT_URL = 'https://accounts.spotify.com/api/token'
API_ROOT_URL = 'https://api.spotify.com/v1/me/player/'
REDIRECT_URI = 'http://openhabianpi.local:8080/static/spotify-auth.html'

class spotify(object):
    """

    A wrapper for the Spotify Web Connect API

    https://developer.spotify.com/web-api/web-api-connect-endpoint-reference/

    """
    def __init__(self):

        self.debug = False
        self.oh = openhab()

        self.client_id = self.oh.getState('spotify_client_id')
        self.client_secret = self.oh.getState('spotify_client_secret')

        self.access_token = self.oh.getState('spotify_access_token')
        self.refresh_token = self.oh.getState('spotify_refresh_token')
        self.token_issued = self.oh.getState('spotify_token_issued')
        self.token_expiry = self.oh.getState('spotify_token_expiry')

        if(self.token_expiry == "NULL"):
            self.refreshCredentials()
        if (self.access_token == "NULL"):
            self.generateCredentials()
        else:
            if (time.time() > float(self.token_expiry)):
                self.refreshCredentials()

    def generateCredentials(self):
        """
        Generate auth and refresh token for the very first time.
        """

        #   Send OAuth payload to get access_token
        payload = { 'code':self.oh.getState('spotify_auth_code'), 'client_id':self.client_id, 'client_secret':self.client_secret, 'redirect_uri':REDIRECT_URI, 'grant_type':'authorization_code' }

        print "-- Calling Token Service for the first time"

        try:
            r = requests.post(ACCOUNT_URL, data=payload, allow_redirects=False)

            if (self.debug): print r.headers
            if (self.debug): print r.json()
            resp = r.json()

            if(r.status_code == 200):
                access_token = resp['access_token']
                refresh_token = resp['refresh_token']
                expires_in = resp['expires_in']

                #   Set and Save the access token
                self.access_token = access_token
                self.refresh_token = refresh_token
                self.token_expiry = time.time() + float(expires_in)
                self.token_issued = time.strftime("%Y-%m-%dT%H:%M:%S")
                self.saveCredentials()

        except:
            print " -> Error getting token:" + str(sys.exc_info()[1])

    def refreshCredentials(self):
        """
        If previous auth token expired, get a new one with refresh token.
        """

        #   Send OAuth payload to get access_token
        payload = { 'refresh_token':self.refresh_token, 'client_id':self.client_id, 'client_secret':self.client_secret, 'redirect_uri':REDIRECT_URI, 'grant_type':'refresh_token' }

        print "-- Calling Token Refresh Service"

        try:
            r = requests.post(ACCOUNT_URL, data=payload, allow_redirects=False)

            if (self.debug): print r.headers
            if (self.debug): print r.json()
            resp = r.json()

            if(r.status_code == 200):
                access_token = resp['access_token']
                expires_in = resp['expires_in']
                if('refresh_token' in resp):
                    refresh_token = resp['refresh_token']
                    self.refresh_token = refresh_token

                #   Set and Save the access token
                self.access_token = access_token
                self.token_expiry = time.time() + float(expires_in)
                self.token_issued = time.strftime("%Y-%m-%dT%H:%M:%S")
                self.saveCredentials()

        except:
            print " -> Error refreshing token:" + str(sys.exc_info()[1])

    def saveCredentials(self):
        """
        Save current tokens to the openhab.
        """

        self.oh.sendCommand('spotify_access_token',self.access_token)
        self.oh.sendCommand('spotify_refresh_token',self.refresh_token)
        self.oh.sendCommand('spotify_token_expiry',self.token_expiry)
        self.oh.sendCommand('spotify_token_issued',self.token_issued)

    def call(self, path, mode=None, payload=None):
        """
        Call the API at the given path.
        """

        if (time.time() > self.token_expiry):
            self.refreshCredentials()
        headers = {"Authorization": "Bearer " + self.access_token, "Content-Type": "application/json" }
        if mode == "POST":
            r = requests.post(API_ROOT_URL + path,  headers=headers, data=payload)
            if(r.status_code != 202):
                print r.content
            return r.status_code
        elif mode == "PUT":
            r = requests.put(API_ROOT_URL + path,  headers=headers, data=payload)
            if(r.status_code != 202):
                print r.content
            return r.status_code
        else:
            r = requests.get(API_ROOT_URL + path,  headers=headers)
            if(r.status_code != 202):
                print r.content
            return r.json()

    def update(self):
        """
        Get a current player state.
        """
        print "-- Calling Service: Update"
        try:
            resp = self.call("")
            if (self.debug): print resp
            if ('item' in resp):

                self.oh.sendCommand('spotify_current_track', getJSONValue(resp, ['item','name']))
                self.oh.sendCommand('spotify_current_artist', getJSONValue(resp, ['item', 'artists', 0, 'name']))
                self.oh.sendCommand('spotify_current_cover', getJSONValue(resp, ['item', 'album', 'images', 1, 'url']))
                self.oh.sendCommand('spotify_current_duration', getJSONValue(resp, ['item', 'duration_ms']))
                self.oh.sendCommand('spotify_current_progress', getJSONValue(resp, ['progress_ms']))
                self.oh.sendCommand('spotify_current_progress', getJSONValue(resp, ['progress_ms']))
                self.oh.sendCommand('spotify_current_playing', mapValues(getJSONValue(resp, ['is_playing']), { 'True': 'ON', 'False': 'OFF' }))
                self.oh.sendCommand('spotify_current_device', getJSONValue(resp, ['device', 'name']))
                self.oh.sendCommand('spotify_current_volume', getJSONValue(resp, ['device', 'volume_percent']))
                self.oh.sendCommand('spotify_current_device_id', getJSONValue(resp, ['device', 'id']))

                print " -> Success"
            else:
                print " -> Item node missing from response :("
        except:
            print " -> Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def volumeUp(self):
        """
        Volume up by 10%
        """
        print "-- Calling Service: Volume Up"
        try:
            vol = int(self.oh.getState('spotify_current_volume'))
            vol = int(round(vol/10)*10 + 10)
            if(vol>100):
                vol = 100
            print " -> Volume To:" + str(vol)
            resp = self.call("volume?volume_percent=" + str(vol),"PUT" )
            self.oh.sendCommand('spotify_current_volume',vol)
            if (self.debug): print resp
        except:
            print " -> VolumeUp Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def volumeDown(self):
        """
        Volume down by 10%
        """
        print "-- Calling Service: Volume Down"
        try:
            vol = int(self.oh.getState('spotify_current_volume'))
            vol = int(round(vol/10)*10 - 10)
            if(vol<0):
                vol = 0
            print "Volume To:" + str(vol)
            resp = self.call("volume?volume_percent=" + str(vol),"PUT" )
            self.oh.sendCommand('spotify_current_volume',vol)
            if (self.debug): print resp
        except:
            print " -> VolumeDown Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def pause(self):
        """
        Pause player
        """
        print "-- Calling Service: Pause"
        try:
            resp = self.call("pause","PUT")
            self.oh.sendCommand('spotify_current_playing',"OFF")
            if (self.debug): print resp
        except:
            print " -> Pause Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def play(self, context_uri = None):
        """
        Resume player
        """
        print "-- Calling Service: Play"

        if (context_uri is None):
            payload = {}
        else:
            payload = json.dumps({ 'context_uri': context_uri })
        print payload

        try:
            resp = self.call("play","PUT", payload = payload)
            if (self.debug): print resp
            self.update()
        except:
            print " -> Play Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def previous(self):
        """
        Skip to previous track
        """
        print "-- Calling Service: Previous"
        try:
            resp = self.call("previous","POST")
            if (self.debug): print resp
            self.update()
        except:
            print " -> Previous Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def next(self):
        """
        Skip to next track
        """
        print "-- Calling Service: Next"
        try:
            resp = self.call("next","POST")
            if (self.debug): print resp
            self.update()
        except:
            print " -> Next Failure: ", sys.exc_info()[0]
            resp = ""

        return resp

    def updateConnectionDateTime(self):
        self.oh.sendCommand('spotify_lastConnectionDateTime',time.strftime("%Y-%m-%dT%H:%M:%S+0000",time.gmtime(time.time())))

def main():

    t1 = time.time()

    c = spotify()

    args = sys.argv

    if(len(args) == 1):
        c.update()
    else:

        if(args[1] == "volume_up"):
            c.volumeUp()
        if(args[1] == "volume_down"):
            c.volumeDown()
        if(args[1] == "play"):
            if(len(args)>2):
                a = ""
                for x in range(2, len(args)):
                    a = a + args[x] + " "
                c.play(a.strip())
            else:
                c.play()
        if(args[1] == "pause"):
            c.pause()
        if(args[1] == "previous"):
            c.previous()
        if(args[1] == "next"):
            c.next()

    c.updateConnectionDateTime()

    t2 = time.time()
    print "Done in " + str(t2-t1) + " seconds"

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import json
import datetime
import sys

import tweepy
from .filter import StreamFilter
from config import debug
import re
import os
from flask import Markup
from pymongo import MongoClient

CONSUMER_KEY = os.environ['CONSUMER_KEY']
CONSUMER_SECRET = os.environ['CONSUMER_SECRET']
ACCESS_TOKEN = os.environ['ACCESS_TOKEN']
ACCESS_TOKEN_SECRET = os.environ['ACCESS_TOKEN_SECRET']

users = StreamFilter.users
track = StreamFilter.track
debug_users = StreamFilter.debug_users

# Start MongoDB
conn = MongoClient(os.environ['MONGO_URI'])
db = conn[os.environ['MONGODB_DATABASE']]
collection = db.tweets

# override tweepy.StreamListener to add logic to on_status
class MyStreamListener(tweepy.StreamListener):

    def on_connect(self):
        print("Connected to Twitter")

    def on_data(self, raw_data):
        super(MyStreamListener, self).on_data(raw_data)
        json_data = json.loads(raw_data)
        if 'id_str' in json_data:
            json_data['_id'] = int(json_data['id_str'])
        else:
            print('on_data fired with no id_str')
        # Store tweet in db as JSON
        collection.insert(json_data)

    def on_status(self, status):

        if debug:
            # Assign var for pertinent data from status
            screen_name = status.user.screen_name
            user_name = status.user.name
            user_id = status.user.id_str
            status_id = status.id_str
            status_text = status.text
            tweet_time = status.created_at
            tweet_link = "http://twitter.com/{0}/status/{1}".format(user_id,
                                                                    status_id)

            mentions = [status.entities['user_mentions']['screen_name']
                        for status.entities['user_mentions']
                        in status.entities['user_mentions']]

            print("@{0} mentioned {1} - {2}".format(screen_name,
                                                    mentions,
                                                    tweet_link))
            print("\t{0}".format(status_text))

    def on_error(self, status_code):
        print("Error {0} encountered".format(status_code))
        if status_code == 420:
            # returning False in on_data disconnects the stream
            print('Encountered status code 420. Quitting')
            return False

    def on_timeout(self):
        print("Timeout occurred at {}".format(datetime.datetime.now()))
        return

    def on_disconnect(self, notice):
        print("Disconnected at {}".format(datetime.datetime.now()))
        print("Twitter sent the following disconnect message: \n{}"
              .format(notice))
        return

def start_stream():

    auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)

    api = tweepy.API(auth)

    myStreamListener = MyStreamListener()

    # encoding() removed from if follow and if track in Stream.filter for Py3
    myStream = tweepy.Stream(auth=api.auth,
                             listener=myStreamListener,
                             retry_count=5)

    # Lookup users whether given an ID (most accurate) or a screen name
    user_ids = []
    screen_names = []

    for item in users:
        for host in item['hosts']:
            if host['user_id']:
                user_ids.append(host['user_id'])
            elif host['screen_name']:
                screen_names.append(host['screen_name'])
            else:
                # TODO report error
                if debug:
                    print('User {} not found'.format(host))

    if debug:
        screen_names.extend(debug_users)

    user_data = api.lookup_users(screen_names=screen_names, user_ids=user_ids)

    if debug:
        for user in user_data:
            print("Following: {0} - @{1} ({2})".format(user.name,
                                                       user.screen_name,
                                                       user.id_str))

    follow_ids = [user.id_str for user in user_data]
    myStream.filter(follow=follow_ids, async=True)

# TODO user_ids should be submitted to view from form
# Return tweets from database
def get_tweets(limit=0,
               user_ids=users,
               mentions=True,
               cursor=None,
               age='old',
               sort='_id',
               order=-1):
    """
    :param limit: maximum number of tweets to return
    :type limit: int
    :param user_ids: list of user IDs to track
    :type user_ids: list
    :param mentions: whether to include mentions of users in user_ids
    :type mentions: bool
    :param cursor: reference tweet for results, default=None
    :type cursor: int
    :param age: direction from cursor, old or new, default='old'
    :type age: str
    :param sort: sort field, default='_id'
    :type sort: str
    :param order: order of returned tweets, default=-1 (descending)
    :type order: int
    :return: list of tweets returned from query
    :rtype: list
    """

    user_query = []
    # TODO remove this for statement once user_ids are passed from site
    for item in user_ids:
        for host in item['hosts']:
            if host['user_id']:
                user_query.append(host['user_id'])

    tweets = []
    tweet_query = {'text': {'$exists': True}}
    if cursor:
        if age is 'old':
            direction = '$lt'
        elif age is 'new':
            direction = '$gt'
        tweet_query['_id'] = {direction: cursor}
        if debug:
            try:
                cursor_tweet = collection.find_one({'_id': cursor})
                print('Cursor tweet {0} created at {1}'
                      .format(cursor,
                              cursor_tweet['created_at']))
            except TypeError as e:
                print('**Error encountered finding the cursor. {} not found'
                      .format(cursor))
                print('**TypeError: {}'.format(e))
            except:
                e = sys.exc_info()[0]
                print('**Other Error: {}'.format(e))

    if mentions:
        tweet_query['$or'] = [{'user.id_str': {'$in': user_query}},
                              {'in_reply_to_user_id_str': {'$in': user_query}}]
    else:
        tweet_query['user.id_str'] = {'$in': user_query}

    if debug:
        print(tweet_query)

    for tweet in collection.find(tweet_query)\
            .sort(sort, order).limit(limit):
        if 'retweeted_status' in tweet:
            tweet['retweeted_status']['text'] = \
                add_entities(tweet['retweeted_status']['text'],
                             tweet['retweeted_status']['entities'])
        tweet['text'] = add_entities(tweet['text'], tweet['entities'])
        tweets.append(tweet)
    return tweets

# add links to entities in tweet
def add_entities(tweet_text, tweet_entities):
    # Per Twitter API, tolerant of possible empty/null values
    if 'urls' in tweet_entities:
        for url in tweet_entities['urls']:
            tweet_text = re.sub(
                url['url'],
                '{2}'
                .format(url['url'],
                        url['expanded_url'],
                        url['display_url']),
                tweet_text, flags=re.IGNORECASE)
    if 'user_mentions' in tweet_entities:
        for user in tweet_entities['user_mentions']:
            tweet_text = re.sub(
                '@{0}'.format(user['screen_name']),
                '@{0}'.format(user['screen_name'],
                                                  user['screen_name']),
                tweet_text, flags=re.IGNORECASE)
    if 'media' in tweet_entities:
        for media in tweet_entities['media']:
            tweet_text = tweet_text.replace(media['url'], '')
    if 'hashtags' in tweet_entities:
        for hashtag in tweet_entities['hashtags']:
            tweet_text = re.sub(
                '#{0}'.format(hashtag['text']),
                '#{0}'
                .format(hashtag['text']),
                tweet_text, flags=re.IGNORECASE)
    tweet_text = Markup(tweet_text)
    return tweet_text

if __name__ == '__main__':
    start_stream()<commit_msg>TODO backoff strategy<commit_after>import json
import datetime
import sys

import tweepy
from .filter import StreamFilter
from config import debug
import re
import os
from flask import Markup
from pymongo import MongoClient

CONSUMER_KEY = os.environ['CONSUMER_KEY']
CONSUMER_SECRET = os.environ['CONSUMER_SECRET']
ACCESS_TOKEN = os.environ['ACCESS_TOKEN']
ACCESS_TOKEN_SECRET = os.environ['ACCESS_TOKEN_SECRET']

users = StreamFilter.users
track = StreamFilter.track
debug_users = StreamFilter.debug_users

# Start MongoDB
conn = MongoClient(os.environ['MONGO_URI'])
db = conn[os.environ['MONGODB_DATABASE']]
collection = db.tweets

# override tweepy.StreamListener to add logic to on_status
class MyStreamListener(tweepy.StreamListener):

    def on_connect(self):
        print("Connected to Twitter")

    def on_data(self, raw_data):
        super(MyStreamListener, self).on_data(raw_data)
        json_data = json.loads(raw_data)
        if 'id_str' in json_data:
            json_data['_id'] = int(json_data['id_str'])
        else:
            print('on_data fired with no id_str')
        # Store tweet in db as JSON
        collection.insert(json_data)

    def on_status(self, status):

        if debug:
            # Assign var for pertinent data from status
            screen_name = status.user.screen_name
            user_name = status.user.name
            user_id = status.user.id_str
            status_id = status.id_str
            status_text = status.text
            tweet_time = status.created_at
            tweet_link = "http://twitter.com/{0}/status/{1}".format(user_id,
                                                                    status_id)

            mentions = [status.entities['user_mentions']['screen_name']
                        for status.entities['user_mentions']
                        in status.entities['user_mentions']]

            print("@{0} mentioned {1} - {2}".format(screen_name,
                                                    mentions,
                                                    tweet_link))
            print("\t{0}".format(status_text))

    def on_error(self, status_code):
        # TODO build exponential backoff strategy for rate limiting
        print("Error {0} encountered".format(status_code))
        if status_code == 420:
            # returning False in on_data disconnects the stream
            print('Encountered status code 420. Quitting')
            return False

    def on_timeout(self):
        print("Timeout occurred at {}".format(datetime.datetime.now()))
        return

    def on_disconnect(self, notice):
        print("Disconnected at {}".format(datetime.datetime.now()))
        print("Twitter sent the following disconnect message: \n{}"
              .format(notice))
        return

def start_stream():

    auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)

    api = tweepy.API(auth)

    myStreamListener = MyStreamListener()

    # encoding() removed from if follow and if track in Stream.filter for Py3
    myStream = tweepy.Stream(auth=api.auth,
                             listener=myStreamListener,
                             retry_count=5)

    # Lookup users whether given an ID (most accurate) or a screen name
    user_ids = []
    screen_names = []

    for item in users:
        for host in item['hosts']:
            if host['user_id']:
                user_ids.append(host['user_id'])
            elif host['screen_name']:
                screen_names.append(host['screen_name'])
            else:
                # TODO report error
                if debug:
                    print('User {} not found'.format(host))

    if debug:
        screen_names.extend(debug_users)

    user_data = api.lookup_users(screen_names=screen_names, user_ids=user_ids)

    if debug:
        for user in user_data:
            print("Following: {0} - @{1} ({2})".format(user.name,
                                                       user.screen_name,
                                                       user.id_str))

    follow_ids = [user.id_str for user in user_data]
    myStream.filter(follow=follow_ids, async=True)

# TODO user_ids should be submitted to view from form
# Return tweets from database
def get_tweets(limit=0,
               user_ids=users,
               mentions=True,
               cursor=None,
               age='old',
               sort='_id',
               order=-1):
    """
    :param limit: maximum number of tweets to return
    :type limit: int
    :param user_ids: list of user IDs to track
    :type user_ids: list
    :param mentions: whether to include mentions of users in user_ids
    :type mentions: bool
    :param cursor: reference tweet for results, default=None
    :type cursor: int
    :param age: direction from cursor, old or new, default='old'
    :type age: str
    :param sort: sort field, default='_id'
    :type sort: str
    :param order: order of returned tweets, default=-1 (descending)
    :type order: int
    :return: list of tweets returned from query
    :rtype: list
    """

    user_query = []
    # TODO remove this for statement once user_ids are passed from site
    for item in user_ids:
        for host in item['hosts']:
            if host['user_id']:
                user_query.append(host['user_id'])

    tweets = []
    tweet_query = {'text': {'$exists': True}}
    if cursor:
        if age is 'old':
            direction = '$lt'
        elif age is 'new':
            direction = '$gt'
        tweet_query['_id'] = {direction: cursor}
        if debug:
            try:
                cursor_tweet = collection.find_one({'_id': cursor})
                print('Cursor tweet {0} created at {1}'
                      .format(cursor,
                              cursor_tweet['created_at']))
            except TypeError as e:
                print('**Error encountered finding the cursor. {} not found'
                      .format(cursor))
                print('**TypeError: {}'.format(e))
            except:
                e = sys.exc_info()[0]
                print('**Other Error: {}'.format(e))

    if mentions:
        tweet_query['$or'] = [{'user.id_str': {'$in': user_query}},
                              {'in_reply_to_user_id_str': {'$in': user_query}}]
    else:
        tweet_query['user.id_str'] = {'$in': user_query}

    if debug:
        print(tweet_query)

    for tweet in collection.find(tweet_query)\
            .sort(sort, order).limit(limit):
        if 'retweeted_status' in tweet:
            tweet['retweeted_status']['text'] = \
                add_entities(tweet['retweeted_status']['text'],
                             tweet['retweeted_status']['entities'])
        tweet['text'] = add_entities(tweet['text'], tweet['entities'])
        tweets.append(tweet)
    return tweets

# add links to entities in tweet
def add_entities(tweet_text, tweet_entities):
    # Per Twitter API, tolerant of possible empty/null values
    if 'urls' in tweet_entities:
        for url in tweet_entities['urls']:
            tweet_text = re.sub(
                url['url'],
                '{2}'
                .format(url['url'],
                        url['expanded_url'],
                        url['display_url']),
                tweet_text, flags=re.IGNORECASE)
    if 'user_mentions' in tweet_entities:
        for user in tweet_entities['user_mentions']:
            tweet_text = re.sub(
                '@{0}'.format(user['screen_name']),
                '@{0}'.format(user['screen_name'],
                                                  user['screen_name']),
                tweet_text, flags=re.IGNORECASE)
    if 'media' in tweet_entities:
        for media in tweet_entities['media']:
            tweet_text = tweet_text.replace(media['url'], '')
    if 'hashtags' in tweet_entities:
        for hashtag in tweet_entities['hashtags']:
            tweet_text = re.sub(
                '#{0}'.format(hashtag['text']),
                '#{0}'
                .format(hashtag['text']),
                tweet_text, flags=re.IGNORECASE)
    tweet_text = Markup(tweet_text)
    return tweet_text

if __name__ == '__main__':
    start_stream()<|endoftext|>


<commit_before>import re
import os

__author__ = 'Thierry Schellenbach'
__copyright__ = 'Copyright 2014, Thierry Schellenbach'
__credits__ = ['Thierry Schellenbach, mellowmorning.com, @tschellenbach']
__license__ = 'BSD'
__version__ = '2.3.8'
__maintainer__ = 'Thierry Schellenbach'
__email__ = 'thierry@getstream.io'
__status__ = 'Production'

def connect(api_key=None, api_secret=None, app_id=None, version='v1.0', timeout=3.0, location=None):
    '''
    Returns a Client object

    :param api_key: your api key or heroku url
    :param api_secret: the api secret
    :param app_id: the app id (used for listening to feed changes)
    '''
    from stream.client import StreamClient
    stream_url = os.environ.get('STREAM_URL')
    # support for the heroku STREAM_URL syntax
    if stream_url and not api_key:
        pattern = re.compile(
            'https\:\/\/(\w+)\:(\w+)\@([\w-]*).*\?app_id=(\d+)', re.IGNORECASE)
        result = pattern.match(stream_url)
        if result and len(result.groups()) == 4:
            api_key, api_secret, location, app_id = result.groups()
            location = None if location == 'getstream' else location
        else:
            raise ValueError('Invalid api key or heroku url')

    return StreamClient(api_key, api_secret, app_id, version, timeout, location=location)
<commit_msg>Change email to support email<commit_after>import re
import os

__author__ = 'Thierry Schellenbach'
__copyright__ = 'Copyright 2014, Thierry Schellenbach'
__credits__ = ['Thierry Schellenbach, mellowmorning.com, @tschellenbach']
__license__ = 'BSD'
__version__ = '2.3.8'
__maintainer__ = 'Thierry Schellenbach'
__email__ = 'support@getstream.io'
__status__ = 'Production'

def connect(api_key=None, api_secret=None, app_id=None, version='v1.0', timeout=3.0, location=None):
    '''
    Returns a Client object

    :param api_key: your api key or heroku url
    :param api_secret: the api secret
    :param app_id: the app id (used for listening to feed changes)
    '''
    from stream.client import StreamClient
    stream_url = os.environ.get('STREAM_URL')
    # support for the heroku STREAM_URL syntax
    if stream_url and not api_key:
        pattern = re.compile(
            'https\:\/\/(\w+)\:(\w+)\@([\w-]*).*\?app_id=(\d+)', re.IGNORECASE)
        result = pattern.match(stream_url)
        if result and len(result.groups()) == 4:
            api_key, api_secret, location, app_id = result.groups()
            location = None if location == 'getstream' else location
        else:
            raise ValueError('Invalid api key or heroku url')

    return StreamClient(api_key, api_secret, app_id, version, timeout, location=location)
<|endoftext|>


<commit_before>from .response import JSONResponse
from .view import View

class Resource(View):
    """
    Base class for HTTP Resources (RESTful?)
    """

    # TODO: Optimize for alphanumeric as base pattern
    id_pattern = '{id}'

    # Allow nested resources
    nested_collection_resources = None
    nested_entity_resources = None

    def __init__(self, *args, **kwargs):
        self._parent = kwargs.pop('parent') if 'parent' in kwargs else None
        super(Resource, self).__init__(*args, **kwargs)

    def response(self, *args, **kwargs):
        if self.is_entity():
            return EntityResponse(self, *args, **kwargs)
        else:
            return CollectionResponse(self, *args, **kwargs)

    def get_resource_name(self):
        return self.resource_name \
            if hasattr(self, 'resource_name') \
            else self.__class__.__name__.lower()

    def default_kwargs_for_urls(self):
        """
        Default keyword arguments for building the resource's urls.

        :return: dict
        """
        return {}

    def get_index_url(self, resource=None, **kwargs):
        """
        Builds the url of the resource's index.

        :param resource: name of the resource or None
        :param kwargs: additional keyword arguments to build the url
        :return: url of the resource's index
        """
        default_kwargs = self.default_kwargs_for_urls() \
            if resource == self.get_resource_name() else {}

        default_kwargs.update(kwargs)

        return self.get_full_url(
            self.app.reverse(
                '{}_index'.format(resource or self.get_resource_name()),
                **default_kwargs
            )
        )

    def get_definition_url(self, resource=None, **kwargs):
        default_kwargs = self.default_kwargs_for_urls() \
            if resource is None else {}
        default_kwargs.update(kwargs)

        return self.get_full_url(
            self.app.reverse(
                '{}_definition'.format(resource or self.get_resource_name()),
                **default_kwargs
            )
        )

    def get_object_url(self, id, resource=None, **kwargs):
        default_kwargs = self.default_kwargs_for_urls() \
            if resource is None else {}
        default_kwargs.update(kwargs)

        return self.get_full_url(
            self.app.reverse(
                '{}_item'.format(resource or self.get_resource_name()),
                id=id,
                **default_kwargs
            )
        )

    def get_self_url(self):
        resource = self.get_resource_name()
        return self.get_index_url(resource) if self.is_collection() \
            else self.get_object_url(
                self.request.match_info.get('id', None), resource
            )

    def get_encoder_class(self):
        return getattr(self, 'encoder_class', None)

    def is_entity(self):
        """
        Hacky solution to detect if current instance of the class is an entity
        or collection endpoint.
        """
        return 'id' in self.request.match_info

    def is_collection(self):
        return not self.is_entity()

    def has_nested_resources(self):
        if self.is_collection():
            return True if self.nested_collection_resources else False
        else:
            return True if self.nested_entity_resources else False

    def get_nested_resources(self):
        if self.is_collection():
            return self.nested_collection_resources
        else:
            return self.nested_entity_resources

    def get_nested_urls(self):

        current_args = self.request.match_info.copy()
        if 'id' in current_args:
            id = current_args.pop('id')
            current_args['{}_id'.format(self.get_resource_name())] = id

        return {
            k: self.get_index_url(c.__name__.lower(), **current_args)
            for k, c in self.get_nested_resources().items()
        }

    def get_parent(self):
        """Returns the url to the parent endpoint."""
        if self.is_entity():
            return self.get_index_url()
        elif self._parent is not None:
            resource = self._parent.rsplit('_', 1)[0]
            parts = self.default_kwargs_for_urls()

            if '{}_id'.format(resource) in parts:
                id = parts.pop('{}_id'.format(resource))
                parts['id'] = id
            return self.get_full_url(
                self.app.reverse(
                    self._parent, **parts
                )
            )

class ResourceResponse(JSONResponse):

    def __init__(self, resource, kind=None, extra_content=None, **kwargs):
        data = dict(
            element='resource:{}'.format(kind),
            self=resource.get_self_url()
        )

        if resource.has_nested_resources():
            data['resources'] = resource.get_nested_urls()

        if extra_content is not None:
            data.update(extra_content)

        if hasattr(resource, 'definition'):
            data['definition'] = resource.get_definition_url()

        data['parent'] = resource.get_parent() or None

        super(ResourceResponse, self).__init__(
            data, cls=resource.get_encoder_class(), **kwargs
        )

class CollectionResponse(ResourceResponse):

    def __init__(self, resource, extra_content=None, **kwargs):

        super(CollectionResponse, self).__init__(
            resource, 'collection', extra_content=extra_content, **kwargs)

class EntityResponse(ResourceResponse):

    def __init__(self, resource, body=None, extra_content=None, **kwargs):

        if extra_content is None:
            extra_content = {}

        extra_content['body'] = body

        super(EntityResponse, self).__init__(
            resource, 'entity', extra_content=extra_content, **kwargs)
<commit_msg>fix how nested urls are reversed to make it work for nested Resource and View subclasses<commit_after>from .response import JSONResponse
from .view import View

class Resource(View):
    """
    Base class for HTTP Resources (RESTful?)
    """

    # TODO: Optimize for alphanumeric as base pattern
    id_pattern = '{id}'

    # Allow nested resources
    nested_collection_resources = None
    nested_entity_resources = None

    def __init__(self, *args, **kwargs):
        self._parent = kwargs.pop('parent') if 'parent' in kwargs else None
        super(Resource, self).__init__(*args, **kwargs)

    def response(self, *args, **kwargs):
        if self.is_entity():
            return EntityResponse(self, *args, **kwargs)
        else:
            return CollectionResponse(self, *args, **kwargs)

    def get_resource_name(self):
        return self.resource_name \
            if hasattr(self, 'resource_name') \
            else self.__class__.__name__.lower()

    def default_kwargs_for_urls(self):
        """
        Default keyword arguments for building the resource's urls.

        :return: dict
        """
        return {}

    def get_index_url(self, resource=None, **kwargs):
        """
        Builds the url of the resource's index.

        :param resource: name of the resource or None
        :param kwargs: additional keyword arguments to build the url
        :return: url of the resource's index
        """
        default_kwargs = self.default_kwargs_for_urls() \
            if resource == self.get_resource_name() else {}

        default_kwargs.update(kwargs)

        return self.get_full_url(
            self.app.reverse(
                '{}_index'.format(resource or self.get_resource_name()),
                **default_kwargs
            )
        )

    def get_definition_url(self, resource=None, **kwargs):
        default_kwargs = self.default_kwargs_for_urls() \
            if resource is None else {}
        default_kwargs.update(kwargs)

        return self.get_full_url(
            self.app.reverse(
                '{}_definition'.format(resource or self.get_resource_name()),
                **default_kwargs
            )
        )

    def get_object_url(self, id, resource=None, **kwargs):
        default_kwargs = self.default_kwargs_for_urls() \
            if resource is None else {}
        default_kwargs.update(kwargs)

        return self.get_full_url(
            self.app.reverse(
                '{}_item'.format(resource or self.get_resource_name()),
                id=id,
                **default_kwargs
            )
        )

    def get_self_url(self):
        resource = self.get_resource_name()
        return self.get_index_url(resource) if self.is_collection() \
            else self.get_object_url(
                self.request.match_info.get('id', None), resource
            )

    def get_encoder_class(self):
        return getattr(self, 'encoder_class', None)

    def is_entity(self):
        """
        Hacky solution to detect if current instance of the class is an entity
        or collection endpoint.
        """
        return 'id' in self.request.match_info

    def is_collection(self):
        return not self.is_entity()

    def has_nested_resources(self):
        if self.is_collection():
            return True if self.nested_collection_resources else False
        else:
            return True if self.nested_entity_resources else False

    def get_nested_resources(self):
        if self.is_collection():
            return self.nested_collection_resources
        else:
            return self.nested_entity_resources

    def get_nested_urls(self):

        current_args = self.request.match_info.copy()
        if 'id' in current_args:
            id = current_args.pop('id')
            current_args['{}_id'.format(self.get_resource_name())] = id

        def _nested_url(k, c):
            if isinstance(c, Resource):
                return self.get_index_url(c.__name__.lower(), **current_args)

            return self.get_full_url(
                self.app.reverse(
                    '{}'.format(c.__name__.lower()),
                    **current_args
                )
            )

        return {
            k: _nested_url(k, c)
            for k, c in self.get_nested_resources().items()
        }

    def get_parent(self):
        """Returns the url to the parent endpoint."""
        if self.is_entity():
            return self.get_index_url()
        elif self._parent is not None:
            resource = self._parent.rsplit('_', 1)[0]
            parts = self.default_kwargs_for_urls()

            if '{}_id'.format(resource) in parts:
                id = parts.pop('{}_id'.format(resource))
                parts['id'] = id
            return self.get_full_url(
                self.app.reverse(
                    self._parent, **parts
                )
            )

class ResourceResponse(JSONResponse):

    def __init__(self, resource, kind=None, extra_content=None, **kwargs):
        data = dict(
            element='resource:{}'.format(kind),
            self=resource.get_self_url()
        )

        if resource.has_nested_resources():
            data['resources'] = resource.get_nested_urls()

        if extra_content is not None:
            data.update(extra_content)

        if hasattr(resource, 'definition'):
            data['definition'] = resource.get_definition_url()

        data['parent'] = resource.get_parent() or None

        super(ResourceResponse, self).__init__(
            data, cls=resource.get_encoder_class(), **kwargs
        )

class CollectionResponse(ResourceResponse):

    def __init__(self, resource, extra_content=None, **kwargs):

        super(CollectionResponse, self).__init__(
            resource, 'collection', extra_content=extra_content, **kwargs)

class EntityResponse(ResourceResponse):

    def __init__(self, resource, body=None, extra_content=None, **kwargs):

        if extra_content is None:
            extra_content = {}

        extra_content['body'] = body

        super(EntityResponse, self).__init__(
            resource, 'entity', extra_content=extra_content, **kwargs)
<|endoftext|>


<commit_before># coding: utf8
from __future__ import unicode_literals, print_function

import pytest
import time
import os
from wasabi.printer import Printer
from wasabi.util import MESSAGES, supports_ansi

def test_printer():
    p = Printer(no_print=True)
    text = "This is a test."
    if supports_ansi():
        assert p.good(text) in (
            "\x1b[38;5;2m\u2714 This is a test.\x1b[0m",
            "\x1b[38;5;2m This is a test.\x1b[0m",
        )
        assert p.fail(text) in (
            "\x1b[38;5;1m\u2718 This is a test.\x1b[0m",
            "\x1b[38;5;1m This is a test.\x1b[0m",
        )
        assert p.warn(text) in (
            "\x1b[38;5;3m\u26a0 This is a test.\x1b[0m",
            "\x1b[38;5;3m This is a test.\x1b[0m",
        )
        assert p.info(text) in (
            "\x1b[38;5;4m\u2139 This is a test.\x1b[0m",
            "\x1b[38;5;4m This is a test.\x1b[0m",
        )
        assert p.text(text) == text
    else:
        assert p.good(text) in (text, "\u2714 This is a test.")
        assert p.fail(text) in (text, "\u2718 This is a test.")
        assert p.warn(text) in (text, "\u26a0 This is a test.")
        assert p.info(text) in (text, "\u2139 This is a test.")
        assert p.text(text) == text

def test_printer_no_pretty():
    p = Printer(no_print=True, pretty=False)
    text = "This is a test."
    assert p.good(text) == text
    assert p.fail(text) == text
    assert p.warn(text) == text
    assert p.info(text) == text
    assert p.text(text) == text

def test_printer_custom():
    colors = {"yellow": 220, "purple": 99}
    icons = {"warn": "\u26a0\ufe0f", "question": "?"}
    p = Printer(no_print=True, colors=colors, icons=icons)
    text = "This is a test."
    purple_question = p.text(text, color="purple", icon="question")
    warning = p.warn(text)
    assert purple_question in (
        "\x1b[38;5;99m? This is a test.\x1b[0m",
        "{} {}".format(icons["question"], text),
    )
    assert warning in ("\x1b[38;5;3m\u26a0\ufe0f This is a test.\x1b[0m", text)

def test_printer_counts():
    p = Printer()
    text = "This is a test."
    for i in range(2):
        p.good(text)
    for i in range(1):
        p.fail(text)
    for i in range(4):
        p.warn(text)
    assert p.counts[MESSAGES.GOOD] == 2
    assert p.counts[MESSAGES.FAIL] == 1
    assert p.counts[MESSAGES.WARN] == 4

def test_printer_divider():
    p = Printer(line_max=20, no_print=True)
    p.divider() == "\x1b[1m\n================\x1b[0m"
    p.divider("test") == "\x1b[1m\n====== test ======\x1b[0m"
    p.divider("test", char="*") == "\x1b[1m\n****** test ******\x1b[0m"
    assert (
        p.divider("This is a very long text, it is very long")
        == "\x1b[1m\n This is a very long text, it is very long \x1b[0m"
    )
    with pytest.raises(ValueError):
        p.divider("test", char="~.")

@pytest.mark.parametrize("hide_animation", [False, True])
def test_printer_loading(hide_animation):
    p = Printer(hide_animation=hide_animation)
    print("\n")
    with p.loading("Loading..."):
        time.sleep(1)
    p.good("Success!")

    with p.loading("Something else..."):
        time.sleep(2)
    p.good("Yo!")

    with p.loading("Loading..."):
        time.sleep(1)
    p.good("Success!")

def test_printer_loading_raises_exception():
    def loading_with_exception():
        p = Printer()
        print("\n")
        with p.loading():
            raise Exception("This is an error.")

    with pytest.raises(Exception):
        loading_with_exception()

def test_printer_log_friendly():
    text = "This is a test."
    ENV_LOG_FRIENDLY = "WASABI_LOG_FRIENDLY"
    os.environ[ENV_LOG_FRIENDLY] = "True"
    p = Printer(no_print=True)
    assert p.good(text) in ("\u2714 This is a test.", text)
    del os.environ[ENV_LOG_FRIENDLY]

def test_printer_log_friendly_prefix():
    text = "This is a test."
    ENV_LOG_FRIENDLY = "CUSTOM_LOG_FRIENDLY"
    os.environ[ENV_LOG_FRIENDLY] = "True"
    p = Printer(no_print=True, env_prefix="CUSTOM")
    assert p.good(text) in (text, "\u2714 This is a test.")
    print(p.good(text))
    del os.environ[ENV_LOG_FRIENDLY]

def test_printer_none_encoding(monkeypatch):
    """Test that printer works even if sys.stdout.encoding is set to None. This
    previously caused a very confusing error."""
    monkeypatch.setattr("sys.stdout.encoding", None)
    p = Printer()
<commit_msg>Update test_printer.py<commit_after># coding: utf8
from __future__ import unicode_literals, print_function

import pytest
import time
import os
from wasabi.printer import Printer
from wasabi.util import MESSAGES, supports_ansi

def test_printer():
    p = Printer(no_print=True)
    text = "This is a test."
    if supports_ansi():
        assert p.good(text) in (
            "\x1b[38;5;2m\u2714 This is a test.\x1b[0m",
            "\x1b[38;5;2mThis is a test.\x1b[0m",
        )
        assert p.fail(text) in (
            "\x1b[38;5;1m\u2718 This is a test.\x1b[0m",
            "\x1b[38;5;1mThis is a test.\x1b[0m",
        )
        assert p.warn(text) in (
            "\x1b[38;5;3m\u26a0 This is a test.\x1b[0m",
            "\x1b[38;5;3mThis is a test.\x1b[0m",
        )
        assert p.info(text) in (
            "\x1b[38;5;4m\u2139 This is a test.\x1b[0m",
            "\x1b[38;5;4mThis is a test.\x1b[0m",
        )
        assert p.text(text) == text
    else:
        assert p.good(text) in (text, "\u2714 This is a test.")
        assert p.fail(text) in (text, "\u2718 This is a test.")
        assert p.warn(text) in (text, "\u26a0 This is a test.")
        assert p.info(text) in (text, "\u2139 This is a test.")
        assert p.text(text) == text

def test_printer_no_pretty():
    p = Printer(no_print=True, pretty=False)
    text = "This is a test."
    assert p.good(text) == text
    assert p.fail(text) == text
    assert p.warn(text) == text
    assert p.info(text) == text
    assert p.text(text) == text

def test_printer_custom():
    colors = {"yellow": 220, "purple": 99}
    icons = {"warn": "\u26a0\ufe0f", "question": "?"}
    p = Printer(no_print=True, colors=colors, icons=icons)
    text = "This is a test."
    purple_question = p.text(text, color="purple", icon="question")
    warning = p.warn(text)
    assert purple_question in (
        "\x1b[38;5;99m? This is a test.\x1b[0m",
        "{} {}".format(icons["question"], text),
    )
    assert warning in ("\x1b[38;5;3m\u26a0\ufe0f This is a test.\x1b[0m", text)

def test_printer_counts():
    p = Printer()
    text = "This is a test."
    for i in range(2):
        p.good(text)
    for i in range(1):
        p.fail(text)
    for i in range(4):
        p.warn(text)
    assert p.counts[MESSAGES.GOOD] == 2
    assert p.counts[MESSAGES.FAIL] == 1
    assert p.counts[MESSAGES.WARN] == 4

def test_printer_divider():
    p = Printer(line_max=20, no_print=True)
    p.divider() == "\x1b[1m\n================\x1b[0m"
    p.divider("test") == "\x1b[1m\n====== test ======\x1b[0m"
    p.divider("test", char="*") == "\x1b[1m\n****** test ******\x1b[0m"
    assert (
        p.divider("This is a very long text, it is very long")
        == "\x1b[1m\n This is a very long text, it is very long \x1b[0m"
    )
    with pytest.raises(ValueError):
        p.divider("test", char="~.")

@pytest.mark.parametrize("hide_animation", [False, True])
def test_printer_loading(hide_animation):
    p = Printer(hide_animation=hide_animation)
    print("\n")
    with p.loading("Loading..."):
        time.sleep(1)
    p.good("Success!")

    with p.loading("Something else..."):
        time.sleep(2)
    p.good("Yo!")

    with p.loading("Loading..."):
        time.sleep(1)
    p.good("Success!")

def test_printer_loading_raises_exception():
    def loading_with_exception():
        p = Printer()
        print("\n")
        with p.loading():
            raise Exception("This is an error.")

    with pytest.raises(Exception):
        loading_with_exception()

def test_printer_log_friendly():
    text = "This is a test."
    ENV_LOG_FRIENDLY = "WASABI_LOG_FRIENDLY"
    os.environ[ENV_LOG_FRIENDLY] = "True"
    p = Printer(no_print=True)
    assert p.good(text) in ("\u2714 This is a test.", text)
    del os.environ[ENV_LOG_FRIENDLY]

def test_printer_log_friendly_prefix():
    text = "This is a test."
    ENV_LOG_FRIENDLY = "CUSTOM_LOG_FRIENDLY"
    os.environ[ENV_LOG_FRIENDLY] = "True"
    p = Printer(no_print=True, env_prefix="CUSTOM")
    assert p.good(text) in (text, "\u2714 This is a test.")
    print(p.good(text))
    del os.environ[ENV_LOG_FRIENDLY]

def test_printer_none_encoding(monkeypatch):
    """Test that printer works even if sys.stdout.encoding is set to None. This
    previously caused a very confusing error."""
    monkeypatch.setattr("sys.stdout.encoding", None)
    p = Printer()
<|endoftext|>