<commit_before>#!/usr/bin/env python
from __future__ import division

__author__ = "Jai Ram Rideout"
__copyright__ = "Copyright 2012, The QIIME project"
__credits__ = ["Jai Ram Rideout", "Greg Caporaso"]
__license__ = "GPL"
__version__ = "1.5.0-dev"
__maintainer__ = "Jai Ram Rideout"
__email__ = "jai.rideout@gmail.com"
__status__ = "Development"

"""Contains functions used in the most_wanted_otus.py script."""

from collections import defaultdict
from operator import itemgetter
from os import makedirs
from os.path import basename, join, splitext
from tempfile import NamedTemporaryFile

from biom.parse import parse_biom_table

from cogent import DNA, LoadSeqs
from cogent.app.blast import blast_seqs, Blastall
from cogent.app.formatdb import build_blast_db_from_fasta_path
from cogent.parse.blast import BlastResult
from cogent.parse.fasta import MinimalFastaParser
from cogent.util.misc import remove_files

from qiime.parse import parse_mapping_file_to_dict
from qiime.util import (add_filename_suffix, parse_command_line_parameters,
        get_options_lookup, make_option, qiime_system_call)

def generate_most_wanted_list(otu_table_fp, rep_set_fp, gg_fp, nt_fp,
        mapping_fp, mapping_category, top_n, min_abundance, max_abundance,
        min_categories, max_gg_similarity, jobs_to_start, output_dir):
    try:
        makedirs(output_dir)
    except OSError:
        pass

    # First filter to keep only new (non-GG) OTUs.
    print "Filtering out all GG reference OTUs.\n"
    novel_otu_table_fp = join(output_dir, add_filename_suffix(otu_table_fp,
        '_novel'))
    stdout, stderr, ret_val = qiime_system_call(
        'filter_otus_from_otu_table.py -i %s -o %s -e %s' %
        (otu_table_fp, novel_otu_table_fp, gg_fp))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # Next filter to keep only abundant otus in the specified range (looking
    # only at extremely abundant OTUs has the problem of yielding too many
    # that are similar to stuff in the nt database).
    print "Filtering out all OTUs that do not fall within the specified " + \
          "abundance threshold.\n"
    novel_abund_otu_table_fp = join(output_dir, add_filename_suffix(novel_otu_table_fp,
            '_min%d_max%d' % (min_abundance, max_abundance)))
    stdout, stderr, ret_val = qiime_system_call(
        'filter_otus_from_otu_table.py -i %s -o %s -n %d -x %d' %
        (novel_otu_table_fp, novel_abund_otu_table_fp, min_abundance,
         max_abundance))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # Next, collapse by mapping_category.
    print "Collapsing OTU table by %s.\n" % mapping_category
    otu_table_by_samp_type_fp = join(output_dir, add_filename_suffix(novel_abund_otu_table_fp,
            '_%s' % mapping_category))
    stdout, stderr, ret_val = qiime_system_call(
        'summarize_otu_by_cat.py -c %s -o %s -m %s -i %s' %
        (novel_abund_otu_table_fp, otu_table_by_samp_type_fp, mapping_category,
         mapping_fp))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # Filter to contain only otus in the specified minimum number of sample
    # types.
    print "Filtering OTU table to include only OTUs that appear in at " + \
          "least %d sample groups.\n" % min_categories
    otu_table_by_samp_type_ms_fp = join(output_dir, add_filename_suffix(
            otu_table_by_samp_type_fp, '_ms%d' % min_categories))
    stdout, stderr, ret_val = qiime_system_call(
        'filter_otus_from_otu_table.py -i %s -o %s -s %d' %
        (otu_table_by_samp_type_fp, otu_table_by_samp_type_ms_fp,
         min_categories))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # Now that we have a filtered down OTU table of good candidate OTUs, filter
    # the corresponding representative set to include only these candidate
    # sequences.
    print "Filtering representative set to include only the latest " + \
          "candidate OTUs.\n"
    candidate_rep_set_fp = join(output_dir, add_filename_suffix(
            rep_set_fp, '_most_wanted_candidates'))
    stdout, stderr, ret_val = qiime_system_call(
        'filter_fasta.py -f %s -o %s -b %s' %
        (rep_set_fp, candidate_rep_set_fp, otu_table_by_samp_type_ms_fp))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # Filter to contain only otus that don't hit GG at a certain similarity
    # threshold.
    print "Running uclust to get list of sequences that don't hit the " + \
          "maximum GG similarity threshold.\n"
    uclust_output_dir = join(output_dir, 'most_wanted_candidates_%s_%s'
            % (basename(gg_fp), str(max_gg_similarity)))
    stdout, stderr, ret_val = qiime_system_call(
        'parallel_pick_otus_uclust_ref.py -i %s -o %s -r %s -s %s -O %d'
        % (candidate_rep_set_fp, uclust_output_dir, gg_fp,
           str(max_gg_similarity), jobs_to_start))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    print "Filtering candidate sequences to only include uclust failures.\n"
    cand_gg_dis_rep_set_fp = join(output_dir,
            add_filename_suffix(candidate_rep_set_fp,
                                '_failures'))
    stdout, stderr, ret_val = qiime_system_call(
        'filter_fasta.py -f %s -s %s -o %s' % (candidate_rep_set_fp,
        join(uclust_output_dir,
        splitext(basename(candidate_rep_set_fp))[0] + '_failures.txt'),
        cand_gg_dis_rep_set_fp))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # BLAST the failures against nt.
    print "BLASTing candidate sequences against nt database.\n"
    blast_output_dir = join(output_dir, 'blast_output')
    blast_cmd = 'parallel_blast.py -i %s -o %s -r %s -D -e 1e-3 -O %d' % (cand_gg_dis_rep_set_fp, blast_output_dir, nt_fp, jobs_to_start)
    print blast_cmd
    stdout, stderr, ret_val = qiime_system_call(blast_cmd)
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # Read in our filtered down candidate seqs file and latest filtered and
    # collapsed OTU table. We'll need to compute some stats on these to include
    # in our report.
    print "Collating results and formatting for output.\n"
    mw_seqs = {}
    for seq_id, seq in MinimalFastaParser(open(cand_gg_dis_rep_set_fp, 'U')):
        seq_id = seq_id.strip().split()[0]
        mw_seqs[seq_id] = seq
    otu_table_by_samp_type_ms = parse_biom_table(
            open(otu_table_by_samp_type_ms_fp, 'U'))

    # We'll sort the BLAST results by percent identity (ascending) and pick the
    # top n.
    blast_results = open(join(blast_output_dir,
        splitext(basename(cand_gg_dis_rep_set_fp))[0] + '_blast_out.txt'), 'U')
    top_n_mw = []
    for line in blast_results:
        # Skip headers.
        line = line.strip()
        if line and not line.startswith('#'):
            line = line.split('\t')
            top_n_mw.append((line[0], line[1], float(line[2])))
    top_n_mw = sorted(top_n_mw, key=itemgetter(2))[:top_n]

    # Write results out to tsv and HTML table.
    mw_tsv_f = open(join(output_dir,
                    'top_%d_most_wanted_otus.txt' % top_n), 'w')
    mw_html_f = open(join(output_dir,
                    'top_%d_most_wanted_otus.html' % top_n), 'w')
    tsv_header = 'OTU ID\tSequence\tGreengenes taxonomy\t' + \
                 'NCBI nt closest match\tNCBI nt % identity'
    mw_tsv_f.write(tsv_header + '\n')

    tsv_header += '\tAbundance by %s' % mapping_category
    html_header = ''
    for col in tsv_header.split('\t'):
        html_header += '%s	' % col
    mw_html_f.write('

' + html_header + '
')

    for otu_id, subject_id, percent_identity in top_n_mw:
        # Grab all necessary information to be included in our report.
        seq = mw_seqs[otu_id]
        tax = otu_table_by_samp_type_ms.ObservationMetadata[
            otu_table_by_samp_type_ms.getObservationIndex(otu_id)]['taxonomy']
        gb_id = subject_id.split('|')[3]
        ncbi_link = 'http://www.ncbi.nlm.nih.gov/nuccore/%s' % gb_id

        # Compute the abundance of each most wanted OTU in each sample
        # grouping and create a pie chart to go in the HTML table.
        samp_types = otu_table_by_samp_type_ms.SampleIds
        counts = otu_table_by_samp_type_ms.observationData(otu_id)
        if len(counts) != len(samp_types):
            raise ValueError("The number of observation counts does not match "
                             "the number of samples in the OTU table.")
        # Piechart code modified from matplotlib example:
        # http://matplotlib.sourceforge.net/examples/pylab_examples/pie_demo.html

        # Make a square figure and axes.
        figure(1, figsize=(6,6))
        ax = axes([0.1, 0.1, 0.8, 0.8])

        # Will auto-normalize the counts.
        pie(counts, labels=samp_types, autopct='%1.1f%%', shadow=True)

        output_img_dir = join(output_dir, 'img')
        try:
            makedirs(output_img_dir)
        except OSError:
            pass

        pie_chart_fp = join(output_img_dir, 'abundance_by_%s_%s.png' %
                            (mapping_category, otu_id))
        savefig(pie_chart_fp)

        mw_tsv_f.write('%s\t%s\t%s\t%s\t%s\n' %
                       (otu_id, seq, tax, gb_id, percent_identity))

        mw_html_f.write('
%s	%s	%s	'
                '%s	%s	'
                '
' % (otu_id, seq, tax, ncbi_link,
                gb_id, percent_identity, pie_chart_fp))
    mw_html_f.write('
')
    mw_tsv_f.close()
    mw_html_f.close()
<commit_msg>Added missing import statements to most wanted.<commit_after>#!/usr/bin/env python
from __future__ import division

__author__ = "Jai Ram Rideout"
__copyright__ = "Copyright 2012, The QIIME project"
__credits__ = ["Jai Ram Rideout", "Greg Caporaso"]
__license__ = "GPL"
__version__ = "1.5.0-dev"
__maintainer__ = "Jai Ram Rideout"
__email__ = "jai.rideout@gmail.com"
__status__ = "Development"

"""Contains functions used in the most_wanted_otus.py script."""

from collections import defaultdict
from operator import itemgetter
from os import makedirs
from os.path import basename, join, splitext
from tempfile import NamedTemporaryFile

from pylab import axes, figure, pie, savefig

from biom.parse import parse_biom_table

from cogent import DNA, LoadSeqs
from cogent.app.blast import blast_seqs, Blastall
from cogent.app.formatdb import build_blast_db_from_fasta_path
from cogent.parse.blast import BlastResult
from cogent.parse.fasta import MinimalFastaParser
from cogent.util.misc import remove_files

from qiime.parse import parse_mapping_file_to_dict
from qiime.util import (add_filename_suffix, parse_command_line_parameters,
        get_options_lookup, make_option, qiime_system_call)

def generate_most_wanted_list(otu_table_fp, rep_set_fp, gg_fp, nt_fp,
        mapping_fp, mapping_category, top_n, min_abundance, max_abundance,
        min_categories, max_gg_similarity, jobs_to_start, output_dir):
    try:
        makedirs(output_dir)
    except OSError:
        pass

    # First filter to keep only new (non-GG) OTUs.
    print "Filtering out all GG reference OTUs.\n"
    novel_otu_table_fp = join(output_dir, add_filename_suffix(otu_table_fp,
        '_novel'))
    stdout, stderr, ret_val = qiime_system_call(
        'filter_otus_from_otu_table.py -i %s -o %s -e %s' %
        (otu_table_fp, novel_otu_table_fp, gg_fp))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # Next filter to keep only abundant otus in the specified range (looking
    # only at extremely abundant OTUs has the problem of yielding too many
    # that are similar to stuff in the nt database).
    print "Filtering out all OTUs that do not fall within the specified " + \
          "abundance threshold.\n"
    novel_abund_otu_table_fp = join(output_dir, add_filename_suffix(novel_otu_table_fp,
            '_min%d_max%d' % (min_abundance, max_abundance)))
    stdout, stderr, ret_val = qiime_system_call(
        'filter_otus_from_otu_table.py -i %s -o %s -n %d -x %d' %
        (novel_otu_table_fp, novel_abund_otu_table_fp, min_abundance,
         max_abundance))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # Next, collapse by mapping_category.
    print "Collapsing OTU table by %s.\n" % mapping_category
    otu_table_by_samp_type_fp = join(output_dir, add_filename_suffix(novel_abund_otu_table_fp,
            '_%s' % mapping_category))
    stdout, stderr, ret_val = qiime_system_call(
        'summarize_otu_by_cat.py -c %s -o %s -m %s -i %s' %
        (novel_abund_otu_table_fp, otu_table_by_samp_type_fp, mapping_category,
         mapping_fp))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # Filter to contain only otus in the specified minimum number of sample
    # types.
    print "Filtering OTU table to include only OTUs that appear in at " + \
          "least %d sample groups.\n" % min_categories
    otu_table_by_samp_type_ms_fp = join(output_dir, add_filename_suffix(
            otu_table_by_samp_type_fp, '_ms%d' % min_categories))
    stdout, stderr, ret_val = qiime_system_call(
        'filter_otus_from_otu_table.py -i %s -o %s -s %d' %
        (otu_table_by_samp_type_fp, otu_table_by_samp_type_ms_fp,
         min_categories))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # Now that we have a filtered down OTU table of good candidate OTUs, filter
    # the corresponding representative set to include only these candidate
    # sequences.
    print "Filtering representative set to include only the latest " + \
          "candidate OTUs.\n"
    candidate_rep_set_fp = join(output_dir, add_filename_suffix(
            rep_set_fp, '_most_wanted_candidates'))
    stdout, stderr, ret_val = qiime_system_call(
        'filter_fasta.py -f %s -o %s -b %s' %
        (rep_set_fp, candidate_rep_set_fp, otu_table_by_samp_type_ms_fp))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # Filter to contain only otus that don't hit GG at a certain similarity
    # threshold.
    print "Running uclust to get list of sequences that don't hit the " + \
          "maximum GG similarity threshold.\n"
    uclust_output_dir = join(output_dir, 'most_wanted_candidates_%s_%s'
            % (basename(gg_fp), str(max_gg_similarity)))
    stdout, stderr, ret_val = qiime_system_call(
        'parallel_pick_otus_uclust_ref.py -i %s -o %s -r %s -s %s -O %d'
        % (candidate_rep_set_fp, uclust_output_dir, gg_fp,
           str(max_gg_similarity), jobs_to_start))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    print "Filtering candidate sequences to only include uclust failures.\n"
    cand_gg_dis_rep_set_fp = join(output_dir,
            add_filename_suffix(candidate_rep_set_fp,
                                '_failures'))
    stdout, stderr, ret_val = qiime_system_call(
        'filter_fasta.py -f %s -s %s -o %s' % (candidate_rep_set_fp,
        join(uclust_output_dir,
        splitext(basename(candidate_rep_set_fp))[0] + '_failures.txt'),
        cand_gg_dis_rep_set_fp))
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # BLAST the failures against nt.
    print "BLASTing candidate sequences against nt database.\n"
    blast_output_dir = join(output_dir, 'blast_output')
    blast_cmd = 'parallel_blast.py -i %s -o %s -r %s -D -e 1e-3 -O %d' % (cand_gg_dis_rep_set_fp, blast_output_dir, nt_fp, jobs_to_start)
    print blast_cmd
    stdout, stderr, ret_val = qiime_system_call(blast_cmd)
    print stdout
    print stderr
    if ret_val != 0:
        exit(1)

    # Read in our filtered down candidate seqs file and latest filtered and
    # collapsed OTU table. We'll need to compute some stats on these to include
    # in our report.
    print "Collating results and formatting for output.\n"
    mw_seqs = {}
    for seq_id, seq in MinimalFastaParser(open(cand_gg_dis_rep_set_fp, 'U')):
        seq_id = seq_id.strip().split()[0]
        mw_seqs[seq_id] = seq
    otu_table_by_samp_type_ms = parse_biom_table(
            open(otu_table_by_samp_type_ms_fp, 'U'))

    # We'll sort the BLAST results by percent identity (ascending) and pick the
    # top n.
    blast_results = open(join(blast_output_dir,
        splitext(basename(cand_gg_dis_rep_set_fp))[0] + '_blast_out.txt'), 'U')
    top_n_mw = []
    for line in blast_results:
        # Skip headers.
        line = line.strip()
        if line and not line.startswith('#'):
            line = line.split('\t')
            top_n_mw.append((line[0], line[1], float(line[2])))
    top_n_mw = sorted(top_n_mw, key=itemgetter(2))[:top_n]

    # Write results out to tsv and HTML table.
    mw_tsv_f = open(join(output_dir,
                    'top_%d_most_wanted_otus.txt' % top_n), 'w')
    mw_html_f = open(join(output_dir,
                    'top_%d_most_wanted_otus.html' % top_n), 'w')
    tsv_header = 'OTU ID\tSequence\tGreengenes taxonomy\t' + \
                 'NCBI nt closest match\tNCBI nt % identity'
    mw_tsv_f.write(tsv_header + '\n')

    tsv_header += '\tAbundance by %s' % mapping_category
    html_header = ''
    for col in tsv_header.split('\t'):
        html_header += '%s	' % col
    mw_html_f.write('

' + html_header + '
')

    for otu_id, subject_id, percent_identity in top_n_mw:
        # Grab all necessary information to be included in our report.
        seq = mw_seqs[otu_id]
        tax = otu_table_by_samp_type_ms.ObservationMetadata[
            otu_table_by_samp_type_ms.getObservationIndex(otu_id)]['taxonomy']
        gb_id = subject_id.split('|')[3]
        ncbi_link = 'http://www.ncbi.nlm.nih.gov/nuccore/%s' % gb_id

        # Compute the abundance of each most wanted OTU in each sample
        # grouping and create a pie chart to go in the HTML table.
        samp_types = otu_table_by_samp_type_ms.SampleIds
        counts = otu_table_by_samp_type_ms.observationData(otu_id)
        if len(counts) != len(samp_types):
            raise ValueError("The number of observation counts does not match "
                             "the number of samples in the OTU table.")
        # Piechart code modified from matplotlib example:
        # http://matplotlib.sourceforge.net/examples/pylab_examples/pie_demo.html

        # Make a square figure and axes.
        figure(1, figsize=(6,6))
        ax = axes([0.1, 0.1, 0.8, 0.8])

        # Will auto-normalize the counts.
        pie(counts, labels=samp_types, autopct='%1.1f%%', shadow=True)

        output_img_dir = join(output_dir, 'img')
        try:
            makedirs(output_img_dir)
        except OSError:
            pass

        pie_chart_fp = join(output_img_dir, 'abundance_by_%s_%s.png' %
                            (mapping_category, otu_id))
        savefig(pie_chart_fp)

        mw_tsv_f.write('%s\t%s\t%s\t%s\t%s\n' %
                       (otu_id, seq, tax, gb_id, percent_identity))

        mw_html_f.write('
%s	%s	%s	'
                '%s	%s	'
                '
' % (otu_id, seq, tax, ncbi_link,
                gb_id, percent_identity, pie_chart_fp))
    mw_html_f.write('
')
    mw_tsv_f.close()
    mw_html_f.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#-----------------------
# Copyright 2014 Halfmoon Labs, Inc.
# All Rights Reserved
#-----------------------

import bitcoinrpc

from commontools import log, error_reply

#---------------------------------------
class BitcoindServer(object):

	#-----------------------------------
	def __init__(self, server, port, user, passwd, use_https=True, passphrase=None):

		self.passphrase = passphrase
		self.server = server

		self.bitcoind = bitcoinrpc.connect_to_remote(user, passwd,
										host=server, port=port,
										use_https=use_https)

	#-----------------------------------
	def blocks(self):
		reply = {}
		info = self.bitcoind.getinfo()
		reply['blocks'] = info.blocks
		return reply

	#-----------------------------------
	#helper function
	def unlock_wallet(self, passphrase, timeout = 100):

		info = self.bitcoind.walletpassphrase(passphrase, timeout, True)
		return info             #info will be True or False

	#-----------------------------------
	def sendtoaddress(self, bicoinaddress, amount):

		try:
			status = self.bitcoind.sendtoaddress(bitcoinaddress, float(amount))
			log.debug(status)
			return status
		except Exception as e:
			log.debug(str(e))
			return error_reply(str(e))

	#-----------------------------------
	def validateaddress(self, bitcoinaddress):

		try:
			status = self.bitcoind.validateaddress(bitcoinaddress)
			log.debug(status)
			return status
		except Exception as e:
			log.debug(str(e))
			return error_reply(str(e))

	#-----------------------------------
	def importprivkey(self, bitcoinprivkey, label='', rescan=False):

		try:
			if rescan is True:
				status = self.bitcoind.importprivkey(bitcoinprivkey, label, True)
			else:
				status = self.bitcoind.importprivkey(bitcoinprivkey, label, False)
			log.debug(status)
			return status
		except Exception as e:
			log.debug(str(e))
			return error_reply(str(e))
<commit_msg>fixed small bug in unlock wallet<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#-----------------------
# Copyright 2014 Halfmoon Labs, Inc.
# All Rights Reserved
#-----------------------

import bitcoinrpc

from commontools import log, error_reply

#---------------------------------------
class BitcoindServer(object):

	#-----------------------------------
	def __init__(self, server, port, user, passwd, use_https=True, passphrase=None):

		self.passphrase = passphrase
		self.server = server

		self.bitcoind = bitcoinrpc.connect_to_remote(user, passwd,
										host=server, port=port,
										use_https=use_https)

	#-----------------------------------
	def blocks(self):
		reply = {}
		info = self.bitcoind.getinfo()
		reply['blocks'] = info.blocks
		return reply

	#-----------------------------------
	#helper function
	def unlock_wallet(self, timeout = 120):

		info = self.bitcoind.walletpassphrase(self.passphrase, timeout, True)
		return info             #info will be True or False

	#-----------------------------------
	def sendtoaddress(self, bicoinaddress, amount):

		try:
			status = self.bitcoind.sendtoaddress(bitcoinaddress, float(amount))
			log.debug(status)
			return status
		except Exception as e:
			log.debug(str(e))
			return error_reply(str(e))

	#-----------------------------------
	def validateaddress(self, bitcoinaddress):

		try:
			status = self.bitcoind.validateaddress(bitcoinaddress)
			log.debug(status)
			return status
		except Exception as e:
			log.debug(str(e))
			return error_reply(str(e))

	#-----------------------------------
	def importprivkey(self, bitcoinprivkey, label='', rescan=False):

		try:
			if rescan is True:
				status = self.bitcoind.importprivkey(bitcoinprivkey, label, True)
			else:
				status = self.bitcoind.importprivkey(bitcoinprivkey, label, False)
			log.debug(status)
			return status
		except Exception as e:
			log.debug(str(e))
			return error_reply(str(e))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# OpenFisca -- A versatile microsimulation software
# By: OpenFisca Team <contact@openfisca.fr>
#
# Copyright (C) 2011, 2012, 2013, 2014 OpenFisca Team
# https://github.com/openfisca
#
# This file is part of OpenFisca.
#
# OpenFisca is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# OpenFisca is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import collections
import json

from openfisca_web_site import conf, contexts, conv, model, wsgihelpers

class Node(model.Page):
    @property
    def routings(self):
        return super(Node, self).routings + (
            ('POST', '^/?$', self.submit),
            )

    @wsgihelpers.wsgify
    def submit(self, req):
        ctx = contexts.Ctx(req)
        headers = wsgihelpers.handle_cross_origin_resource_sharing(ctx)

        assert req.method == 'POST', req.method

        content_type = req.content_type
        if content_type is not None:
            content_type = content_type.split(';', 1)[0].strip()
        if content_type == 'application/json':
            simulation, error = conv.pipe(
                conv.make_input_to_json(object_pairs_hook = collections.OrderedDict),
                conv.test_isinstance(dict),
                conv.not_none,
                )(req.body, state = ctx)
            if error is not None:
                return wsgihelpers.respond_json(ctx,
                    collections.OrderedDict(sorted(dict(
                        apiVersion = '1.0',
                        error = collections.OrderedDict(sorted(dict(
                            code = 400,  # Bad Request
                            errors = [error],
                            message = ctx._(u'Invalid JSON in request POST body'),
                            ).iteritems())),
                        method = req.script_name,
                        params = req.body,
                        url = req.url.decode('utf-8'),
                        ).iteritems())),
                    headers = headers,
                    )
            api_url = conf['urls.api']
        else:
            # URL-encoded POST.
            params = req.POST
            inputs = dict(
                api_url = params.get('api_url'),
                simulation = params.get('simulation'),
                )
            data, errors = conv.struct(
                dict(
                    api_url = conv.pipe(
                        conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                            full = True),
                        conv.default(conf['urls.api']),
                        ),
                    simulation = conv.pipe(
                        conv.make_input_to_json(object_pairs_hook = collections.OrderedDict),
                        conv.test_isinstance(dict),
                        conv.not_none,
                        ),
                    ),
                )(inputs, state = ctx)
            if errors is not None:
                return wsgihelpers.respond_json(ctx,
                    collections.OrderedDict(sorted(dict(
                        apiVersion = '1.0',
                        context = inputs.get('context'),
                        error = collections.OrderedDict(sorted(dict(
                            code = 400,  # Bad Request
                            errors = [errors],
                            message = ctx._(u'Bad parameters in request'),
                            ).iteritems())),
                        method = req.script_name,
                        params = inputs,
                        url = req.url.decode('utf-8'),
                        ).iteritems())),
                    headers = headers,
                    )
            api_url = data['api_url']
            simulation = data['simulation']
        simulation_text = unicode(json.dumps(simulation, encoding = 'utf-8', ensure_ascii = False, indent = 2))

        response = req.response
        response.headers.update(headers)
        return self.render(api_url = api_url, simulation_text = simulation_text)
<commit_msg>Add support for GET in trace.<commit_after># -*- coding: utf-8 -*-

# OpenFisca -- A versatile microsimulation software
# By: OpenFisca Team <contact@openfisca.fr>
#
# Copyright (C) 2011, 2012, 2013, 2014 OpenFisca Team
# https://github.com/openfisca
#
# This file is part of OpenFisca.
#
# OpenFisca is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# OpenFisca is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import collections
import json

from openfisca_web_site import conf, contexts, conv, model, wsgihelpers

class Node(model.Page):
    @property
    def routings(self):
        return (
            (('GET', 'POST'), '^/?$', self.trace),
            ) + super(Node, self).routings

    @wsgihelpers.wsgify
    def trace(self, req):
        ctx = contexts.Ctx(req)
        headers = wsgihelpers.handle_cross_origin_resource_sharing(ctx)

        content_type = req.content_type
        if content_type is not None:
            content_type = content_type.split(';', 1)[0].strip()
        if req.method == 'POST' and content_type == 'application/json':
            simulation, error = conv.pipe(
                conv.make_input_to_json(object_pairs_hook = collections.OrderedDict),
                conv.test_isinstance(dict),
                conv.not_none,
                )(req.body, state = ctx)
            if error is not None:
                return wsgihelpers.respond_json(ctx,
                    collections.OrderedDict(sorted(dict(
                        apiVersion = '1.0',
                        error = collections.OrderedDict(sorted(dict(
                            code = 400,  # Bad Request
                            errors = [error],
                            message = ctx._(u'Invalid JSON in request POST body'),
                            ).iteritems())),
                        method = req.script_name,
                        params = req.body,
                        url = req.url.decode('utf-8'),
                        ).iteritems())),
                    headers = headers,
                    )
            api_url = conf['urls.api']
        else:
            # URL-encoded GET or POST.
            params = req.params
            inputs = dict(
                api_url = params.get('api_url'),
                simulation = params.get('simulation'),
                )
            data, errors = conv.struct(
                dict(
                    api_url = conv.pipe(
                        conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                            full = True),
                        conv.default(conf['urls.api']),
                        ),
                    simulation = conv.pipe(
                        conv.make_input_to_json(object_pairs_hook = collections.OrderedDict),
                        conv.test_isinstance(dict),
                        # When None, use default value defined in atom.mako.
                        ),
                    ),
                )(inputs, state = ctx)
            if errors is not None:
                return wsgihelpers.respond_json(ctx,
                    collections.OrderedDict(sorted(dict(
                        apiVersion = '1.0',
                        context = inputs.get('context'),
                        error = collections.OrderedDict(sorted(dict(
                            code = 400,  # Bad Request
                            errors = [errors],
                            message = ctx._(u'Bad parameters in request'),
                            ).iteritems())),
                        method = req.script_name,
                        params = inputs,
                        url = req.url.decode('utf-8'),
                        ).iteritems())),
                    headers = headers,
                    )
            api_url = data['api_url']
            simulation = data['simulation']
        simulation_text = unicode(json.dumps(simulation, encoding = 'utf-8', ensure_ascii = False, indent = 2)) \
            if simulation is not None else None

        response = req.response
        response.headers.update(headers)
        return self.render(api_url = api_url, simulation_text = simulation_text)
<|endoftext|>


<commit_before>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
from selenium.common import exceptions
from selenium.webdriver.common import by

from openstack_dashboard.test.integration_tests.regions import baseregion

class NavigationAccordionRegion(baseregion.BaseRegion):
    """Navigation menu located in the left."""
    _project_access_security_locator = (
        by.By.CSS_SELECTOR, 'a[href*="/project/access_and_security/"]')
    _settings_change_password_locator = (
        by.By.CSS_SELECTOR, 'a[href*="/settings/password//"]')
    _project_bar_locator = (by.By.XPATH,
                            ".//*[@id='main_content']//div[contains(text(),"
                            "'Project')]")

    @property
    def project_bar(self):
        return self._get_element(*self._project_bar_locator)

    _first_level_item_selected_locator = (
        by.By.CSS_SELECTOR, 'li.openstack-dashboard.selenium-active > a')
    _second_level_item_selected_locator = (
        by.By.CSS_SELECTOR, 'li.nav-header.selenium-active > a')

    _first_level_item_xpath_template = (
        "//li[contains(concat('', @class, ''), 'openstack-dashboard') "
        "and contains(., '%s')]/a")
    _second_level_item_xpath_template = (
        "//li[contains(concat('', @class, ''), 'nav-header') "
        "and contains(., '%s')]/a")
    _third_level_item_xpath_template = (
        "//li[contains(concat('', @class, ''), 'openstack-panel') and "
        "contains(., '%s')]/a")

    _parent_item_locator = (by.By.XPATH, '..')
    _menu_list_locator = (by.By.CSS_SELECTOR, 'ul')
    _expanded_menu_class = 'in'
    _transitioning_menu_class = 'collapsing'

    def _get_first_level_item_locator(self, text):
        return (by.By.XPATH,
                self._first_level_item_xpath_template % text)

    def _get_second_level_item_locator(self, text):
        return (by.By.XPATH,
                self._second_level_item_xpath_template % text)

    def _get_third_level_item_locator(self, text):
        return (by.By.XPATH,
                self._third_level_item_xpath_template % text)

    def get_first_level_selected_item(self):
        if self._is_element_present(*self._first_level_item_selected_locator):
            return self._get_element(*self._first_level_item_selected_locator)
        else:
            return None

    def get_second_level_selected_item(self):
        if self._is_element_present(*self._second_level_item_selected_locator):
            return self._get_element(*self._second_level_item_selected_locator)
        else:
            return None

    @property
    def access_security(self):
        return self._get_element(*self._project_access_security_locator)

    @property
    def change_password(self):
        return self._get_element(*self._settings_change_password_locator)

    def _wait_until_transition_ends(self, item, to_be_expanded=False):
        def predicate(d):
            classes = item.get_attribute('class').split()
            if to_be_expanded:
                status = self._expanded_menu_class in classes
            else:
                status = self._expanded_menu_class not in classes
            return status and self._transitioning_menu_class not in classes
        self._wait_until(predicate)

    def _click_menu_item(self, text, loc_craft_func, get_selected_func=None):
        """Click on menu item if not selected.

        Menu animation that visualize transition from one selection to
        another take some time - if clicked on item during this animation
        nothing happens, therefore it is necessary to wait for the transition
        to complete first.

        Third-level menus are handled differently from others. First, they do
        not need to collapse other 3rd-level menu item prior to clicking them
        (because 3rd-level menus are atomic). Second, clicking them doesn't
        initiate an animated transition, hence no need to wait until it
        finishes.
        """
        is_already_within_required_item = False
        if get_selected_func is not None:
            selected_item = get_selected_func()
            if selected_item:
                if text != selected_item.text:
                    # In case different item was chosen previously scroll it,
                    # because otherwise selenium will complain with
                    # MoveTargetOutOfBoundsException
                    selected_item.click()
                    self._wait_until_transition_ends(selected_item)
                else:
                    is_already_within_required_item = True

        if not is_already_within_required_item:
            item = self._get_item(text, loc_craft_func)
            item.click()
            if get_selected_func is not None:
                self._wait_until_transition_ends(
                    self._get_menu_list_next_to_menu_title(item),
                    to_be_expanded=True)

    def _get_item(self, text, loc_craft_func):
        item_locator = loc_craft_func(text)
        return self._get_element(*item_locator)

    def _get_menu_list_next_to_menu_title(self, title_item):
        parent_item = title_item.find_element(*self._parent_item_locator)
        return parent_item.find_element(*self._menu_list_locator)

    def click_on_menu_items(self, first_level=None,
                            second_level=None,
                            third_level=None):
        if first_level:
            self._click_menu_item(first_level,
                                  self._get_first_level_item_locator,
                                  self.get_first_level_selected_item)
        if second_level:
            self._click_menu_item(second_level,
                                  self._get_second_level_item_locator,
                                  self.get_second_level_selected_item)

        if third_level:
            self._click_menu_item(third_level,
                                  self._get_third_level_item_locator)

class DropDownMenuRegion(baseregion.BaseRegion):
    """Drop down menu region."""

    _menu_items_locator = (by.By.CSS_SELECTOR,
                           'ul.dropdown-menu > li > *')
    _menu_first_child_locator = (by.By.CSS_SELECTOR,
                                 'a[data-toggle="dropdown"]')

    @property
    def menu_items(self):
        self.open()
        menu_items = self._get_elements(*self._menu_items_locator)
        return menu_items

    def is_open(self):
        """Returns True if drop down menu is open, otherwise False."""
        return "open" in self.src_elem.get_attribute('class')

    def open(self):
        """Opens menu by clicking on the first child of the source element."""
        if self.is_open() is False:
            self._get_element(*self._menu_first_child_locator).click()

class UserDropDownMenuRegion(DropDownMenuRegion):
    """Drop down menu located in the right side of the topbar,
    contains links to settings and help.
    """
    _menu_first_child_locator = (by.By.CSS_SELECTOR, '*')
    _settings_link_locator = (by.By.CSS_SELECTOR,
                              'a[href*="/settings/"]')
    _help_link_locator = (by.By.CSS_SELECTOR,
                          'ul#editor_list li:nth-of-type(2) > a')
    _logout_link_locator = (by.By.CSS_SELECTOR,
                            'a[href*="/auth/logout/"]')

    @property
    def settings_link(self):
        return self._get_element(*self._settings_link_locator)

    @property
    def help_link(self):
        return self._get_element(*self._help_link_locator)

    @property
    def logout_link(self):
        return self._get_element(*self._logout_link_locator)

    def click_on_settings(self):
        self.open()
        self.settings_link.click()

    def click_on_help(self):
        self.open()
        self.help_link.click()

    def click_on_logout(self):
        self.open()
        self.logout_link.click()

class TabbedMenuRegion(baseregion.BaseRegion):

    _tab_locator = (by.By.CSS_SELECTOR, 'a')
    _default_src_locator = (by.By.CSS_SELECTOR, 'ul.nav.nav-tabs')

    def switch_to(self, index=0):
        self._get_elements(*self._tab_locator)[index].click()

class ProjectDropDownRegion(DropDownMenuRegion):

    _menu_first_child_locator = (by.By.CSS_SELECTOR, '*')
    _menu_items_locator = (by.By.CSS_SELECTOR,
                           'div.context-lists a')

    def click_on_project(self, name):
        for item in self.menu_items:
            if item.text == name:
                item.click()
                break
        else:
            raise exceptions.NoSuchElementException(
                "Not found element with text: %s" % name)
<commit_msg>Eliminate ambiguity when matching panel in i9n tests<commit_after>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
from selenium.common import exceptions
from selenium.webdriver.common import by

from openstack_dashboard.test.integration_tests.regions import baseregion

class NavigationAccordionRegion(baseregion.BaseRegion):
    """Navigation menu located in the left."""
    _project_access_security_locator = (
        by.By.CSS_SELECTOR, 'a[href*="/project/access_and_security/"]')
    _settings_change_password_locator = (
        by.By.CSS_SELECTOR, 'a[href*="/settings/password//"]')
    _project_bar_locator = (by.By.XPATH,
                            ".//*[@id='main_content']//div[contains(text(),"
                            "'Project')]")

    @property
    def project_bar(self):
        return self._get_element(*self._project_bar_locator)

    _first_level_item_selected_locator = (
        by.By.CSS_SELECTOR, 'li.openstack-dashboard.selenium-active > a')
    _second_level_item_selected_locator = (
        by.By.CSS_SELECTOR, 'li.nav-header.selenium-active > a')

    _first_level_item_xpath_template = (
        "//li[contains(concat('', @class, ''), 'openstack-dashboard') "
        "and contains(., '%s')]/a")
    _second_level_item_xpath_template = (
        "//li[contains(concat('', @class, ''), 'nav-header') "
        "and contains(., '%s')]/a")
    _third_level_item_xpath_template = (
        ".//li[contains(concat('', @class, ''), 'openstack-panel') and "
        "contains(., '%s')]/a")

    _parent_item_locator = (by.By.XPATH, '..')
    _menu_list_locator = (by.By.CSS_SELECTOR, 'ul')
    _expanded_menu_class = 'in'
    _transitioning_menu_class = 'collapsing'

    def _get_first_level_item_locator(self, text):
        return (by.By.XPATH,
                self._first_level_item_xpath_template % text)

    def _get_second_level_item_locator(self, text):
        return (by.By.XPATH,
                self._second_level_item_xpath_template % text)

    def _get_third_level_item_locator(self, text):
        return (by.By.XPATH,
                self._third_level_item_xpath_template % text)

    def get_first_level_selected_item(self):
        if self._is_element_present(*self._first_level_item_selected_locator):
            return self._get_element(*self._first_level_item_selected_locator)
        else:
            return None

    def get_second_level_selected_item(self):
        if self._is_element_present(*self._second_level_item_selected_locator):
            return self._get_element(*self._second_level_item_selected_locator)
        else:
            return None

    @property
    def access_security(self):
        return self._get_element(*self._project_access_security_locator)

    @property
    def change_password(self):
        return self._get_element(*self._settings_change_password_locator)

    def _wait_until_transition_ends(self, item, to_be_expanded=False):
        def predicate(d):
            classes = item.get_attribute('class').split()
            if to_be_expanded:
                status = self._expanded_menu_class in classes
            else:
                status = self._expanded_menu_class not in classes
            return status and self._transitioning_menu_class not in classes
        self._wait_until(predicate)

    def _click_menu_item(self, text, loc_craft_func, get_selected_func=None,
                         src_elem=None):
        """Click on menu item if not selected.

        Menu animation that visualize transition from one selection to
        another take some time - if clicked on item during this animation
        nothing happens, therefore it is necessary to wait for the transition
        to complete first.

        Third-level menus are handled differently from others. First, they do
        not need to collapse other 3rd-level menu item prior to clicking them
        (because 3rd-level menus are atomic). Second, clicking them doesn't
        initiate an animated transition, hence no need to wait until it
        finishes. Also an ambiguity is possible when matching third-level menu
        items, because different dashboards have panels with the same name (for
        example Volumes/Images/Instances both at Project and Admin dashboards).
        To avoid this ambiguity, an argument `src_elem` is used. Whenever
        dashboard or a panel group is clicked, its wrapper is returned to be
        used as `src_elem` in a subsequent call for clicking third-level item.
        This way the set of panel labels being matched is restricted to the
        descendants of that particular dashboard or panel group.
        """
        is_already_within_required_item = False
        selected_item = None
        if get_selected_func is not None:
            selected_item = get_selected_func()
            if selected_item:
                if text != selected_item.text:
                    # In case different item was chosen previously, collapse
                    # it. Otherwise selenium will complain with
                    # MoveTargetOutOfBoundsException
                    selected_item.click()
                    self._wait_until_transition_ends(selected_item)
                else:
                    is_already_within_required_item = True

        if not is_already_within_required_item:
            item = self._get_item(text, loc_craft_func, src_elem)
            item.click()
            if get_selected_func is not None:
                self._wait_until_transition_ends(
                    self._get_menu_list_next_to_menu_title(item),
                    to_be_expanded=True)
            return item
        return selected_item

    def _get_item(self, text, loc_craft_func, src_elem=None):
        item_locator = loc_craft_func(text)
        src_elem = src_elem or self.src_elem
        return src_elem.find_element(*item_locator)

    def _get_menu_list_next_to_menu_title(self, title_item):
        parent_item = title_item.find_element(*self._parent_item_locator)
        return parent_item.find_element(*self._menu_list_locator)

    def click_on_menu_items(self, first_level=None,
                            second_level=None,
                            third_level=None):
        src_elem = None
        if first_level:
            src_elem = self._click_menu_item(
                first_level,
                self._get_first_level_item_locator,
                self.get_first_level_selected_item)
        if second_level:
            src_elem = self._click_menu_item(
                second_level,
                self._get_second_level_item_locator,
                self.get_second_level_selected_item)

        if third_level:
            # NOTE(tsufiev): possible dashboard/panel group label passed as
            # `src_elem` is a sibling of <ul> with all the panels it contains.
            # So to get the panel within specified dashboard/panel group, we
            # need to traverse upwards first. When `src_elem` is not specified
            # (true for Settings pseudo-dashboard), we cannot and should not
            # go upward.
            if src_elem:
                src_elem = src_elem.find_element(*self._parent_item_locator)
            self._click_menu_item(third_level,
                                  self._get_third_level_item_locator,
                                  src_elem=src_elem)

class DropDownMenuRegion(baseregion.BaseRegion):
    """Drop down menu region."""

    _menu_items_locator = (by.By.CSS_SELECTOR,
                           'ul.dropdown-menu > li > *')
    _menu_first_child_locator = (by.By.CSS_SELECTOR,
                                 'a[data-toggle="dropdown"]')

    @property
    def menu_items(self):
        self.open()
        menu_items = self._get_elements(*self._menu_items_locator)
        return menu_items

    def is_open(self):
        """Returns True if drop down menu is open, otherwise False."""
        return "open" in self.src_elem.get_attribute('class')

    def open(self):
        """Opens menu by clicking on the first child of the source element."""
        if self.is_open() is False:
            self._get_element(*self._menu_first_child_locator).click()

class UserDropDownMenuRegion(DropDownMenuRegion):
    """Drop down menu located in the right side of the topbar,
    contains links to settings and help.
    """
    _menu_first_child_locator = (by.By.CSS_SELECTOR, '*')
    _settings_link_locator = (by.By.CSS_SELECTOR,
                              'a[href*="/settings/"]')
    _help_link_locator = (by.By.CSS_SELECTOR,
                          'ul#editor_list li:nth-of-type(2) > a')
    _logout_link_locator = (by.By.CSS_SELECTOR,
                            'a[href*="/auth/logout/"]')

    @property
    def settings_link(self):
        return self._get_element(*self._settings_link_locator)

    @property
    def help_link(self):
        return self._get_element(*self._help_link_locator)

    @property
    def logout_link(self):
        return self._get_element(*self._logout_link_locator)

    def click_on_settings(self):
        self.open()
        self.settings_link.click()

    def click_on_help(self):
        self.open()
        self.help_link.click()

    def click_on_logout(self):
        self.open()
        self.logout_link.click()

class TabbedMenuRegion(baseregion.BaseRegion):

    _tab_locator = (by.By.CSS_SELECTOR, 'a')
    _default_src_locator = (by.By.CSS_SELECTOR, 'ul.nav.nav-tabs')

    def switch_to(self, index=0):
        self._get_elements(*self._tab_locator)[index].click()

class ProjectDropDownRegion(DropDownMenuRegion):

    _menu_first_child_locator = (by.By.CSS_SELECTOR, '*')
    _menu_items_locator = (by.By.CSS_SELECTOR,
                           'div.context-lists a')

    def click_on_project(self, name):
        for item in self.menu_items:
            if item.text == name:
                item.click()
                break
        else:
            raise exceptions.NoSuchElementException(
                "Not found element with text: %s" % name)
<|endoftext|>


<commit_before>"""avakas

The avakas tool is meant as an interface around version
metadata for assorted flavours of software projects.

For more information see https://github.com/otakup0pe/avakas
"""

from __future__ import print_function

import datetime
import os
import sys
import argparse

from git import Repo

from .avakas import detect_project_flavor
from .errors import AvakasError

def get_repo(directory):
    """Load the git repository."""
    return Repo(directory, search_parent_directories=True)

def git_rev(directory):
    """Returns the first eight characters of HEAD"""
    return str(get_repo(directory).head.commit)[0:8]

def add_metadata(project, buildmeta=False, **kwargs):
    """
    Add metadata for set/bump actions
    """
    directory = kwargs['directory'][0]

    git_str = str(git_rev(directory))
    if buildmeta:
        metadata = (git_str,)
        metadata += ci_build_meta()
        project.apply_metadata(*metadata)

    return project

def ci_build_meta():
    """Return any CI system specific build metadata"""
    meta = ()
    if 'BUILD_NUMBER' in os.environ:
        meta = (os.environ['BUILD_NUMBER'],)
    elif 'TRAVIS_BUILD_NUMBER' in os.environ:
        meta = (os.environ['TRAVIS_BUILD_NUMBER'],)
    elif 'CIRCLE_BUILD_NUM' in os.environ:
        meta = (os.environ['CIRCLE_BUILD_NUM'],)
    elif ('GITHUB_RUN_ID' in os.environ) and \
         ('GITHUB_RUN_NUMBER' in os.environ):
        meta = (os.environ['GITHUB_RUN_ID'], os.environ['GITHUB_RUN_NUMBER'],)
    return meta

def cli_show_version(**kwargs):
    """Show the current flavour specific version for a project."""
    project = detect_project_flavor(**kwargs)
    if not project.read():
        raise AvakasError('Unable to extract current version')

    print("%s" % str(project.version))

def cli_bump_version(
        level=None,
        prerelease=False,
        prerelease_date=False, **kwargs):
    """Bump the flavour specific version for a project."""
    project = detect_project_flavor(**kwargs)
    if not project.read():
        raise AvakasError('Unable to extract current version')
    old_version = project.version

    if not project.bump(
            bump=level[0],
            prerelease=prerelease,
            prerelease_prefix=kwargs['prerelease_prefix'],
            build_date=prerelease_date):
        sys.exit(0)
    project = add_metadata(project, **kwargs)
    project.write()

    print("Version updated from %s to %s" %
          (old_version, str(project.version)))

def cli_set_version(prerelease=False,
                    prerelease_date=False,
                    prerelease_prefix=None,
                    **kwargs):
    """Manually set the flavour specific version for a project."""

    version = kwargs['version'][0]
    project = detect_project_flavor(**kwargs)
    original_version = project.version_obj

    project.version = version
    if prerelease:
        prerelease_version = project.get_next_prerelease_version(
            starting_version=original_version,
            prefix=prerelease_prefix,
            new_version=project.version_obj
        )
        project.make_prerelease(prerelease_version, build_date=prerelease_date)
    project = add_metadata(project, **kwargs)
    project.write()

    print("Version set to %s" % version)

def parse_args(parser):
    """Parse our command line arguments."""

    bump_levels = ['patch', 'minor', 'major', 'auto']

    parser = argparse.ArgumentParser(prog="avakas",
                                     description='Process some integers.')

    subparsers = parser.add_subparsers(dest='operation')

    common = argparse.ArgumentParser(add_help=False)
    common.add_argument('--tag-prefix', dest='tag_prefix',
                        help='Prefix for version tag name',
                        default='')

    common.add_argument('--branch', dest='branch',
                        help='Branch to use when updating git',
                        default='master')

    common.add_argument('--remote', dest='remote',
                        help='Git remote',
                        default='origin')

    common.add_argument('--filename', dest='filename',
                        help='File name. Used for fallback versioning.',
                        default='version')
    common.add_argument('--flavor', dest='flavor',
                        help='Automation flavor for the project',
                        default='auto')
    common.add_argument('directory', nargs=1,
                        help='Directory of the project', default=os.getcwd())

    writable = argparse.ArgumentParser(add_help=False)
    writable.add_argument('--build-meta', dest='buildmeta',
                          help='Apply build-meta to version',
                          action='store_true',
                          default=False)
    writable.add_argument('--skip-dirty', dest='skipdirty',
                          help='Skip checking if local repo is dirty',
                          action='store_true',
                          default=False)
    writable.add_argument('--skip-commit-changes', dest='commitchanges',
                          help='Skip commiting generated version files',
                          action='store_false',
                          default=True)
    writable.add_argument('--with-hooks', dest='with_hooks',
                          help='Run git hooks', default=False)
    writable.add_argument('--dry-run',
                          dest='dry',
                          help='Will not push to git',
                          action='store_true')
    writable.add_argument('--prerelease',
                          dest='prerelease',
                          help='Will include prebuild information. If '
                          ' no other prebuild options are specified '
                          ' then it will simply use the build info in place.',
                          action='store_true')
    writable.add_argument('--prerelease-date',
                          dest='prerelease_date',
                          help='Include a string representation of the '
                          'current date, down to the second, as part '
                          'of the prebuild.',
                          action='store_true')
    writable.add_argument('--prerelease-prefix',
                          dest='prerelease_prefix',
                          help='Use the given string as a prebuild prefix',
                          default=None)

    set_p = subparsers.add_parser('set', parents=[common, writable])
    set_p.add_argument('version', nargs=1,
                       help='Desired version to set')

    bump_p = subparsers.add_parser('bump', parents=[common, writable])
    bump_p.add_argument('level', nargs=1, choices=bump_levels,
                        help='Level to bump at', default='auto')
    bump_p.add_argument('--default-bump', dest='default_bump',
                        choices=bump_levels, help='Level to bump at',
                        default=None)

    subparsers.add_parser('show', parents=[common])

    return parser.parse_args()

def main():
    """Dat entrypoint"""
    parser = argparse.ArgumentParser(prog="avakas")
    args = parse_args(parser)

    if args.operation is None:
        parser.print_help()
        sys.exit(0)

    directory = os.path.abspath(args.directory[0])

    if not os.path.exists(directory):
        raise AvakasError("Directory %s does not exist." % directory)

    try:
        if args.operation == 'bump':
            cli_bump_version(**vars(args))
        elif args.operation == 'show':
            cli_show_version(**vars(args))
        elif args.operation == 'set':
            cli_set_version(**vars(args))
        else:
            parser.print_help()
    except AvakasError as err:
        print("Problem: %s" % err.message, file=sys.stderr)
        sys.exit(1)

    sys.exit(0)
<commit_msg>Cleanup: removed unused import<commit_after>"""avakas

The avakas tool is meant as an interface around version
metadata for assorted flavours of software projects.

For more information see https://github.com/otakup0pe/avakas
"""

from __future__ import print_function

import os
import sys
import argparse

from git import Repo

from .avakas import detect_project_flavor
from .errors import AvakasError

def get_repo(directory):
    """Load the git repository."""
    return Repo(directory, search_parent_directories=True)

def git_rev(directory):
    """Returns the first eight characters of HEAD"""
    return str(get_repo(directory).head.commit)[0:8]

def add_metadata(project, buildmeta=False, **kwargs):
    """
    Add metadata for set/bump actions
    """
    directory = kwargs['directory'][0]

    git_str = str(git_rev(directory))
    if buildmeta:
        metadata = (git_str,)
        metadata += ci_build_meta()
        project.apply_metadata(*metadata)

    return project

def ci_build_meta():
    """Return any CI system specific build metadata"""
    meta = ()
    if 'BUILD_NUMBER' in os.environ:
        meta = (os.environ['BUILD_NUMBER'],)
    elif 'TRAVIS_BUILD_NUMBER' in os.environ:
        meta = (os.environ['TRAVIS_BUILD_NUMBER'],)
    elif 'CIRCLE_BUILD_NUM' in os.environ:
        meta = (os.environ['CIRCLE_BUILD_NUM'],)
    elif ('GITHUB_RUN_ID' in os.environ) and \
         ('GITHUB_RUN_NUMBER' in os.environ):
        meta = (os.environ['GITHUB_RUN_ID'], os.environ['GITHUB_RUN_NUMBER'],)
    return meta

def cli_show_version(**kwargs):
    """Show the current flavour specific version for a project."""
    project = detect_project_flavor(**kwargs)
    if not project.read():
        raise AvakasError('Unable to extract current version')

    print("%s" % str(project.version))

def cli_bump_version(
        level=None,
        prerelease=False,
        prerelease_date=False, **kwargs):
    """Bump the flavour specific version for a project."""
    project = detect_project_flavor(**kwargs)
    if not project.read():
        raise AvakasError('Unable to extract current version')
    old_version = project.version

    if not project.bump(
            bump=level[0],
            prerelease=prerelease,
            prerelease_prefix=kwargs['prerelease_prefix'],
            build_date=prerelease_date):
        sys.exit(0)
    project = add_metadata(project, **kwargs)
    project.write()

    print("Version updated from %s to %s" %
          (old_version, str(project.version)))

def cli_set_version(prerelease=False,
                    prerelease_date=False,
                    prerelease_prefix=None,
                    **kwargs):
    """Manually set the flavour specific version for a project."""

    version = kwargs['version'][0]
    project = detect_project_flavor(**kwargs)
    original_version = project.version_obj

    project.version = version
    if prerelease:
        prerelease_version = project.get_next_prerelease_version(
            starting_version=original_version,
            prefix=prerelease_prefix,
            new_version=project.version_obj
        )
        project.make_prerelease(prerelease_version, build_date=prerelease_date)
    project = add_metadata(project, **kwargs)
    project.write()

    print("Version set to %s" % version)

def parse_args(parser):
    """Parse our command line arguments."""

    bump_levels = ['patch', 'minor', 'major', 'auto']

    parser = argparse.ArgumentParser(prog="avakas",
                                     description='Process some integers.')

    subparsers = parser.add_subparsers(dest='operation')

    common = argparse.ArgumentParser(add_help=False)
    common.add_argument('--tag-prefix', dest='tag_prefix',
                        help='Prefix for version tag name',
                        default='')

    common.add_argument('--branch', dest='branch',
                        help='Branch to use when updating git',
                        default='master')

    common.add_argument('--remote', dest='remote',
                        help='Git remote',
                        default='origin')

    common.add_argument('--filename', dest='filename',
                        help='File name. Used for fallback versioning.',
                        default='version')
    common.add_argument('--flavor', dest='flavor',
                        help='Automation flavor for the project',
                        default='auto')
    common.add_argument('directory', nargs=1,
                        help='Directory of the project', default=os.getcwd())

    writable = argparse.ArgumentParser(add_help=False)
    writable.add_argument('--build-meta', dest='buildmeta',
                          help='Apply build-meta to version',
                          action='store_true',
                          default=False)
    writable.add_argument('--skip-dirty', dest='skipdirty',
                          help='Skip checking if local repo is dirty',
                          action='store_true',
                          default=False)
    writable.add_argument('--skip-commit-changes', dest='commitchanges',
                          help='Skip commiting generated version files',
                          action='store_false',
                          default=True)
    writable.add_argument('--with-hooks', dest='with_hooks',
                          help='Run git hooks', default=False)
    writable.add_argument('--dry-run',
                          dest='dry',
                          help='Will not push to git',
                          action='store_true')
    writable.add_argument('--prerelease',
                          dest='prerelease',
                          help='Will include prebuild information. If '
                          ' no other prebuild options are specified '
                          ' then it will simply use the build info in place.',
                          action='store_true')
    writable.add_argument('--prerelease-date',
                          dest='prerelease_date',
                          help='Include a string representation of the '
                          'current date, down to the second, as part '
                          'of the prebuild.',
                          action='store_true')
    writable.add_argument('--prerelease-prefix',
                          dest='prerelease_prefix',
                          help='Use the given string as a prebuild prefix',
                          default=None)

    set_p = subparsers.add_parser('set', parents=[common, writable])
    set_p.add_argument('version', nargs=1,
                       help='Desired version to set')

    bump_p = subparsers.add_parser('bump', parents=[common, writable])
    bump_p.add_argument('level', nargs=1, choices=bump_levels,
                        help='Level to bump at', default='auto')
    bump_p.add_argument('--default-bump', dest='default_bump',
                        choices=bump_levels, help='Level to bump at',
                        default=None)

    subparsers.add_parser('show', parents=[common])

    return parser.parse_args()

def main():
    """Dat entrypoint"""
    parser = argparse.ArgumentParser(prog="avakas")
    args = parse_args(parser)

    if args.operation is None:
        parser.print_help()
        sys.exit(0)

    directory = os.path.abspath(args.directory[0])

    if not os.path.exists(directory):
        raise AvakasError("Directory %s does not exist." % directory)

    try:
        if args.operation == 'bump':
            cli_bump_version(**vars(args))
        elif args.operation == 'show':
            cli_show_version(**vars(args))
        elif args.operation == 'set':
            cli_set_version(**vars(args))
        else:
            parser.print_help()
    except AvakasError as err:
        print("Problem: %s" % err.message, file=sys.stderr)
        sys.exit(1)

    sys.exit(0)
<|endoftext|>


<commit_before>import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import hamming, hanning, triang, blackmanharris, resample
from scipy.fftpack import fft, ifft, fftshift
from scipy.interpolate import interp1d
import math
import sys, os, time

sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../basicFunctions/'))
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../basicFunctions_C/'))

import smsF0DetectionTwm as fd
import smsWavplayer as wp
import smsPeakProcessing as PP

try:
  import basicFunctions_C as GS
except ImportError:
  import smsGenSpecSines as GS
  print "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"
  print "NOTE: Cython modules for some functions were not imported, the processing will be slow"
  print "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"

def hps(x, fs, w, N, t, nH, minf0, maxf0, f0et, maxhd, stocf, maxnpeaksTwm=10):
  # Analysis/synthesis of a sound using the harmonic plus stochastic model, prepared for transformations
  # x: input sound, fs: sampling rate, w: analysis window,
  # N: FFT size (minimum 512), t: threshold in negative dB,
  # nH: maximum number of harmonics, minf0: minimum f0 frequency in Hz,
  # maxf0: maximim f0 frequency in Hz,
  # f0et: error threshold in the f0 detection (ex: 5),
  # maxhd: max. relative deviation in harmonic detection (ex: .2)
  # stocf: decimation factor of mag spectrum for stochastic analysis
  # maxnpeaksTwm: maximum number of peaks used for F0 detection
  # y: output sound, yh: harmonic component, ys: stochastic component

  hN = N/2                                                      # size of positive spectrum
  hM1 = int(math.floor((w.size+1)/2))                           # half analysis window size by rounding
  hM2 = int(math.floor(w.size/2))                               # half analysis window size by floor
  Ns = 512                                                      # FFT size for synthesis
  H = Ns/4                                                      # Hop size used for analysis and synthesis
  hNs = Ns/2                                                    # half of FFT size for synthesis
  pin = max(hNs, hM1)                                           # initialize sound pointer in middle of analysis window
  pend = x.size - max(hNs, hM1)                                 # last sample to start a frame
  fftbuffer = np.zeros(N)                                       # initialize buffer for FFT
  yhw = np.zeros(Ns)                                            # initialize output sound frame
  ysw = np.zeros(Ns)                                            # initialize output sound frame
  yh = np.zeros(x.size)                                         # initialize output array
  ys = np.zeros(x.size)                                         # initialize output array
  w = w / sum(w)                                                # normalize analysis window
  sw = np.zeros(Ns)
  ow = triang(2*H)                                              # overlapping window
  sw[hNs-H:hNs+H] = ow
  bh = blackmanharris(Ns)                                       # synthesis window
  bh = bh / sum(bh)                                             # normalize synthesis window
  wr = bh                                                       # window for residual
  sw[hNs-H:hNs+H] = sw[hNs-H:hNs+H] / bh[hNs-H:hNs+H]
  sws = H*hanning(Ns)/2                                         # synthesis window for stochastic
  lastyhloc = np.zeros(nH)                                      # initialize synthesis harmonic locations
  yhphase = 2*np.pi * np.random.rand(nH)                        # initialize synthesis harmonic phases

  while pin<pend:
  #-----analysis-----
    xw = x[pin-hM1:pin+hM2] * w                                  # window the input sound
    fftbuffer = np.zeros(N)                                      # reset buffer
    fftbuffer[:hM1] = xw[hM2:]                                   # zero-phase window in fftbuffer
    fftbuffer[N-hM2:] = xw[:hM2]
    X = fft(fftbuffer)                                           # compute FFT
    mX = 20 * np.log10(abs(X[:hN]))                              # magnitude spectrum of positive frequencies
    ploc = PP.peakDetection(mX, hN, t)                           # detect spectral peaks
    pX = np.unwrap(np.angle(X[:hN]))                             # unwrapped phase spect. of positive freq.
    iploc, ipmag, ipphase = PP.peakInterp(mX, pX, ploc)          # refine peak values
    f0 = fd.f0DetectionTwm(iploc, ipmag, N, fs, f0et, minf0, maxf0)  # find f0
    hloc = np.zeros(nH)                                          # initialize harmonic locations
    hmag = np.zeros(nH)-100                                      # initialize harmonic magnitudes
    hphase = np.zeros(nH)                                        # initialize harmonic phases
    hf = (f0>0) * (f0*np.arange(1, nH+1))                        # initialize harmonic frequencies
    hi = 0                                                       # initialize harmonic index
    npeaks = ploc.size                                           # number of peaks found
    while f0>0 and hi<nH and hf[hi]<fs/2 :                       # find harmonic peaks
      dev = min(abs(iploc/N*fs - hf[hi]))
      pei = np.argmin(abs(iploc/N*fs - hf[hi]))                  # closest peak
      if ( hi==0 or not any(hloc[:hi]==iploc[pei]) ) and dev<maxhd*hf[hi] :
        hloc[hi] = iploc[pei]                                    # harmonic locations
        hmag[hi] = ipmag[pei]                                    # harmonic magnitudes
        hphase[hi] = ipphase[pei]                                # harmonic phases
      hi += 1                                                    # increase harmonic index
    hloc = (hloc!=0) * (hloc*Ns/N)                               # synth. locs

    ri = pin-hNs-1                                               # input sound pointer for residual analysis
    xw2 = x[ri:ri+Ns]*wr                                         # window the input sound
    fftbuffer = np.zeros(Ns)                                     # reset buffer
    fftbuffer[:hNs] = xw2[hNs:]                                  # zero-phase window in fftbuffer
    fftbuffer[hNs:] = xw2[:hNs]
    X2 = fft(fftbuffer)                                          # compute FFT for residual analysis
    Xh = GS.genSpecSines(hloc, hmag, hphase, Ns)                 # generate sines
    Xr = X2-Xh                                                   # get the residual complex spectrum
    mXr = 20 * np.log10(abs(Xr[:hNs]))                           # magnitude spectrum of residual
    mXrenv = resample(np.maximum(-200, mXr), mXr.size*stocf)     # decimate the magnitude spectrum and avoid -Inf

  #-----synthesis data-----
    yhloc = hloc                                                 # synthesis harmonics locs
    yhmag = hmag                                                 # synthesis harmonic amplitudes
    mYrenv = mXrenv                                              # synthesis residual envelope
    yf0 = f0                                                     # synthesis fundamental frequency
  #------transformations----
    #-----clarinet effect, only odd harmonics-----
    # yhmag[1::2] = 0											 # set even harmonic to 0 magnitude

  #-----pitch discretization to temperate scale-----
    # if f0>0:
    #  nst = round(12*np.log2(f0/55))                            # closest semitone
    #  discpitch = 55*2**(nst/12)                                # discretized pitch
    #  fscale = discpitch/f0                                     # pitch transposition factor
    #  yhloc = yhloc*fscale                                      # all harmonic corrected to discretized pitch

  #-----pitch transposition with timbre preseervation -----

    # fscale = 2                                                 # scale factor for pitch transposition
    # ind_valid = np.where(yhloc!=0)[0]                          # using only those harmonic indices which have non zero frequency values
    # if (f0>0):
    #     x_vals = np.append(np.append(0, yhloc[ind_valid]),hNs)      # values of peak locations to be considered for interpolation
    #    y_vals = np.append(np.append(yhmag[0], yhmag[ind_valid]),yhmag[-1])     # values of peak magnitudes to be considered for interpolation
    #     specEnvelope = interp1d(x_vals, y_vals, kind = 'linear',bounds_error=False, fill_value=-100)
    #     yhloc = yhloc*fscale
    #     yhmag[ind_valid] = specEnvelope(yhloc[ind_valid])

  #----- Pitch transposition, Vibrato and tremolo with timbre preseervation -----
    # vtf = 5.0;                                                  # vibrato-tremolo frequency in Hz
    # vd  = 50;                                                   # vibrato depth in cents
    # td  = 3;                                                    # tremolo depth in dB
    # fscale = 1                                                  # scale factor for pitch transposition
    # modf = np.sin(2.0*np.pi*vtf*pin/fs)                         # modulation factor for both vibrato and tremolo (which has to be scaled later)
    # sfscale = fscale*(2.0**(vd/1200.0*modf))                    # affective scale factor together with vibrato affect
    # idx = np.where(yhloc!=0)[0]                                 # using only those harmonic indices which have non zero frequency values
    # if (f0>0):
    #     x_vals = np.append(np.append(0, yhloc[idx]),hNs)        # values of peak locations to be considered for interpolation
    #     y_vals = np.append(np.append(yhmag[0], yhmag[idx]),yhmag[-1])     # values of peak magnitudes to be considered for interpolation
    #     specEnvelope = interp1d(x_vals, y_vals, kind = 'linear',bounds_error=False, fill_value=-100)
    #     yhloc = yhloc*sfscale
    #     yhmag[idx] = specEnvelope(yhloc[idx])
    #     yhmag[idx] = yhmag[idx] + td*modf                       # tremolo

  #-----synthesis-----
    yhphase += 2*np.pi * (lastyhloc+yhloc)/2/Ns*H                # propagate phases
    lastyhloc = yhloc
    Yh = GS.genSpecSines(yhloc, yhmag, yhphase, Ns)              # generate spec sines
    mYs = resample(mYrenv, hNs)                                  # interpolate to original size
    mYs = 10**(mYs/20)                                           # dB to linear magnitude
    if f0>0:
      mYs *= np.cos(np.pi*np.arange(0, hNs)/Ns*fs/yf0)**2        # filter residual
    fc = 1+round(500.0/fs*Ns)                                    # 500 Hz
    mYs[:fc] *= (np.arange(0, fc)/(fc-1))**2                     # HPF
    pYs = 2*np.pi * np.random.rand(hNs)                          # generate phase random values

    Ys = np.zeros(Ns, dtype = complex)
    Ys[:hNs] = mYs * np.exp(1j*pYs)                              # generate positive freq.
    Ys[hNs+1:] = mYs[:0:-1] * np.exp(-1j*pYs[:0:-1])             # generate negative freq.

    fftbuffer = np.zeros(Ns)
    fftbuffer = np.real(ifft(Yh))                                # inverse FFT of harmonic spectrum
    yhw[:hNs-1] = fftbuffer[hNs+1:]                              # undo zer-phase window
    yhw[hNs-1:] = fftbuffer[:hNs+1]

    fftbuffer = np.zeros(Ns)
    fftbuffer = np.real(ifft(Ys))                                # inverse FFT of stochastic approximation spectrum
    ysw[:hNs-1] = fftbuffer[hNs+1:]                              # undo zero-phase window
    ysw[hNs-1:] = fftbuffer[:hNs+1]

    yh[ri:ri+Ns] += sw*yhw                                       # overlap-add for sines
    ys[ri:ri+Ns] += sws*ysw                                      # overlap-add for stoch
    pin += H                                                     # advance sound pointer

  y = yh+ys                                                      # sum harmonic and stochastic components
  return y, yh, ys

def defaultTest():
    str_time = time.time()
    (fs, x) = wp.wavread(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../sounds/sax-phrase-short.wav'))
    w = np.blackman(801)
    N = 1024
    t = -90
    nH = 50
    minf0 = 350
    maxf0 = 700
    f0et = 10
    maxhd = 0.2
    stocf = 0.5
    maxnpeaksTwm = 5
    y, yh, ys = hps(x, fs, w, N, t, nH, minf0, maxf0, f0et, maxhd, stocf, maxnpeaksTwm)
    print "time taken for computation " + str(time.time()-str_time)

if __name__ == '__main__':
    (fs, x) = wp.wavread(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../sounds/sax-phrase-short.wav'))
    w = np.blackman(801)
    N = 1024
    t = -90
    nH = 50
    minf0 = 350
    maxf0 = 700
    f0et = 10
    maxhd = 0.2
    stocf = 0.5
    maxnpeaksTwm = 5
    y, yh, ys = hps(x, fs, w, N, t, nH, minf0, maxf0, f0et, maxhd, stocf, maxnpeaksTwm)

    wp.play(y, fs)
    wp.play(yh, fs)
    wp.play(ys, fs)<commit_msg>changed transformations<commit_after>import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import hamming, hanning, triang, blackmanharris, resample
from scipy.fftpack import fft, ifft, fftshift
from scipy.interpolate import interp1d
import math
import sys, os, time

sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../basicFunctions/'))
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../basicFunctions_C/'))

import smsF0DetectionTwm as fd
import smsWavplayer as wp
import smsPeakProcessing as PP

try:
  import basicFunctions_C as GS
except ImportError:
  import smsGenSpecSines as GS
  print "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"
  print "NOTE: Cython modules for some functions were not imported, the processing will be slow"
  print "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"

def hps(x, fs, w, N, t, nH, minf0, maxf0, f0et, maxhd, stocf, maxnpeaksTwm=10):
  # Analysis/synthesis of a sound using the harmonic plus stochastic model, prepared for transformations
  # x: input sound, fs: sampling rate, w: analysis window,
  # N: FFT size (minimum 512), t: threshold in negative dB,
  # nH: maximum number of harmonics, minf0: minimum f0 frequency in Hz,
  # maxf0: maximim f0 frequency in Hz,
  # f0et: error threshold in the f0 detection (ex: 5),
  # maxhd: max. relative deviation in harmonic detection (ex: .2)
  # stocf: decimation factor of mag spectrum for stochastic analysis
  # maxnpeaksTwm: maximum number of peaks used for F0 detection
  # y: output sound, yh: harmonic component, ys: stochastic component

  hN = N/2                                                      # size of positive spectrum
  hM1 = int(math.floor((w.size+1)/2))                           # half analysis window size by rounding
  hM2 = int(math.floor(w.size/2))                               # half analysis window size by floor
  Ns = 512                                                      # FFT size for synthesis
  H = Ns/4                                                      # Hop size used for analysis and synthesis
  hNs = Ns/2                                                    # half of FFT size for synthesis
  pin = max(hNs, hM1)                                           # initialize sound pointer in middle of analysis window
  pend = x.size - max(hNs, hM1)                                 # last sample to start a frame
  fftbuffer = np.zeros(N)                                       # initialize buffer for FFT
  yhw = np.zeros(Ns)                                            # initialize output sound frame
  ysw = np.zeros(Ns)                                            # initialize output sound frame
  yh = np.zeros(x.size)                                         # initialize output array
  ys = np.zeros(x.size)                                         # initialize output array
  w = w / sum(w)                                                # normalize analysis window
  sw = np.zeros(Ns)
  ow = triang(2*H)                                              # overlapping window
  sw[hNs-H:hNs+H] = ow
  bh = blackmanharris(Ns)                                       # synthesis window
  bh = bh / sum(bh)                                             # normalize synthesis window
  wr = bh                                                       # window for residual
  sw[hNs-H:hNs+H] = sw[hNs-H:hNs+H] / bh[hNs-H:hNs+H]
  sws = H*hanning(Ns)/2                                         # synthesis window for stochastic
  lastyhloc = np.zeros(nH)                                      # initialize synthesis harmonic locations
  yhphase = 2*np.pi * np.random.rand(nH)                        # initialize synthesis harmonic phases

  while pin<pend:
  #-----analysis-----
    xw = x[pin-hM1:pin+hM2] * w                                  # window the input sound
    fftbuffer = np.zeros(N)                                      # reset buffer
    fftbuffer[:hM1] = xw[hM2:]                                   # zero-phase window in fftbuffer
    fftbuffer[N-hM2:] = xw[:hM2]
    X = fft(fftbuffer)                                           # compute FFT
    mX = 20 * np.log10(abs(X[:hN]))                              # magnitude spectrum of positive frequencies
    ploc = PP.peakDetection(mX, hN, t)                           # detect spectral peaks
    pX = np.unwrap(np.angle(X[:hN]))                             # unwrapped phase spect. of positive freq.
    iploc, ipmag, ipphase = PP.peakInterp(mX, pX, ploc)          # refine peak values
    f0 = fd.f0DetectionTwm(iploc, ipmag, N, fs, f0et, minf0, maxf0)  # find f0
    hloc = np.zeros(nH)                                          # initialize harmonic locations
    hmag = np.zeros(nH)-100                                      # initialize harmonic magnitudes
    hphase = np.zeros(nH)                                        # initialize harmonic phases
    hf = (f0>0) * (f0*np.arange(1, nH+1))                        # initialize harmonic frequencies
    hi = 0                                                       # initialize harmonic index
    npeaks = ploc.size                                           # number of peaks found
    while f0>0 and hi<nH and hf[hi]<fs/2 :                       # find harmonic peaks
      dev = min(abs(iploc/N*fs - hf[hi]))
      pei = np.argmin(abs(iploc/N*fs - hf[hi]))                  # closest peak
      if ( hi==0 or not any(hloc[:hi]==iploc[pei]) ) and dev<maxhd*hf[hi] :
        hloc[hi] = iploc[pei]                                    # harmonic locations
        hmag[hi] = ipmag[pei]                                    # harmonic magnitudes
        hphase[hi] = ipphase[pei]                                # harmonic phases
      hi += 1                                                    # increase harmonic index
    hloc = (hloc!=0) * (hloc*Ns/N)                               # synth. locs

    ri = pin-hNs-1                                               # input sound pointer for residual analysis
    xw2 = x[ri:ri+Ns]*wr                                         # window the input sound
    fftbuffer = np.zeros(Ns)                                     # reset buffer
    fftbuffer[:hNs] = xw2[hNs:]                                  # zero-phase window in fftbuffer
    fftbuffer[hNs:] = xw2[:hNs]
    X2 = fft(fftbuffer)                                          # compute FFT for residual analysis
    Xh = GS.genSpecSines(hloc, hmag, hphase, Ns)                 # generate sines
    Xr = X2-Xh                                                   # get the residual complex spectrum
    mXr = 20 * np.log10(abs(Xr[:hNs]))                           # magnitude spectrum of residual
    mXrenv = resample(np.maximum(-200, mXr), mXr.size*stocf)     # decimate the magnitude spectrum and avoid -Inf

  #-----synthesis data-----
    yhloc = hloc                                                 # synthesis harmonics locs
    yhmag = hmag                                                 # synthesis harmonic amplitudes
    mYrenv = mXrenv                                              # synthesis residual envelope
    yf0 = f0                                                     # synthesis fundamental frequency
  #------transformations----
    #-----clarinet effect, only odd harmonics-----
    # yhmag[1::2] = 0											 # set even harmonic to 0 magnitude

  #-----pitch discretization to temperate scale-----
    # if f0>0:
    #  nst = round(12*np.log2(f0/55))                            # closest semitone
    #  discpitch = 55*2**(nst/12)                                # discretized pitch
    #  fscale = discpitch/f0                                     # pitch transposition factor
    #  yhloc = yhloc*fscale                                      # all harmonic corrected to discretized pitch

  #-----pitch transposition with timbre preseervation -----

    # fscale = .5                                                # scale factor for pitch transposition
    # ind_valid = np.where(yhloc!=0)[0]                          # using only those harmonic indices which have non zero frequency values
    # if (f0>0):
    #     x_vals = np.append(np.append(0, yhloc[ind_valid]),hNs)      # values of peak locations to be considered for interpolation
    #     y_vals = np.append(np.append(yhmag[0], yhmag[ind_valid]),yhmag[-1])     # values of peak magnitudes to be considered for interpolation
    #     specEnvelope = interp1d(x_vals, y_vals, kind = 'linear',bounds_error=False, fill_value=-100)
    #     yhloc = yhloc*fscale
    #     yhmag[ind_valid] = specEnvelope(yhloc[ind_valid])

  #----- Pitch transposition, Vibrato and tremolo with timbre preseervation -----
    # vtf = 5.0;                                                  # vibrato-tremolo frequency in Hz
    # vd  = 50;                                                   # vibrato depth in cents
    # td  = 3;                                                    # tremolo depth in dB
    # fscale = 1                                                  # scale factor for pitch transposition
    # modf = np.sin(2.0*np.pi*vtf*pin/fs)                         # modulation factor for both vibrato and tremolo (which has to be scaled later)
    # sfscale = fscale*(2.0**(vd/1200.0*modf))                    # affective scale factor together with vibrato affect
    # idx = np.where(yhloc!=0)[0]                                 # using only those harmonic indices which have non zero frequency values
    # if (f0>0):
    #     x_vals = np.append(np.append(0, yhloc[idx]),hNs)        # values of peak locations to be considered for interpolation
    #     y_vals = np.append(np.append(yhmag[0], yhmag[idx]),yhmag[-1])     # values of peak magnitudes to be considered for interpolation
    #     specEnvelope = interp1d(x_vals, y_vals, kind = 'linear',bounds_error=False, fill_value=-100)
    #     yhloc = yhloc*sfscale
    #     yhmag[idx] = specEnvelope(yhloc[idx])
    #     yhmag[idx] = yhmag[idx] + td*modf                       # tremolo

  #-----synthesis-----
    yhphase += 2*np.pi * (lastyhloc+yhloc)/2/Ns*H                # propagate phases
    lastyhloc = yhloc
    Yh = GS.genSpecSines(yhloc, yhmag, yhphase, Ns)              # generate spec sines
    mYs = resample(mYrenv, hNs)                                  # interpolate to original size
    mYs = 10**(mYs/20)                                           # dB to linear magnitude
    if f0>0:
      mYs *= np.cos(np.pi*np.arange(0, hNs)/Ns*fs/yf0)**2        # filter residual
    fc = 1+round(500.0/fs*Ns)                                    # 500 Hz
    mYs[:fc] *= (np.arange(0, fc)/(fc-1))**2                     # HPF
    pYs = 2*np.pi * np.random.rand(hNs)                          # generate phase random values

    Ys = np.zeros(Ns, dtype = complex)
    Ys[:hNs] = mYs * np.exp(1j*pYs)                              # generate positive freq.
    Ys[hNs+1:] = mYs[:0:-1] * np.exp(-1j*pYs[:0:-1])             # generate negative freq.

    fftbuffer = np.zeros(Ns)
    fftbuffer = np.real(ifft(Yh))                                # inverse FFT of harmonic spectrum
    yhw[:hNs-1] = fftbuffer[hNs+1:]                              # undo zer-phase window
    yhw[hNs-1:] = fftbuffer[:hNs+1]

    fftbuffer = np.zeros(Ns)
    fftbuffer = np.real(ifft(Ys))                                # inverse FFT of stochastic approximation spectrum
    ysw[:hNs-1] = fftbuffer[hNs+1:]                              # undo zero-phase window
    ysw[hNs-1:] = fftbuffer[:hNs+1]

    yh[ri:ri+Ns] += sw*yhw                                       # overlap-add for sines
    ys[ri:ri+Ns] += sws*ysw                                      # overlap-add for stoch
    pin += H                                                     # advance sound pointer

  y = yh+ys                                                      # sum harmonic and stochastic components
  return y, yh, ys

def defaultTest():
    str_time = time.time()
    (fs, x) = wp.wavread(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../sounds/sax-phrase-short.wav'))
    w = np.blackman(801)
    N = 1024
    t = -90
    nH = 50
    minf0 = 350
    maxf0 = 700
    f0et = 10
    maxhd = 0.2
    stocf = 0.5
    maxnpeaksTwm = 5
    y, yh, ys = hps(x, fs, w, N, t, nH, minf0, maxf0, f0et, maxhd, stocf, maxnpeaksTwm)
    print "time taken for computation " + str(time.time()-str_time)

if __name__ == '__main__':
    (fs, x) = wp.wavread(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../sounds/speech-female.wav'))
    w = np.blackman(901)
    N = 1024
    t = -90
    nH = 40
    minf0 = 120
    maxf0 = 400
    f0et = 5
    maxhd = 0.2
    stocf = 0.5
    maxnpeaksTwm = 10
    y, yh, ys = hps(x, fs, w, N, t, nH, minf0, maxf0, f0et, maxhd, stocf, maxnpeaksTwm)

    wp.play(y, fs)
    wp.play(yh, fs)
    wp.play(ys, fs)<|endoftext|>